Commit 71d24109 authored by Jacob Vosmaer's avatar Jacob Vosmaer

Merge branch 'fj-detect-content-headers' into 'master'

Add proxy layer to calculate content type and disposition headers

See merge request gitlab-org/gitlab-workhorse!335
parents 47f0e667 b82d4a4a
package headers
import (
"mime"
"net/http"
"regexp"
svg "github.com/h2non/go-is-svg"
)
var (
ImageTypeRegex = regexp.MustCompile(`^image/*`)
SvgMimeTypeRegex = regexp.MustCompile(`^image/svg\+xml$`)
TextTypeRegex = regexp.MustCompile(`^text/*`)
VideoTypeRegex = regexp.MustCompile(`^video/*`)
AttachmentRegex = regexp.MustCompile(`^attachment`)
)
// Mime types that can't be inlined. Usually subtypes of main types
var forbiddenInlineTypes = []*regexp.Regexp{SvgMimeTypeRegex}
// Mime types that can be inlined. We can add global types like "image/" or
// specific types like "text/plain". If there is a specific type inside a global
// allowed type that can't be inlined we must add it to the forbiddenInlineTypes var.
// One example of this is the mime type "image". We allow all images to be
// inlined except for SVGs.
var allowedInlineTypes = []*regexp.Regexp{ImageTypeRegex, TextTypeRegex, VideoTypeRegex}
func SafeContentHeaders(data []byte, contentDisposition string) (string, string) {
contentType := safeContentType(data)
contentDisposition = safeContentDisposition(contentType, contentDisposition)
return contentType, contentDisposition
}
func safeContentType(data []byte) string {
// Special case for svg because DetectContentType detects it as text
if svg.Is(data) {
return "image/svg+xml"
}
// Override any existing Content-Type header from other ResponseWriters
contentType := http.DetectContentType(data)
// If the content is text type, we set to plain, because we don't
// want to render it inline if they're html or javascript
if isType(contentType, TextTypeRegex) {
return "text/plain; charset=utf-8"
}
return contentType
}
func safeContentDisposition(contentType string, contentDisposition string) string {
existingDisposition, file := extractContentDispositionFile(contentDisposition)
// If the existing disposition is attachment we return that. This allow us
// to force a download from GitLab (ie: RawController)
if AttachmentRegex.MatchString(existingDisposition) {
return attachmentDisposition(file)
}
// Checks for mime types that are forbidden to be inline
for _, element := range forbiddenInlineTypes {
if isType(contentType, element) {
return attachmentDisposition(file)
}
}
// Checks for mime types allowed to be inline
for _, element := range allowedInlineTypes {
if isType(contentType, element) {
return inlineDisposition(file)
}
}
// Anything else is set to attachment
return attachmentDisposition(file)
}
func extractContentDispositionFile(disposition string) (string, string) {
if disposition == "" {
return "", ""
}
existingDisposition, params, err := mime.ParseMediaType(disposition)
if err != nil {
return "", ""
}
return existingDisposition, params["filename"]
}
func attachmentDisposition(file string) string {
return disposition("attachment", file)
}
func inlineDisposition(file string) string {
return disposition("inline", file)
}
func disposition(disposition string, file string) string {
params := map[string]string{}
if file != "" {
params["filename"] = file
}
return mime.FormatMediaType(disposition, params)
}
func isType(contentType string, mimeType *regexp.Regexp) bool {
return mimeType.MatchString(contentType)
}
package headers
import (
"net/http"
"strconv"
)
// Max number of bytes that http.DetectContentType needs to get the content type
const MaxDetectSize = 512
// HTTP Headers
const (
ContentDispositionHeader = "Content-Disposition"
ContentTypeHeader = "Content-Type"
// Workhorse related headers
GitlabWorkhorseSendDataHeader = "Gitlab-Workhorse-Send-Data"
XSendFileHeader = "X-Sendfile"
XSendFileTypeHeader = "X-Sendfile-Type"
// Signal header that indicates Workhorse should detect and set the content headers
GitlabWorkhorseDetectContentTypeHeader = "Gitlab-Workhorse-Detect-Content-Type"
)
var ResponseHeaders = []string{
XSendFileHeader,
GitlabWorkhorseSendDataHeader,
GitlabWorkhorseDetectContentTypeHeader,
}
func IsDetectContentTypeHeaderPresent(rw http.ResponseWriter) bool {
header, err := strconv.ParseBool(rw.Header().Get(GitlabWorkhorseDetectContentTypeHeader))
if err != nil || !header {
return false
}
return true
}
// AnyResponseHeaderPresent checks in the ResponseWriter if there is any Response Header
func AnyResponseHeaderPresent(rw http.ResponseWriter) bool {
// If this header is not present means that we want the old behavior
if !IsDetectContentTypeHeaderPresent(rw) {
return false
}
for _, header := range ResponseHeaders {
if rw.Header().Get(header) != "" {
return true
}
}
return false
}
// RemoveResponseHeaders removes any ResponseHeader from the ResponseWriter
func RemoveResponseHeaders(rw http.ResponseWriter) {
for _, header := range ResponseHeaders {
rw.Header().Del(header)
}
}
package headers
import (
"net/http/httptest"
"testing"
"github.com/stretchr/testify/require"
)
func TestIsDetectContentTypeHeaderPresent(t *testing.T) {
rw := httptest.NewRecorder()
rw.Header().Del(GitlabWorkhorseDetectContentTypeHeader)
require.Equal(t, false, IsDetectContentTypeHeaderPresent(rw))
rw.Header().Set(GitlabWorkhorseDetectContentTypeHeader, "true")
require.Equal(t, true, IsDetectContentTypeHeaderPresent(rw))
rw.Header().Set(GitlabWorkhorseDetectContentTypeHeader, "false")
require.Equal(t, false, IsDetectContentTypeHeaderPresent(rw))
rw.Header().Set(GitlabWorkhorseDetectContentTypeHeader, "foobar")
require.Equal(t, false, IsDetectContentTypeHeaderPresent(rw))
}
package contentprocessor
import (
"bytes"
"io"
"net/http"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/headers"
)
type contentDisposition struct {
rw http.ResponseWriter
buf *bytes.Buffer
wroteHeader bool
flushed bool
active bool
removedResponseHeaders bool
status int
sentStatus bool
}
// SetContentHeaders buffers the response if Gitlab-Workhorse-Detect-Content-Type
// header is found and set the proper content headers based on the current
// value of content type and disposition
func SetContentHeaders(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
cd := &contentDisposition{
rw: w,
buf: &bytes.Buffer{},
status: http.StatusOK,
}
defer cd.flush()
h.ServeHTTP(cd, r)
})
}
func (cd *contentDisposition) Header() http.Header {
return cd.rw.Header()
}
func (cd *contentDisposition) Write(data []byte) (int, error) {
// Normal write if we don't need to buffer
if cd.isUnbuffered() {
cd.WriteHeader(cd.status)
return cd.rw.Write(data)
}
// Write the new data into the buffer
n, _ := cd.buf.Write(data)
// If we have enough data to calculate the content headers then flush the Buffer
var err error
if cd.buf.Len() >= headers.MaxDetectSize {
err = cd.flushBuffer()
}
return n, err
}
func (cd *contentDisposition) flushBuffer() error {
if cd.isUnbuffered() {
return nil
}
cd.flushed = true
// If the buffer has any content then we calculate the content headers and
// write in the response
if cd.buf.Len() > 0 {
cd.writeContentHeaders()
cd.WriteHeader(cd.status)
_, err := io.Copy(cd.rw, cd.buf)
return err
}
// If no content is present in the buffer we still need to send the headers
cd.WriteHeader(cd.status)
return nil
}
func (cd *contentDisposition) writeContentHeaders() {
if cd.wroteHeader {
return
}
cd.wroteHeader = true
contentType, contentDisposition := headers.SafeContentHeaders(cd.buf.Bytes(), cd.Header().Get(headers.ContentDispositionHeader))
cd.Header().Set(headers.ContentTypeHeader, contentType)
cd.Header().Set(headers.ContentDispositionHeader, contentDisposition)
}
func (cd *contentDisposition) WriteHeader(status int) {
if cd.sentStatus {
return
}
cd.status = status
if cd.isUnbuffered() {
cd.rw.WriteHeader(cd.status)
cd.sentStatus = true
}
}
// If we find any response header, then we must calculate the content headers
// If we don't find any, the data is not buffered and it works as
// a usual ResponseWriter
func (cd *contentDisposition) isUnbuffered() bool {
if !cd.removedResponseHeaders {
if headers.IsDetectContentTypeHeaderPresent(cd.rw) {
cd.active = true
}
cd.removedResponseHeaders = true
// We ensure to clear any response header from the response
headers.RemoveResponseHeaders(cd.rw)
}
return cd.flushed || !cd.active
}
func (cd *contentDisposition) flush() {
cd.flushBuffer()
}
package contentprocessor
import (
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/headers"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
"github.com/stretchr/testify/require"
)
func TestFailSetContentTypeAndDisposition(t *testing.T) {
testCaseBody := "Hello world!"
h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, err := io.WriteString(w, testCaseBody)
require.NoError(t, err)
})
resp := makeRequest(t, h, testCaseBody)
require.Equal(t, "", resp.Header.Get(headers.ContentDispositionHeader))
require.Equal(t, "", resp.Header.Get(headers.ContentTypeHeader))
}
func TestSuccessSetContentTypeAndDispositionFeatureEnabled(t *testing.T) {
testCaseBody := "Hello world!"
resp := makeRequest(t, nil, testCaseBody)
require.Equal(t, "inline", resp.Header.Get(headers.ContentDispositionHeader))
require.Equal(t, "text/plain; charset=utf-8", resp.Header.Get(headers.ContentTypeHeader))
}
func TestSetProperContentTypeAndDisposition(t *testing.T) {
testCases := []struct {
desc string
contentType string
contentDisposition string
body string
}{
{
desc: "text type",
contentType: "text/plain; charset=utf-8",
contentDisposition: "inline",
body: "Hello world!",
},
{
desc: "HTML type",
contentType: "text/plain; charset=utf-8",
contentDisposition: "inline",
body: "<html><body>Hello world!</body></html>",
},
{
desc: "Javascript type",
contentType: "text/plain; charset=utf-8",
contentDisposition: "inline",
body: "<script>alert(\"foo\")</script>",
},
{
desc: "Image type",
contentType: "image/png",
contentDisposition: "inline",
body: testhelper.LoadFile(t, "testdata/image.png"),
},
{
desc: "SVG type",
contentType: "image/svg+xml",
contentDisposition: "attachment",
body: testhelper.LoadFile(t, "testdata/image.svg"),
},
{
desc: "Application type",
contentType: "application/pdf",
contentDisposition: "attachment",
body: testhelper.LoadFile(t, "testdata/file.pdf"),
},
{
desc: "Application executable type",
contentType: "application/octet-stream",
contentDisposition: "attachment",
body: testhelper.LoadFile(t, "testdata/file.swf"),
},
{
desc: "Video type",
contentType: "video/mp4",
contentDisposition: "inline",
body: testhelper.LoadFile(t, "testdata/video.mp4"),
},
{
desc: "Audio type",
contentType: "audio/mpeg",
contentDisposition: "attachment",
body: testhelper.LoadFile(t, "testdata/audio.mp3"),
},
{
desc: "JSON type",
contentType: "text/plain; charset=utf-8",
contentDisposition: "inline",
body: "{ \"glossary\": { \"title\": \"example glossary\", \"GlossDiv\": { \"title\": \"S\" } } }",
},
{
desc: "Forged file with png extension but SWF content",
contentType: "application/octet-stream",
contentDisposition: "attachment",
body: testhelper.LoadFile(t, "testdata/forgedfile.png"),
},
{
desc: "BMPR file",
contentType: "application/octet-stream",
contentDisposition: "attachment",
body: testhelper.LoadFile(t, "testdata/file.bmpr"),
},
{
desc: "STL file",
contentType: "application/octet-stream",
contentDisposition: "attachment",
body: testhelper.LoadFile(t, "testdata/file.stl"),
},
{
desc: "RDoc file",
contentType: "text/plain; charset=utf-8",
contentDisposition: "inline",
body: testhelper.LoadFile(t, "testdata/file.rdoc"),
},
{
desc: "IPYNB file",
contentType: "text/plain; charset=utf-8",
contentDisposition: "inline",
body: testhelper.LoadFile(t, "testdata/file.ipynb"),
},
{
desc: "Sketch file",
contentType: "application/zip",
contentDisposition: "attachment",
body: testhelper.LoadFile(t, "testdata/file.sketch"),
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
resp := makeRequest(t, nil, tc.body)
require.Equal(t, tc.contentType, resp.Header.Get(headers.ContentTypeHeader))
require.Equal(t, tc.contentDisposition, resp.Header.Get(headers.ContentDispositionHeader))
})
}
}
func TestFailOverrideContentType(t *testing.T) {
testCase := struct {
contentType string
body string
}{
contentType: "text/plain; charset=utf-8",
body: "<html><body>Hello world!</body></html>",
}
h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
// We are pretending to be upstream or an inner layer of the ResponseWriter chain
w.Header().Set(headers.GitlabWorkhorseDetectContentTypeHeader, "true")
w.Header().Set(headers.ContentTypeHeader, "text/html; charset=utf-8")
_, err := io.WriteString(w, testCase.body)
require.NoError(t, err)
})
resp := makeRequest(t, h, testCase.body)
require.Equal(t, testCase.contentType, resp.Header.Get(headers.ContentTypeHeader))
}
func TestSuccessOverrideContentDispositionFromInlineToAttachment(t *testing.T) {
testCaseBody := "Hello world!"
h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
// We are pretending to be upstream or an inner layer of the ResponseWriter chain
w.Header().Set(headers.ContentDispositionHeader, "attachment")
w.Header().Set(headers.GitlabWorkhorseDetectContentTypeHeader, "true")
_, err := io.WriteString(w, testCaseBody)
require.NoError(t, err)
})
resp := makeRequest(t, h, testCaseBody)
require.Equal(t, "attachment", resp.Header.Get(headers.ContentDispositionHeader))
}
func TestFailOverrideContentDispositionFromAttachmentToInline(t *testing.T) {
testCaseBody := testhelper.LoadFile(t, "testdata/file.pdf")
h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
// We are pretending to be upstream or an inner layer of the ResponseWriter chain
w.Header().Set(headers.ContentDispositionHeader, "inline")
w.Header().Set(headers.GitlabWorkhorseDetectContentTypeHeader, "true")
_, err := io.WriteString(w, testCaseBody)
require.NoError(t, err)
})
resp := makeRequest(t, h, testCaseBody)
require.Equal(t, "attachment", resp.Header.Get(headers.ContentDispositionHeader))
}
func TestHeadersDelete(t *testing.T) {
for _, code := range []int{200, 400} {
recorder := httptest.NewRecorder()
rw := &contentDisposition{rw: recorder}
for _, name := range headers.ResponseHeaders {
rw.Header().Set(name, "foobar")
}
rw.WriteHeader(code)
for _, name := range headers.ResponseHeaders {
if header := recorder.Header().Get(name); header != "" {
t.Fatalf("HTTP %d response: expected header to be empty, found %q", code, name)
}
}
}
}
func TestWriteHeadersCalledOnce(t *testing.T) {
recorder := httptest.NewRecorder()
rw := &contentDisposition{rw: recorder}
rw.WriteHeader(400)
require.Equal(t, 400, rw.status)
require.Equal(t, true, rw.sentStatus)
rw.WriteHeader(200)
require.Equal(t, 400, rw.status)
}
func makeRequest(t *testing.T, handler http.HandlerFunc, body string) *http.Response {
if handler == nil {
handler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
// We are pretending to be upstream
w.Header().Set(headers.GitlabWorkhorseDetectContentTypeHeader, "true")
_, err := io.WriteString(w, body)
require.NoError(t, err)
})
}
req, _ := http.NewRequest("GET", "/", nil)
rw := httptest.NewRecorder()
SetContentHeaders(handler).ServeHTTP(rw, req)
resp := rw.Result()
respBody, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.Equal(t, body, string(respBody))
return resp
}
...@@ -15,8 +15,6 @@ type Injecter interface { ...@@ -15,8 +15,6 @@ type Injecter interface {
type Prefix string type Prefix string
const HeaderKey = "Gitlab-Workhorse-Send-Data"
func (p Prefix) Match(s string) bool { func (p Prefix) Match(s string) bool {
return strings.HasPrefix(s, string(p)) return strings.HasPrefix(s, string(p))
} }
......
...@@ -3,9 +3,11 @@ package senddata ...@@ -3,9 +3,11 @@ package senddata
import ( import (
"net/http" "net/http"
"github.com/prometheus/client_golang/prometheus" "gitlab.com/gitlab-org/gitlab-workhorse/internal/headers"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata/contentprocessor"
"github.com/prometheus/client_golang/prometheus"
) )
var ( var (
...@@ -39,7 +41,7 @@ type sendDataResponseWriter struct { ...@@ -39,7 +41,7 @@ type sendDataResponseWriter struct {
} }
func SendData(h http.Handler, injecters ...Injecter) http.Handler { func SendData(h http.Handler, injecters ...Injecter) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return contentprocessor.SetContentHeaders(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s := sendDataResponseWriter{ s := sendDataResponseWriter{
rw: w, rw: w,
req: r, req: r,
...@@ -47,7 +49,7 @@ func SendData(h http.Handler, injecters ...Injecter) http.Handler { ...@@ -47,7 +49,7 @@ func SendData(h http.Handler, injecters ...Injecter) http.Handler {
} }
defer s.flush() defer s.flush()
h.ServeHTTP(&s, r) h.ServeHTTP(&s, r)
}) }))
} }
func (s *sendDataResponseWriter) Header() http.Header { func (s *sendDataResponseWriter) Header() http.Header {
...@@ -74,13 +76,15 @@ func (s *sendDataResponseWriter) WriteHeader(status int) { ...@@ -74,13 +76,15 @@ func (s *sendDataResponseWriter) WriteHeader(status int) {
return return
} }
s.Header().Del(HeaderKey)
s.rw.WriteHeader(s.status) s.rw.WriteHeader(s.status)
} }
func (s *sendDataResponseWriter) tryInject() bool { func (s *sendDataResponseWriter) tryInject() bool {
header := s.Header().Get(HeaderKey) if s.hijacked {
s.Header().Del(HeaderKey) return false
}
header := s.Header().Get(headers.GitlabWorkhorseSendDataHeader)
if header == "" { if header == "" {
return false return false
} }
......
...@@ -9,20 +9,9 @@ import ( ...@@ -9,20 +9,9 @@ import (
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
)
func TestHeaderDelete(t *testing.T) {
for _, code := range []int{200, 400} {
recorder := httptest.NewRecorder()
rw := &sendDataResponseWriter{rw: recorder, req: &http.Request{}}
rw.Header().Set(HeaderKey, "foobar")
rw.WriteHeader(code)
if header := recorder.Header().Get(HeaderKey); header != "" { "gitlab.com/gitlab-org/gitlab-workhorse/internal/headers"
t.Fatalf("HTTP %d response: expected header to be empty, found %q", code, header) )
}
}
}
func TestWriter(t *testing.T) { func TestWriter(t *testing.T) {
upstreamResponse := "hello world" upstreamResponse := "hello world"
...@@ -49,7 +38,7 @@ func TestWriter(t *testing.T) { ...@@ -49,7 +38,7 @@ func TestWriter(t *testing.T) {
recorder := httptest.NewRecorder() recorder := httptest.NewRecorder()
rw := &sendDataResponseWriter{rw: recorder, injecters: []Injecter{&testInjecter{}}} rw := &sendDataResponseWriter{rw: recorder, injecters: []Injecter{&testInjecter{}}}
rw.Header().Set(HeaderKey, tc.headerValue) rw.Header().Set(headers.GitlabWorkhorseSendDataHeader, tc.headerValue)
n, err := rw.Write([]byte(upstreamResponse)) n, err := rw.Write([]byte(upstreamResponse))
require.NoError(t, err) require.NoError(t, err)
......
...@@ -7,17 +7,19 @@ via the X-Sendfile mechanism. All that is needed in the Rails code is the ...@@ -7,17 +7,19 @@ via the X-Sendfile mechanism. All that is needed in the Rails code is the
package sendfile package sendfile
import ( import (
"fmt"
"io"
"io/ioutil"
"net/http" "net/http"
"regexp" "regexp"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/headers"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/log" "gitlab.com/gitlab-org/gitlab-workhorse/internal/log"
) )
const sendFileResponseHeader = "X-Sendfile"
var ( var (
sendFileRequests = prometheus.NewCounterVec( sendFileRequests = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
...@@ -57,7 +59,7 @@ func SendFile(h http.Handler) http.Handler { ...@@ -57,7 +59,7 @@ func SendFile(h http.Handler) http.Handler {
req: req, req: req,
} }
// Advertise to upstream (Rails) that we support X-Sendfile // Advertise to upstream (Rails) that we support X-Sendfile
req.Header.Set("X-Sendfile-Type", "X-Sendfile") req.Header.Set(headers.XSendFileTypeHeader, headers.XSendFileHeader)
defer s.flush() defer s.flush()
h.ServeHTTP(s, req) h.ServeHTTP(s, req)
}) })
...@@ -88,8 +90,8 @@ func (s *sendFileResponseWriter) WriteHeader(status int) { ...@@ -88,8 +90,8 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
return return
} }
if file := s.Header().Get(sendFileResponseHeader); file != "" { file := s.Header().Get(headers.XSendFileHeader)
s.Header().Del(sendFileResponseHeader) if file != "" && !s.hijacked {
// Mark this connection as hijacked // Mark this connection as hijacked
s.hijacked = true s.hijacked = true
...@@ -109,6 +111,15 @@ func sendFileFromDisk(w http.ResponseWriter, r *http.Request, file string) { ...@@ -109,6 +111,15 @@ func sendFileFromDisk(w http.ResponseWriter, r *http.Request, file string) {
"uri": helper.ScrubURLParams(r.RequestURI), "uri": helper.ScrubURLParams(r.RequestURI),
}).Print("Send file") }).Print("Send file")
contentTypeHeaderPresent := false
if headers.IsDetectContentTypeHeaderPresent(w) {
// Removing the GitlabWorkhorseDetectContentTypeHeader header to
// avoid handling the response by the senddata handler
w.Header().Del(headers.GitlabWorkhorseDetectContentTypeHeader)
contentTypeHeaderPresent = true
}
content, fi, err := helper.OpenFile(file) content, fi, err := helper.OpenFile(file)
if err != nil { if err != nil {
http.NotFound(w, r) http.NotFound(w, r)
...@@ -118,6 +129,20 @@ func sendFileFromDisk(w http.ResponseWriter, r *http.Request, file string) { ...@@ -118,6 +129,20 @@ func sendFileFromDisk(w http.ResponseWriter, r *http.Request, file string) {
countSendFileMetrics(fi.Size(), r) countSendFileMetrics(fi.Size(), r)
if contentTypeHeaderPresent {
data, err := ioutil.ReadAll(io.LimitReader(content, headers.MaxDetectSize))
if err != nil {
helper.Fail500(w, r, fmt.Errorf("Error reading the file"))
return
}
content.Seek(0, io.SeekStart)
contentType, contentDisposition := headers.SafeContentHeaders(data, w.Header().Get(headers.ContentDispositionHeader))
w.Header().Set(headers.ContentTypeHeader, contentType)
w.Header().Set(headers.ContentDispositionHeader, contentDisposition)
}
http.ServeContent(w, r, "", fi.ModTime(), content) http.ServeContent(w, r, "", fi.ModTime(), content)
} }
......
...@@ -7,6 +7,8 @@ import ( ...@@ -7,6 +7,8 @@ import (
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/headers"
) )
func TestResponseWriter(t *testing.T) { func TestResponseWriter(t *testing.T) {
...@@ -40,7 +42,7 @@ func TestResponseWriter(t *testing.T) { ...@@ -40,7 +42,7 @@ func TestResponseWriter(t *testing.T) {
rw := httptest.NewRecorder() rw := httptest.NewRecorder()
sf := &sendFileResponseWriter{rw: rw, req: r} sf := &sendFileResponseWriter{rw: rw, req: r}
sf.Header().Set(sendFileResponseHeader, tc.sendfileHeader) sf.Header().Set(headers.XSendFileHeader, tc.sendfileHeader)
upstreamBody := []byte(upstreamResponse) upstreamBody := []byte(upstreamResponse)
n, err := sf.Write(upstreamBody) n, err := sf.Write(upstreamBody)
...@@ -58,3 +60,113 @@ func TestResponseWriter(t *testing.T) { ...@@ -58,3 +60,113 @@ func TestResponseWriter(t *testing.T) {
}) })
} }
} }
func TestAllowExistentContentHeaders(t *testing.T) {
fixturePath := "../../testdata/forgedfile.png"
httpHeaders := map[string]string{
headers.ContentTypeHeader: "image/png",
headers.ContentDispositionHeader: "inline",
}
resp := makeRequest(t, fixturePath, httpHeaders)
require.Equal(t, "image/png", resp.Header.Get(headers.ContentTypeHeader))
require.Equal(t, "inline", resp.Header.Get(headers.ContentDispositionHeader))
}
func TestSuccessOverrideContentHeadersFeatureEnabled(t *testing.T) {
fixturePath := "../../testdata/forgedfile.png"
httpHeaders := map[string]string{
headers.ContentTypeHeader: "image/png",
headers.ContentDispositionHeader: "inline",
"Range": "bytes=1-2",
}
resp := makeRequest(t, fixturePath, httpHeaders)
require.Equal(t, "image/png", resp.Header.Get(headers.ContentTypeHeader))
require.Equal(t, "inline", resp.Header.Get(headers.ContentDispositionHeader))
}
func TestSuccessOverrideContentHeadersRangeRequestFeatureEnabled(t *testing.T) {
fixturePath := "../../testdata/forgedfile.png"
fixtureContent, err := ioutil.ReadFile(fixturePath)
require.NoError(t, err)
r, err := http.NewRequest("GET", "/foo", nil)
r.Header.Set("Range", "bytes=1-2")
require.NoError(t, err)
rw := httptest.NewRecorder()
sf := &sendFileResponseWriter{rw: rw, req: r}
sf.Header().Set(headers.XSendFileHeader, fixturePath)
sf.Header().Set(headers.ContentTypeHeader, "image/png")
sf.Header().Set(headers.ContentDispositionHeader, "inline")
sf.Header().Set(headers.GitlabWorkhorseDetectContentTypeHeader, "true")
upstreamBody := []byte(fixtureContent)
_, err = sf.Write(upstreamBody)
require.NoError(t, err)
rw.Flush()
resp := rw.Result()
body := resp.Body
data, err := ioutil.ReadAll(body)
require.NoError(t, err)
require.NoError(t, body.Close())
require.Len(t, data, 2)
require.Equal(t, "application/octet-stream", resp.Header.Get(headers.ContentTypeHeader))
require.Equal(t, "attachment", resp.Header.Get(headers.ContentDispositionHeader))
}
func TestSuccessInlineWhitelistedTypesFeatureEnabled(t *testing.T) {
fixturePath := "../../testdata/image.png"
httpHeaders := map[string]string{
headers.ContentDispositionHeader: "inline",
headers.GitlabWorkhorseDetectContentTypeHeader: "true",
}
resp := makeRequest(t, fixturePath, httpHeaders)
require.Equal(t, "image/png", resp.Header.Get(headers.ContentTypeHeader))
require.Equal(t, "inline", resp.Header.Get(headers.ContentDispositionHeader))
}
func makeRequest(t *testing.T, fixturePath string, httpHeaders map[string]string) *http.Response {
fixtureContent, err := ioutil.ReadFile(fixturePath)
require.NoError(t, err)
r, err := http.NewRequest("GET", "/foo", nil)
require.NoError(t, err)
rw := httptest.NewRecorder()
sf := &sendFileResponseWriter{rw: rw, req: r}
sf.Header().Set(headers.XSendFileHeader, fixturePath)
for name, value := range httpHeaders {
sf.Header().Set(name, value)
}
upstreamBody := []byte("hello")
n, err := sf.Write(upstreamBody)
require.NoError(t, err)
require.Equal(t, len(upstreamBody), n, "bytes written")
rw.Flush()
resp := rw.Result()
body := resp.Body
data, err := ioutil.ReadAll(body)
require.NoError(t, err)
require.NoError(t, body.Close())
require.Equal(t, fixtureContent, data)
return resp
}
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"bytes" "bytes"
"errors" "errors"
"fmt" "fmt"
"io/ioutil"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os" "os"
...@@ -151,3 +152,11 @@ func RootDir() string { ...@@ -151,3 +152,11 @@ func RootDir() string {
} }
return path.Join(path.Dir(currentFile), "../..") return path.Join(path.Dir(currentFile), "../..")
} }
func LoadFile(t *testing.T, filePath string) string {
content, err := ioutil.ReadFile(path.Join(RootDir(), filePath))
if err != nil {
t.Fatal(err)
}
return string(content)
}
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import os.path\n",
"import time\n",
"import logging\n",
"import datetime\n",
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.3"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
= Title1
Example
= Title2
Example
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 330 82">
<title>SVG logo combined with the W3C logo, set horizontally</title>
<desc>The logo combines three entities displayed horizontally: the W3C logo with the text 'W3C'; the drawing of a flower or star shape with eight arms; and the text 'SVG'. These three entities are set horizontally.</desc>
<metadata>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:cc="http://creativecommons.org/ns#" xmlns:xhtml="http://www.w3.org/1999/xhtml/vocab#" xmlns:dc="http://purl.org/dc/elements/1.1/">
<cc:Work rdf:about="">
<dc:title>SVG logo combined with the W3C logo</dc:title>
<dc:format>image/svg+xml</dc:format>
<rdfs:seeAlso rdf:resource="http://www.w3.org/2007/10/sw-logos.html"/>
<dc:date>2007-11-01</dc:date>
<xhtml:license rdf:resource="http://www.w3.org/Consortium/Legal/2002/copyright-documents-20021231"/>
<cc:morePermissions rdf:resource="http://www.w3.org/2007/10/sw-logos.html#LogoWithW3C"/>
<cc:attributionURL rdf:reource="http://www.w3.org/2001/sw/"/>
<dc:description>The logo combines three entities displayed horizontally: the W3C logo with the text 'W3C'; the drawing of a flower or star shape with eight arms; and the text 'SVG'. These three entities are set horizontally.
</dc:description>
</cc:Work>
</rdf:RDF>
</metadata>
<text x="0" y="75" font-size="83" fill-opacity="0" font-family="Trebuchet" letter-spacing="-12">W3C</text>
<text x="180" y="75" font-size="83" fill-opacity="0" font-family="Trebuchet" font-weight="bold">SVG</text>
<defs>
<g id="SVG" fill="#005A9C">
<path id="S" d="M 5.482,31.319 C2.163,28.001 0.109,23.419 0.109,18.358 C0.109,8.232 8.322,0.024 18.443,0.024 C28.569,0.024 36.782,8.232 36.782,18.358 L26.042,18.358 C26.042,14.164 22.638,10.765 18.443,10.765 C14.249,10.765 10.850,14.164 10.850,18.358 C10.850,20.453 11.701,22.351 13.070,23.721 L13.075,23.721 C14.450,25.101 15.595,25.500 18.443,25.952 L18.443,25.952 C23.509,26.479 28.091,28.006 31.409,31.324 L31.409,31.324 C34.728,34.643 36.782,39.225 36.782,44.286 C36.782,54.412 28.569,62.625 18.443,62.625 C8.322,62.625 0.109,54.412 0.109,44.286 L10.850,44.286 C10.850,48.480 14.249,51.884 18.443,51.884 C22.638,51.884 26.042,48.480 26.042,44.286 C26.042,42.191 25.191,40.298 23.821,38.923 L23.816,38.923 C22.441,37.548 20.468,37.074 18.443,36.697 L18.443,36.692 C13.533,35.939 8.800,34.638 5.482,31.319 L5.482,31.319 L5.482,31.319 Z"/>
<path id="V" d="M 73.452,0.024 L60.482,62.625 L49.742,62.625 L36.782,0.024 L47.522,0.024 L55.122,36.687 L62.712,0.024 L73.452,0.024 Z"/>
<path id="G" d="M 91.792,25.952 L110.126,25.952 L110.126,44.286 L110.131,44.286 C110.131,54.413 101.918,62.626 91.792,62.626 C81.665,62.626 73.458,54.413 73.458,44.286 L73.458,44.286 L73.458,18.359 L73.453,18.359 C73.453,8.233 81.665,0.025 91.792,0.025 C101.913,0.025 110.126,8.233 110.126,18.359 L99.385,18.359 C99.385,14.169 95.981,10.765 91.792,10.765 C87.597,10.765 84.198,14.169 84.198,18.359 L84.198,44.286 L84.198,44.286 C84.198,48.481 87.597,51.880 91.792,51.880 C95.981,51.880 99.380,48.481 99.385,44.291 L99.385,44.286 L99.385,36.698 L91.792,36.698 L91.792,25.952 L91.792,25.952 Z"/>
</g>
</defs>
<g shape-rendering="geometricPrecision" text-rendering="geometricPrecision" image-rendering="optimizeQuality">
<g>
<g id="w3c-logo">
<g>
<title>W3</title>
<path d="M33.695,10.802l12.062,41.016l12.067-41.016h8.731L46.587,78.188h-0.831l-12.48-41.759L20.797,78.188 h-0.832L0,10.802h8.736l12.061,41.016l8.154-27.618l-3.993-13.397H33.695z" fill="#005A9C"/>
<path d="M91.355,56.557c0,6.104-1.624,11.234-4.862,15.394c-3.248,4.158-7.45,6.237-12.607,6.237 c-3.882,0-7.263-1.238-10.148-3.702c-2.885-2.47-5.02-5.812-6.406-10.022l6.82-2.829c1.001,2.552,2.317,4.562,3.953,6.028 c1.636,1.469,3.56,2.207,5.781,2.207c2.329,0,4.3-1.306,5.909-3.911c1.609-2.606,2.411-5.738,2.411-9.401 c0-4.049-0.861-7.179-2.582-9.399c-1.995-2.604-5.129-3.912-9.397-3.912h-3.327v-3.991l11.646-20.133H64.484l-3.911,6.655h-2.493 V10.802h32.441v4.075l-12.31,21.217c4.324,1.385,7.596,3.911,9.815,7.571C90.246,47.324,91.355,51.618,91.355,56.557z" fill="#005A9C"/>
</g>
<g>
<title>C</title>
<path d="M125.211,10.425l1.414,8.6l-5.008,9.583c0,0-1.924-4.064-5.117-6.314 c-2.693-1.899-4.447-2.309-7.186-1.746c-3.527,0.73-7.516,4.938-9.258,10.13c-2.084,6.21-2.104,9.218-2.178,11.978 c-0.115,4.428,0.58,7.043,0.58,7.043s-3.04-5.626-3.011-13.866c0.018-5.882,0.947-11.218,3.666-16.479 c2.396-4.627,5.95-7.404,9.109-7.728c3.264-0.343,5.848,1.229,7.841,2.938c2.089,1.788,4.213,5.698,4.213,5.698L125.211,10.425z" fill="#221B0A"/>
<path d="M125.823,59.099c0,0-2.208,3.957-3.589,5.48c-1.379,1.524-3.849,4.209-6.896,5.555 c-3.049,1.343-4.646,1.598-7.661,1.306c-3.01-0.29-5.807-2.032-6.786-2.764c-0.979-0.722-3.486-2.864-4.897-4.854 c-1.42-2-3.634-5.995-3.634-5.995s1.233,4.001,2.007,5.699c0.442,0.977,1.81,3.965,3.749,6.572 c1.805,2.425,5.315,6.604,10.652,7.545c5.336,0.945,9.002-1.449,9.907-2.031c0.907-0.578,2.819-2.178,4.032-3.475 c1.264-1.351,2.459-3.079,3.116-4.108c0.487-0.758,1.276-2.286,1.276-2.286L125.823,59.099z" fill="#221B0A"/>
</g>
<g>
<title>Registered Trademark</title>
<path d="M132.592,5.201c2.493,0,4.485,2.032,4.485,4.525c0,2.533-1.992,4.543-4.505,4.543 c-2.491,0-4.524-2.01-4.524-4.543c0-2.493,2.033-4.525,4.524-4.525H132.592z M132.554,6.107c-1.889,0-3.417,1.629-3.417,3.639 c0,2.029,1.528,3.619,3.436,3.619c1.912,0.019,3.46-1.59,3.46-3.619c0-2.01-1.548-3.639-3.46-3.639H132.554z M131.791,12.361 h-1.067V7.332c0.401-0.058,0.846-0.141,1.61-0.141c0.862,0,1.387,0.141,1.726,0.404c0.28,0.221,0.445,0.563,0.445,1.085 c0,0.603-0.423,1.024-0.966,1.166v0.042c0.441,0.078,0.724,0.479,0.801,1.226c0.103,0.783,0.203,1.085,0.284,1.247h-1.104 c-0.123-0.183-0.203-0.625-0.305-1.31c-0.077-0.542-0.4-0.763-0.942-0.763h-0.481V12.361z M131.791,9.463h0.5 c0.624,0,1.105-0.199,1.105-0.723c0-0.421-0.301-0.744-1.025-0.744c-0.261,0-0.441,0-0.58,0.021V9.463z" fill="#221B0A"/>
</g>
</g>
<g id="logo" transform="scale(0.24) translate(550, 35)">
<g stroke-width="38.0086" stroke="#000">
<g id="svgstar" transform="translate(150, 150)">
<path id="svgbar" fill="#EDA921" d="M-84.1487,-15.8513 a22.4171,22.4171 0 1 0 0,31.7026 h168.2974 a22.4171,22.4171 0 1 0 0,-31.7026 Z"/>
<use xlink:href="#svgbar" transform="rotate(45)"/>
<use xlink:href="#svgbar" transform="rotate(90)"/>
<use xlink:href="#svgbar" transform="rotate(135)"/>
</g>
</g>
<use xlink:href="#svgstar"/>
</g>
<g id="SVG-label">
<use xlink:href="#SVG" transform="scale(1.08) translate(195,10)"/>
</g>
</g>
</g>
</svg>
The MIT License
Copyright (c) 2016 Tomas Aparicio
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
# go-is-svg [![Build Status](https://travis-ci.org/h2non/go-is-svg.png)](https://travis-ci.org/h2non/go-is-svg) [![GoDoc](https://godoc.org/github.com/h2non/go-is-svg?status.svg)](https://godoc.org/github.com/h2non/go-is-svg) [![Coverage Status](https://coveralls.io/repos/github/h2non/go-is-svg/badge.svg?branch=master)](https://coveralls.io/github/h2non/go-is-svg?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/h2non/go-is-svg)](https://goreportcard.com/report/github.com/h2non/go-is-svg)
Tiny package to verify if a given file buffer is an SVG image in Go (golang).
See also [filetype](https://github.com/h2non/filetype) package for binary files type inference.
## Installation
```bash
go get -u github.com/h2non/go-is-svg
```
## Example
```go
package main
import (
"fmt"
"io/ioutil"
svg "github.com/h2non/go-is-svg"
)
func main() {
buf, err := ioutil.ReadFile("_example/example.svg")
if err != nil {
fmt.Printf("Error: %s\n", err)
return
}
if svg.Is(buf) {
fmt.Println("File is an SVG")
} else {
fmt.Println("File is NOT an SVG")
}
}
```
Run example:
```bash
go run _example/example.go
```
## License
MIT - Tomas Aparicio
package issvg
import (
"regexp"
"unicode/utf8"
)
var (
htmlCommentRegex = regexp.MustCompile("(?i)<!--([\\s\\S]*?)-->")
svgRegex = regexp.MustCompile(`(?i)^\s*(?:<\?xml[^>]*>\s*)?(?:<!doctype svg[^>]*>\s*)?<svg[^>]*>[^*]*<\/svg>\s*$`)
)
// isBinary checks if the given buffer is a binary file.
func isBinary(buf []byte) bool {
if len(buf) < 24 {
return false
}
for i := 0; i < 24; i++ {
charCode, _ := utf8.DecodeRuneInString(string(buf[i]))
if charCode == 65533 || charCode <= 8 {
return true
}
}
return false
}
// Is returns true if the given buffer is a valid SVG image.
func Is(buf []byte) bool {
return !isBinary(buf) && svgRegex.Match(htmlCommentRegex.ReplaceAll(buf, []byte{}))
}
// IsSVG returns true if the given buffer is a valid SVG image.
// Alias to: Is()
func IsSVG(buf []byte) bool {
return Is(buf)
}
...@@ -189,6 +189,12 @@ ...@@ -189,6 +189,12 @@
"version": "v1", "version": "v1",
"versionExact": "v1.2.0" "versionExact": "v1.2.0"
}, },
{
"checksumSHA1": "v52fmR2hxX7o6emMN/dflEqys64=",
"path": "github.com/h2non/go-is-svg",
"revision": "35e8c4b0612ce1ab2a228e25a323b22482db6788",
"revisionTime": "2016-09-27T21:24:52Z"
},
{ {
"checksumSHA1": "mb0MqzDyYEQMgh8+qwVm1RV4cxc=", "checksumSHA1": "mb0MqzDyYEQMgh8+qwVm1RV4cxc=",
"path": "github.com/jfbus/httprs", "path": "github.com/jfbus/httprs",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment