Commit 4e0984d6 authored by Jacob Vosmaer's avatar Jacob Vosmaer

Merge branch 'an-refactor-badgateway' into 'master'

Refactor badgateway to use standardlib interfaces

See merge request gitlab-org/gitlab-workhorse!316
parents bc64cb9b 24bd7c26
...@@ -10,10 +10,10 @@ import ( ...@@ -10,10 +10,10 @@ import (
"github.com/dgrijalva/jwt-go" "github.com/dgrijalva/jwt-go"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api" "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/secret" "gitlab.com/gitlab-org/gitlab-workhorse/internal/secret"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream/roundtripper"
) )
func okHandler(w http.ResponseWriter, _ *http.Request, _ *api.Response) { func okHandler(w http.ResponseWriter, _ *http.Request, _ *api.Response) {
...@@ -34,7 +34,7 @@ func runPreAuthorizeHandler(t *testing.T, ts *httptest.Server, suffix string, ur ...@@ -34,7 +34,7 @@ func runPreAuthorizeHandler(t *testing.T, ts *httptest.Server, suffix string, ur
} }
parsedURL := helper.URLMustParse(ts.URL) parsedURL := helper.URLMustParse(ts.URL)
testhelper.ConfigureSecret() testhelper.ConfigureSecret()
a := api.NewAPI(parsedURL, "123", badgateway.TestRoundTripper(parsedURL)) a := api.NewAPI(parsedURL, "123", roundtripper.NewTestBackendRoundTripper(parsedURL))
response := httptest.NewRecorder() response := httptest.NewRecorder()
a.PreAuthorizeHandler(okHandler, suffix).ServeHTTP(response, httpRequest) a.PreAuthorizeHandler(okHandler, suffix).ServeHTTP(response, httpRequest)
......
...@@ -13,7 +13,6 @@ import ( ...@@ -13,7 +13,6 @@ import (
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
pb "gitlab.com/gitlab-org/gitaly-proto/go" pb "gitlab.com/gitlab-org/gitaly-proto/go"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly" "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/secret" "gitlab.com/gitlab-org/gitlab-workhorse/internal/secret"
...@@ -56,7 +55,7 @@ func init() { ...@@ -56,7 +55,7 @@ func init() {
prometheus.MustRegister(bytesTotal) prometheus.MustRegister(bytesTotal)
} }
func NewAPI(myURL *url.URL, version string, roundTripper *badgateway.RoundTripper) *API { func NewAPI(myURL *url.URL, version string, roundTripper http.RoundTripper) *API {
return &API{ return &API{
Client: &http.Client{Transport: roundTripper}, Client: &http.Client{Transport: roundTripper},
URL: myURL, URL: myURL,
......
...@@ -15,11 +15,11 @@ import ( ...@@ -15,11 +15,11 @@ import (
"testing" "testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api" "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore" "gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy" "gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream/roundtripper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts" "gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
) )
...@@ -113,7 +113,7 @@ func testUploadArtifacts(contentType string, body io.Reader, t *testing.T, ts *h ...@@ -113,7 +113,7 @@ func testUploadArtifacts(contentType string, body io.Reader, t *testing.T, ts *h
httpRequest.Header.Set("Content-Type", contentType) httpRequest.Header.Set("Content-Type", contentType)
response := httptest.NewRecorder() response := httptest.NewRecorder()
parsedURL := helper.URLMustParse(ts.URL) parsedURL := helper.URLMustParse(ts.URL)
roundTripper := badgateway.TestRoundTripper(parsedURL) roundTripper := roundtripper.NewTestBackendRoundTripper(parsedURL)
testhelper.ConfigureSecret() testhelper.ConfigureSecret()
apiClient := api.NewAPI(parsedURL, "123", roundTripper) apiClient := api.NewAPI(parsedURL, "123", roundTripper)
proxyClient := proxy.NewProxy(parsedURL, "123", roundTripper) proxyClient := proxy.NewProxy(parsedURL, "123", roundTripper)
......
...@@ -2,84 +2,30 @@ package badgateway ...@@ -2,84 +2,30 @@ package badgateway
import ( import (
"bytes" "bytes"
"context"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net"
"net/http" "net/http"
"net/url"
"time" "time"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
) )
// defaultDialer is configured to use the values from http.DefaultTransport
var defaultDialer = &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}
// Error is a custom error for pretty Sentry 'issues' // Error is a custom error for pretty Sentry 'issues'
type Error struct{ error } type sentryError struct{ error }
type RoundTripper struct { type roundTripper struct {
Transport *http.Transport next http.RoundTripper
developmentMode bool developmentMode bool
} }
func TestRoundTripper(backend *url.URL) *RoundTripper { // NewRoundTripper creates a RoundTripper with a provided underlying next transport
return NewRoundTripper(backend, "", 0, true) func NewRoundTripper(developmentMode bool, next http.RoundTripper) http.RoundTripper {
} return &roundTripper{next: next, developmentMode: developmentMode}
// NewRoundTripper returns a new RoundTripper instance using the provided values
func NewRoundTripper(backend *url.URL, socket string, proxyHeadersTimeout time.Duration, developmentMode bool) *RoundTripper {
// Copied from the definition of http.DefaultTransport. We can't literally copy http.DefaultTransport because of its hidden internal state.
tr := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: defaultDialer.DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
tr.ResponseHeaderTimeout = proxyHeadersTimeout
if backend != nil && socket == "" {
address := mustParseAddress(backend.Host, backend.Scheme)
tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
return defaultDialer.DialContext(ctx, "tcp", address)
}
} else if socket != "" {
tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
return defaultDialer.DialContext(ctx, "unix", socket)
}
} else {
panic("backend is nil and socket is empty")
}
return &RoundTripper{Transport: tr, developmentMode: developmentMode}
}
func mustParseAddress(address, scheme string) string {
if scheme == "https" {
panic("TLS is not supported for backend connections")
}
for _, suffix := range []string{"", ":" + scheme} {
address += suffix
if host, port, err := net.SplitHostPort(address); err == nil && host != "" && port != "" {
return host + ":" + port
}
}
panic(fmt.Errorf("could not parse host:port from address %q and scheme %q", address, scheme))
} }
func (t *RoundTripper) RoundTrip(r *http.Request) (res *http.Response, err error) { func (t *roundTripper) RoundTrip(r *http.Request) (res *http.Response, err error) {
start := time.Now() start := time.Now()
res, err = t.Transport.RoundTrip(r) res, err = t.next.RoundTrip(r)
// httputil.ReverseProxy translates all errors from this // httputil.ReverseProxy translates all errors from this
// RoundTrip function into 500 errors. But the most likely error // RoundTrip function into 500 errors. But the most likely error
...@@ -90,7 +36,7 @@ func (t *RoundTripper) RoundTrip(r *http.Request) (res *http.Response, err error ...@@ -90,7 +36,7 @@ func (t *RoundTripper) RoundTrip(r *http.Request) (res *http.Response, err error
if err != nil { if err != nil {
helper.LogError( helper.LogError(
r, r,
&Error{fmt.Errorf("badgateway: failed after %.fs: %v", time.Since(start).Seconds(), err)}, &sentryError{fmt.Errorf("badgateway: failed after %.fs: %v", time.Since(start).Seconds(), err)},
) )
message := "GitLab is not responding" message := "GitLab is not responding"
......
...@@ -7,7 +7,6 @@ import ( ...@@ -7,7 +7,6 @@ import (
"net/url" "net/url"
"time" "time"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
) )
...@@ -21,7 +20,7 @@ type Proxy struct { ...@@ -21,7 +20,7 @@ type Proxy struct {
AllowResponseBuffering bool AllowResponseBuffering bool
} }
func NewProxy(myURL *url.URL, version string, roundTripper *badgateway.RoundTripper) *Proxy { func NewProxy(myURL *url.URL, version string, roundTripper http.RoundTripper) *Proxy {
p := Proxy{Version: version, AllowResponseBuffering: true} p := Proxy{Version: version, AllowResponseBuffering: true}
if myURL == nil { if myURL == nil {
......
...@@ -17,12 +17,12 @@ import ( ...@@ -17,12 +17,12 @@ import (
"time" "time"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api" "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore" "gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/objectstore/test" "gitlab.com/gitlab-org/gitlab-workhorse/internal/objectstore/test"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy" "gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream/roundtripper"
) )
var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}) var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
...@@ -325,5 +325,5 @@ func TestInvalidFileNames(t *testing.T) { ...@@ -325,5 +325,5 @@ func TestInvalidFileNames(t *testing.T) {
func newProxy(url string) *proxy.Proxy { func newProxy(url string) *proxy.Proxy {
parsedURL := helper.URLMustParse(url) parsedURL := helper.URLMustParse(url)
return proxy.NewProxy(parsedURL, "123", badgateway.TestRoundTripper(parsedURL)) return proxy.NewProxy(parsedURL, "123", roundtripper.NewTestBackendRoundTripper(parsedURL))
} }
package roundtripper
import (
"context"
"fmt"
"net"
"net/http"
"net/url"
"time"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
)
func mustParseAddress(address, scheme string) string {
if scheme == "https" {
panic("TLS is not supported for backend connections")
}
for _, suffix := range []string{"", ":" + scheme} {
address += suffix
if host, port, err := net.SplitHostPort(address); err == nil && host != "" && port != "" {
return host + ":" + port
}
}
panic(fmt.Errorf("could not parse host:port from address %q and scheme %q", address, scheme))
}
// NewBackendRoundTripper returns a new RoundTripper instance using the provided values
func NewBackendRoundTripper(backend *url.URL, socket string, proxyHeadersTimeout time.Duration, developmentMode bool) http.RoundTripper {
// Copied from the definition of http.DefaultTransport. We can't literally copy http.DefaultTransport because of its hidden internal state.
transport, dialer := newBackendTransport()
transport.ResponseHeaderTimeout = proxyHeadersTimeout
if backend != nil && socket == "" {
address := mustParseAddress(backend.Host, backend.Scheme)
transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
return dialer.DialContext(ctx, "tcp", address)
}
} else if socket != "" {
transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
return dialer.DialContext(ctx, "unix", socket)
}
} else {
panic("backend is nil and socket is empty")
}
return badgateway.NewRoundTripper(developmentMode, transport)
}
// NewTestBackendRoundTripper sets up a RoundTripper for testing purposes
func NewTestBackendRoundTripper(backend *url.URL) http.RoundTripper {
return NewBackendRoundTripper(backend, "", 0, true)
}
package badgateway package roundtripper
import ( import (
"testing" "testing"
......
package roundtripper
import (
"net"
"net/http"
"time"
)
// newBackendTransport setups the default HTTP transport which Workhorse uses
// to communicate with the upstream
func newBackendTransport() (*http.Transport, *net.Dialer) {
dialler := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: dialler.DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
return transport, dialler
}
...@@ -13,10 +13,10 @@ import ( ...@@ -13,10 +13,10 @@ import (
"github.com/sebest/xff" "github.com/sebest/xff"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/config" "gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/upload" "gitlab.com/gitlab-org/gitlab-workhorse/internal/upload"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream/roundtripper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/urlprefix" "gitlab.com/gitlab-org/gitlab-workhorse/internal/urlprefix"
) )
...@@ -31,7 +31,7 @@ type upstream struct { ...@@ -31,7 +31,7 @@ type upstream struct {
config.Config config.Config
URLPrefix urlprefix.Prefix URLPrefix urlprefix.Prefix
Routes []routeEntry Routes []routeEntry
RoundTripper *badgateway.RoundTripper RoundTripper http.RoundTripper
} }
func NewUpstream(cfg config.Config) http.Handler { func NewUpstream(cfg config.Config) http.Handler {
...@@ -41,7 +41,7 @@ func NewUpstream(cfg config.Config) http.Handler { ...@@ -41,7 +41,7 @@ func NewUpstream(cfg config.Config) http.Handler {
if up.Backend == nil { if up.Backend == nil {
up.Backend = DefaultBackend up.Backend = DefaultBackend
} }
up.RoundTripper = badgateway.NewRoundTripper(up.Backend, up.Socket, up.ProxyHeadersTimeout, cfg.DevelopmentMode) up.RoundTripper = roundtripper.NewBackendRoundTripper(up.Backend, up.Socket, up.ProxyHeadersTimeout, cfg.DevelopmentMode)
up.configureURLPrefix() up.configureURLPrefix()
up.configureRoutes() up.configureRoutes()
return &up return &up
......
...@@ -19,7 +19,7 @@ import ( ...@@ -19,7 +19,7 @@ import (
// ErrNotAZip will be used when the file is not a zip archive // ErrNotAZip will be used when the file is not a zip archive
var ErrNotAZip = errors.New("not a zip") var ErrNotAZip = errors.New("not a zip")
// ErrNotAZip will be used when the file can't be found // ErrArchiveNotFound will be used when the file can't be found
var ErrArchiveNotFound = errors.New("archive not found") var ErrArchiveNotFound = errors.New("archive not found")
var httpClient = &http.Client{ var httpClient = &http.Client{
......
...@@ -16,14 +16,15 @@ import ( ...@@ -16,14 +16,15 @@ import (
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy" "gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream/roundtripper"
) )
const testVersion = "123" const testVersion = "123"
func newProxy(url string, rt *badgateway.RoundTripper) *proxy.Proxy { func newProxy(url string, rt http.RoundTripper) *proxy.Proxy {
parsedURL := helper.URLMustParse(url) parsedURL := helper.URLMustParse(url)
if rt == nil { if rt == nil {
rt = badgateway.TestRoundTripper(parsedURL) rt = roundtripper.NewTestBackendRoundTripper(parsedURL)
} }
return proxy.NewProxy(parsedURL, testVersion, rt) return proxy.NewProxy(parsedURL, testVersion, rt)
} }
...@@ -96,8 +97,7 @@ func TestProxyReadTimeout(t *testing.T) { ...@@ -96,8 +97,7 @@ func TestProxyReadTimeout(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
rt := &badgateway.RoundTripper{ rt := badgateway.NewRoundTripper(false, &http.Transport{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment, Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{ Dial: (&net.Dialer{
Timeout: 30 * time.Second, Timeout: 30 * time.Second,
...@@ -105,8 +105,7 @@ func TestProxyReadTimeout(t *testing.T) { ...@@ -105,8 +105,7 @@ func TestProxyReadTimeout(t *testing.T) {
}).Dial, }).Dial,
TLSHandshakeTimeout: 10 * time.Second, TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: time.Millisecond, ResponseHeaderTimeout: time.Millisecond,
}, })
}
p := newProxy(ts.URL, rt) p := newProxy(ts.URL, rt)
w := httptest.NewRecorder() w := httptest.NewRecorder()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment