Skip to content

G
gitlab-ce
Commit ca67c9c7 authored by Nick Thomas's avatar Nick Thomas
Browse files
Options

Merge branch '294007-bump-workhorse' into 'master' 

Update Workhorse to v8.60.0

See merge request gitlab-org/gitlab!51965
parents dbebbbec 7568d488
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 548 additions and 105 deletions
changelogs/unreleased/294007-bump-workhorse.yml 0 → 100644
View file @ ca67c9c7
---
title: Update Workhorse to v8.60.0
merge_request: 51965
author:
type: changed
workhorse/CHANGELOG
View file @ ca67c9c7
# Changelog for gitlab-workhorse
## v8.60.0
### Added
- Support Git HTTP on toplevel repositories
https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/670
- Update GoCloud to v0.21.1+
https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/675
### Changed
- Allow blank S3 regions to be used
https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/677
## v8.59.0
### Fixed
......
workhorse/gitaly_test.go
View file @ ca67c9c7
......@@ -169,6 +169,57 @@ func TestGetInfoRefsProxiedToGitalyInterruptedStream(t *testing.T) {
waitDone(t, done)
}
func TestGetInfoRefsRouting(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.GracefulStop()
apiResponse := gitOkBody(t)
apiResponse.GitalyServer.Address = "unix:" + socketPath
ts := testAuthServer(t, nil, nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
testCases := []struct {
method string
path string
match bool
}{
{"GET", "/toplevel.git/info/refs?service=git-receive-pack", true},
{"GET", "/toplevel.wiki.git/info/refs?service=git-upload-pack", true},
{"GET", "/toplevel/child/project.git/info/refs?service=git-receive-pack", true},
{"GET", "/toplevel/child/project.wiki.git/info/refs?service=git-upload-pack", true},
{"GET", "/toplevel/child/project/snippets/123.git/info/refs?service=git-receive-pack", true},
{"GET", "/snippets/123.git/info/refs?service=git-upload-pack", true},
{"GET", "/foo/bar.git/info/refs", false},
{"GET", "/foo/bar.git/info/refs?service=git-zzz-pack", false},
{"GET", "/.git/info/refs?service=git-upload-pack", false},
{"POST", "/toplevel.git/info/refs?service=git-receive-pack", false},
}
for _, tc := range testCases {
t.Run(tc.path, func(t *testing.T) {
req, err := http.NewRequest(tc.method, ws.URL+tc.path, nil)
require.NoError(t, err)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
body := string(testhelper.ReadAll(t, resp.Body))
if tc.match {
require.Equal(t, 200, resp.StatusCode)
require.Contains(t, body, "\x00", "expect response generated by test gitaly server")
} else {
require.Equal(t, 204, resp.StatusCode)
require.Empty(t, body, "normal request has empty response body")
}
})
}
}
func waitDone(t *testing.T, done chan struct{}) {
t.Helper()
select {
......@@ -259,6 +310,65 @@ func TestPostReceivePackProxiedToGitalyInterrupted(t *testing.T) {
waitDone(t, done)
}
func TestPostReceivePackRouting(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.GracefulStop()
apiResponse := gitOkBody(t)
apiResponse.GitalyServer.Address = "unix:" + socketPath
ts := testAuthServer(t, nil, nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
testCases := []struct {
method string
path string
contentType string
match bool
}{
{"POST", "/toplevel.git/git-receive-pack", "application/x-git-receive-pack-request", true},
{"POST", "/toplevel.wiki.git/git-receive-pack", "application/x-git-receive-pack-request", true},
{"POST", "/toplevel/child/project.git/git-receive-pack", "application/x-git-receive-pack-request", true},
{"POST", "/toplevel/child/project.wiki.git/git-receive-pack", "application/x-git-receive-pack-request", true},
{"POST", "/toplevel/child/project/snippets/123.git/git-receive-pack", "application/x-git-receive-pack-request", true},
{"POST", "/snippets/123.git/git-receive-pack", "application/x-git-receive-pack-request", true},
{"POST", "/foo/bar/git-receive-pack", "application/x-git-receive-pack-request", false},
{"POST", "/foo/bar.git/git-zzz-pack", "application/x-git-receive-pack-request", false},
{"POST", "/.git/git-receive-pack", "application/x-git-receive-pack-request", false},
{"POST", "/toplevel.git/git-receive-pack", "application/x-git-upload-pack-request", false},
{"GET", "/toplevel.git/git-receive-pack", "application/x-git-receive-pack-request", false},
}
for _, tc := range testCases {
t.Run(tc.path, func(t *testing.T) {
req, err := http.NewRequest(
tc.method,
ws.URL+tc.path,
bytes.NewReader(testhelper.GitalyReceivePackResponseMock),
)
require.NoError(t, err)
req.Header.Set("Content-Type", tc.contentType)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
body := string(testhelper.ReadAll(t, resp.Body))
if tc.match {
require.Equal(t, 200, resp.StatusCode)
require.Contains(t, body, "\x00", "expect response generated by test gitaly server")
} else {
require.Equal(t, 204, resp.StatusCode)
require.Empty(t, body, "normal request has empty response body")
}
})
}
}
// ReaderFunc is an adapter to turn a conforming function into an io.Reader.
type ReaderFunc func(b []byte) (int, error)
......@@ -376,6 +486,65 @@ func TestPostUploadPackProxiedToGitalyInterrupted(t *testing.T) {
waitDone(t, done)
}
func TestPostUploadPackRouting(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.GracefulStop()
apiResponse := gitOkBody(t)
apiResponse.GitalyServer.Address = "unix:" + socketPath
ts := testAuthServer(t, nil, nil, 200, apiResponse)
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
testCases := []struct {
method string
path string
contentType string
match bool
}{
{"POST", "/toplevel.git/git-upload-pack", "application/x-git-upload-pack-request", true},
{"POST", "/toplevel.wiki.git/git-upload-pack", "application/x-git-upload-pack-request", true},
{"POST", "/toplevel/child/project.git/git-upload-pack", "application/x-git-upload-pack-request", true},
{"POST", "/toplevel/child/project.wiki.git/git-upload-pack", "application/x-git-upload-pack-request", true},
{"POST", "/toplevel/child/project/snippets/123.git/git-upload-pack", "application/x-git-upload-pack-request", true},
{"POST", "/snippets/123.git/git-upload-pack", "application/x-git-upload-pack-request", true},
{"POST", "/foo/bar/git-upload-pack", "application/x-git-upload-pack-request", false},
{"POST", "/foo/bar.git/git-zzz-pack", "application/x-git-upload-pack-request", false},
{"POST", "/.git/git-upload-pack", "application/x-git-upload-pack-request", false},
{"POST", "/toplevel.git/git-upload-pack", "application/x-git-receive-pack-request", false},
{"GET", "/toplevel.git/git-upload-pack", "application/x-git-upload-pack-request", false},
}
for _, tc := range testCases {
t.Run(tc.path, func(t *testing.T) {
req, err := http.NewRequest(
tc.method,
ws.URL+tc.path,
bytes.NewReader(testhelper.GitalyReceivePackResponseMock),
)
require.NoError(t, err)
req.Header.Set("Content-Type", tc.contentType)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
body := string(testhelper.ReadAll(t, resp.Body))
if tc.match {
require.Equal(t, 200, resp.StatusCode)
require.Contains(t, body, "\x00", "expect response generated by test gitaly server")
} else {
require.Equal(t, 204, resp.StatusCode)
require.Empty(t, body, "normal request has empty response body")
}
})
}
}
func TestGetDiffProxiedToGitalySuccessfully(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.GracefulStop()
......
workhorse/go.mod
View file @ ca67c9c7
......@@ -3,11 +3,11 @@ module gitlab.com/gitlab-org/gitlab-workhorse
go 1.13
require (
github.com/Azure/azure-storage-blob-go v0.10.0
github.com/Azure/azure-storage-blob-go v0.11.1-0.20201209121048-6df5d9af221d
github.com/BurntSushi/toml v0.3.1
github.com/FZambia/sentinel v1.0.0
github.com/alecthomas/chroma v0.7.3
github.com/aws/aws-sdk-go v1.31.13
github.com/aws/aws-sdk-go v1.36.1
github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/disintegration/imaging v1.6.2
......@@ -15,7 +15,7 @@ require (
github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721
github.com/golang/protobuf v1.4.3
github.com/gomodule/redigo v2.0.0+incompatible
github.com/gorilla/websocket v1.4.0
github.com/gorilla/websocket v1.4.1
github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/johannesboyne/gofakes3 v0.0.0-20200510090907-02d71f533bec
......@@ -30,11 +30,15 @@ require (
github.com/stretchr/testify v1.6.1
gitlab.com/gitlab-org/gitaly v1.74.0
gitlab.com/gitlab-org/labkit v1.0.0
gocloud.dev v0.20.0
gocloud.dev v0.21.1-0.20201223184910-5094f54ed8bb
golang.org/x/lint v0.0.0-20200302205851-738671d3881b
golang.org/x/net v0.0.0-20200625001655-4c5254603344
golang.org/x/tools v0.0.0-20200608174601-1b747fd94509
google.golang.org/grpc v1.29.1
golang.org/x/net v0.0.0-20201224014010-6772e930b67b
golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061 // indirect
golang.org/x/text v0.3.5 // indirect
golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516
google.golang.org/genproto v0.0.0-20210111234610-22ae2b108f89 // indirect
google.golang.org/grpc v1.34.1
google.golang.org/grpc/examples v0.0.0-20201226181154-53788aa5dcb4 // indirect
honnef.co/go/tools v0.0.1-2020.1.5
)
......
workhorse/go.sum
View file @ ca67c9c7
This diff is collapsed.
workhorse/internal/filestore/save_file_opts.go
View file @ ca67c9c7
......@@ -146,7 +146,7 @@ func (c *ObjectStorageConfig) IsGoCloud() bool {
func (c *ObjectStorageConfig) IsValid() bool {
if c.IsAWS() {
return c.S3Config.Bucket != "" && c.S3Config.Region != "" && c.s3CredentialsValid()
return c.S3Config.Bucket != "" && c.s3CredentialsValid()
} else if c.IsGoCloud() {
// We could parse and validate the URL, but GoCloud providers
// such as AzureRM don't have a fallback to normal HTTP, so we
......
workhorse/internal/filestore/save_file_opts_test.go
View file @ ca67c9c7
......@@ -187,6 +187,9 @@ func TestUseWorkhorseClientEnabled(t *testing.T) {
iamConfig := missingCfg
iamConfig.S3Config.UseIamProfile = true
missingRegion := cfg
missingRegion.S3Config.Region = ""
tests := []struct {
name string
UseWorkhorseClient bool
......@@ -245,6 +248,13 @@ func TestUseWorkhorseClientEnabled(t *testing.T) {
},
expected: false,
},
{
name: "missing S3 region",
UseWorkhorseClient: true,
remoteTempObjectID: "test-object",
objectStorageConfig: missingRegion,
expected: true,
},
}
for _, test := range tests {
......
workhorse/internal/git/info-refs.go
View file @ ca67c9c7
......@@ -20,6 +20,11 @@ func GetInfoRefsHandler(a *api.API) http.Handler {
return repoPreAuthorizeHandler(a, handleGetInfoRefs)
}
func IsSmartInfoRefs(r *http.Request) bool {
service := r.URL.Query().Get("service")
return r.Method == "GET" && (service == "git-upload-pack" || service == "git-receive-pack")
}
func handleGetInfoRefs(rw http.ResponseWriter, r *http.Request, a *api.Response) {
responseWriter := NewHttpResponseWriter(rw)
// Log 0 bytes in because we ignore the request body (and there usually is none anyway).
......
workhorse/internal/git/info-refs_test.go 0 → 100644
View file @ ca67c9c7
package git
import (
"net/http"
"net/url"
"testing"
"github.com/stretchr/testify/require"
)
func TestIsSmartInfoRefs(t *testing.T) {
testCases := []struct {
method string
url string
match bool
}{
{"GET", "?service=git-upload-pack", true},
{"GET", "?service=git-receive-pack", true},
{"GET", "", false},
{"GET", "?service=", false},
{"GET", "?service=foo", false},
{"POST", "?service=git-upload-pack", false},
{"POST", "?service=git-receive-pack", false},
}
for _, tc := range testCases {
url, err := url.Parse(tc.url)
require.NoError(t, err)
r := http.Request{Method: tc.method, URL: url}
require.Equal(t, tc.match, IsSmartInfoRefs(&r))
}
}
workhorse/internal/upstream/routes.go
View file @ ca67c9c7
......@@ -53,13 +53,13 @@ type uploadPreparers struct {
}
const (
apiPattern = `^/api/`
ciAPIPattern = `^/ci/api/`
gitProjectPattern = `^/([^/]+/){1,}[^/]+\.git/`
projectPattern = `^/([^/]+/){1,}[^/]+/`
snippetUploadPattern = `^/uploads/personal_snippet`
userUploadPattern = `^/uploads/user`
importPattern = `^/import/`
apiPattern = `\A/api/`
ciAPIPattern = `\A/ci/api/`
gitRepositoryPattern = `\A/.+\.git/`
projectPattern = `\A/([^/]+/){1,}[^/]+/`
snippetUploadPattern = `\A/uploads/personal_snippet`
userUploadPattern = `\A/uploads/user`
importPattern = `\A/import/`
)
func compileRegexp(regexpStr string) *regexp.Regexp {
......@@ -222,10 +222,10 @@ func (u *upstream) configureRoutes() {
u.Routes = []routeEntry{
// Git Clone
u.route("GET", gitProjectPattern+`info/refs\z`, git.GetInfoRefsHandler(api)),
u.route("POST", gitProjectPattern+`git-upload-pack\z`, contentEncodingHandler(git.UploadPack(api)), withMatcher(isContentType("application/x-git-upload-pack-request"))),
u.route("POST", gitProjectPattern+`git-receive-pack\z`, contentEncodingHandler(git.ReceivePack(api)), withMatcher(isContentType("application/x-git-receive-pack-request"))),
u.route("PUT", gitProjectPattern+`gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`, lfs.PutStore(api, signingProxy, preparers.lfs), withMatcher(isContentType("application/octet-stream"))),
u.route("GET", gitRepositoryPattern+`info/refs\z`, git.GetInfoRefsHandler(api), withMatcher(git.IsSmartInfoRefs)),
u.route("POST", gitRepositoryPattern+`git-upload-pack\z`, contentEncodingHandler(git.UploadPack(api)), withMatcher(isContentType("application/x-git-upload-pack-request"))),
u.route("POST", gitRepositoryPattern+`git-receive-pack\z`, contentEncodingHandler(git.ReceivePack(api)), withMatcher(isContentType("application/x-git-receive-pack-request"))),
u.route("PUT", gitRepositoryPattern+`gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`, lfs.PutStore(api, signingProxy, preparers.lfs), withMatcher(isContentType("application/octet-stream"))),
// CI Artifacts
u.route("POST", apiPattern+`v4/jobs/[0-9]+/artifacts\z`, contentEncodingHandler(artifacts.UploadArtifacts(api, signingProxy, preparers.artifacts))),
......
workhorse/main_test.go
View file @ ca67c9c7
......@@ -694,6 +694,12 @@ func testAuthServer(t *testing.T, url *regexp.Regexp, params url.Values, code in
return testhelper.TestServerWithHandler(url, func(w http.ResponseWriter, r *http.Request) {
require.NotEmpty(t, r.Header.Get("X-Request-Id"))
// return a 204 No Content response if we don't receive the JWT header
if r.Header.Get(secret.RequestHeader) == "" {
w.WriteHeader(204)
return
}
w.Header().Set("Content-Type", api.ResponseContentType)
logEntry := log.WithFields(log.Fields{
......
workhorse/upload_test.go
View file @ ca67c9c7
......@@ -234,7 +234,7 @@ func TestLfsUpload(t *testing.T) {
reqBody := "test data"
rspBody := "test success"
oid := "916f0027a575074ce72a331777c3478d6513f786a591bd892da1a577bf2335f9"
resource := fmt.Sprintf("/%s/gitlab-lfs/objects/%s/%d", testRepo, oid, len(reqBody))
resource := fmt.Sprintf("/gitlab-org/gitlab-test.git/gitlab-lfs/objects/%s/%d", oid, len(reqBody))
lfsApiResponse := fmt.Sprintf(
`{"TempPath":%q, "LfsOid":%q, "LfsSize": %d}`,
......@@ -292,6 +292,74 @@ func TestLfsUpload(t *testing.T) {
require.Equal(t, rspBody, string(rspData))
}
func TestLfsUploadRouting(t *testing.T) {
reqBody := "test data"
rspBody := "test success"
oid := "916f0027a575074ce72a331777c3478d6513f786a591bd892da1a577bf2335f9"
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(secret.RequestHeader) == "" {
w.WriteHeader(204)
} else {
fmt.Fprint(w, rspBody)
}
})
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
testCases := []struct {
method string
path string
contentType string
match bool
}{
{"PUT", "/toplevel.git/gitlab-lfs/objects", "application/octet-stream", true},
{"PUT", "/toplevel.wiki.git/gitlab-lfs/objects", "application/octet-stream", true},
{"PUT", "/toplevel/child/project.git/gitlab-lfs/objects", "application/octet-stream", true},
{"PUT", "/toplevel/child/project.wiki.git/gitlab-lfs/objects", "application/octet-stream", true},
{"PUT", "/toplevel/child/project/snippets/123.git/gitlab-lfs/objects", "application/octet-stream", true},
{"PUT", "/snippets/123.git/gitlab-lfs/objects", "application/octet-stream", true},
{"PUT", "/foo/bar/gitlab-lfs/objects", "application/octet-stream", false},
{"PUT", "/foo/bar.git/gitlab-lfs/objects/zzz", "application/octet-stream", false},
{"PUT", "/.git/gitlab-lfs/objects", "application/octet-stream", false},
{"PUT", "/toplevel.git/gitlab-lfs/objects", "application/zzz", false},
{"POST", "/toplevel.git/gitlab-lfs/objects", "application/octet-stream", false},
}
for _, tc := range testCases {
t.Run(tc.path, func(t *testing.T) {
resource := fmt.Sprintf(tc.path+"/%s/%d", oid, len(reqBody))
req, err := http.NewRequest(
tc.method,
ws.URL+resource,
strings.NewReader(reqBody),
)
require.NoError(t, err)
req.Header.Set("Content-Type", tc.contentType)
req.ContentLength = int64(len(reqBody))
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
rspData, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
if tc.match {
require.Equal(t, 200, resp.StatusCode)
require.Equal(t, rspBody, string(rspData), "expect response generated by test upstream server")
} else {
require.Equal(t, 204, resp.StatusCode)
require.Empty(t, rspData, "normal request has empty response body")
}
})
}
}
func packageUploadTestServer(t *testing.T, resource string, reqBody string, rspBody string) *httptest.Server {
return testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, r.Method, "PUT")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7