Commit 1897fbbd authored by Jacob Vosmaer (GitLab)'s avatar Jacob Vosmaer (GitLab)

Merge branch 'artifacts-entry' into 'master'

Use Injecter to send entry instead of calling GitLab API

Implement https://gitlab.com/gitlab-org/gitlab-ce/issues/19224

I failed to update `internal/artifacts/artifact_download_test.go` :( Still trying how to fix that and `main_test.go`

See merge request !53
parents f3d5a7cb df5bbed5
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"log"
"mime" "mime"
"net/http" "net/http"
"os" "os"
...@@ -13,11 +14,40 @@ import ( ...@@ -13,11 +14,40 @@ import (
"strings" "strings"
"syscall" "syscall"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts" "gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
) )
type entry struct{ senddata.Prefix }
type entryParams struct{ Archive, Entry string }
var SendEntry = &entry{"artifacts-entry:"}
// Artifacts downloader doesn't support ranges when downloading a single file
func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
var params entryParams
if err := e.Unpack(&params, sendData); err != nil {
helper.Fail500(w, fmt.Errorf("SendEntry: unpack sendData: %v", err))
return
}
log.Printf("SendEntry: sending %q from %q for %q", params.Entry, params.Archive, r.URL.Path)
if params.Archive == "" || params.Entry == "" {
helper.Fail500(w, errors.New("SendEntry: Archive or Entry is empty"))
return
}
err := unpackFileFromZip(params.Archive, params.Entry, w.Header(), w)
if os.IsNotExist(err) {
http.NotFound(w, r)
} else if err != nil {
helper.Fail500(w, fmt.Errorf("SendEntry: %v", err))
}
}
func detectFileContentType(fileName string) string { func detectFileContentType(fileName string) string {
contentType := mime.TypeByExtension(filepath.Ext(fileName)) contentType := mime.TypeByExtension(filepath.Ext(fileName))
if contentType == "" { if contentType == "" {
...@@ -80,21 +110,3 @@ func waitCatFile(cmd *exec.Cmd) error { ...@@ -80,21 +110,3 @@ func waitCatFile(cmd *exec.Cmd) error {
return fmt.Errorf("wait for %v to finish: %v", cmd.Args, err) return fmt.Errorf("wait for %v to finish: %v", cmd.Args, err)
} }
// Artifacts downloader doesn't support ranges when downloading a single file
func DownloadArtifact(myAPI *api.API) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
if a.Archive == "" || a.Entry == "" {
helper.Fail500(w, errors.New("DownloadArtifact: Archive or Path is empty"))
return
}
err := unpackFileFromZip(a.Archive, a.Entry, w.Header(), w)
if os.IsNotExist(err) {
http.NotFound(w, r)
return
} else if err != nil {
helper.Fail500(w, fmt.Errorf("DownloadArtifact: %v", err))
}
}, "")
}
...@@ -3,7 +3,6 @@ package artifacts ...@@ -3,7 +3,6 @@ package artifacts
import ( import (
"archive/zip" "archive/zip"
"encoding/base64" "encoding/base64"
"encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
...@@ -11,40 +10,29 @@ import ( ...@@ -11,40 +10,29 @@ import (
"os" "os"
"testing" "testing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper" "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
) )
func testArtifactDownloadServer(t *testing.T, archive string, entry string) *httptest.Server { func testEntryServer(t *testing.T, archive string, entry string) *httptest.ResponseRecorder {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" { if r.Method != "GET" {
t.Fatal("Expected GET request") t.Fatal("Expected GET request")
} }
w.Header().Set("Content-Type", "application/json") encodedEntry := base64.StdEncoding.EncodeToString([]byte(entry))
jsonParams := fmt.Sprintf(`{"Archive":"%s","Entry":"%s"}`, archive, encodedEntry)
data := base64.URLEncoding.EncodeToString([]byte(jsonParams))
data, err := json.Marshal(&api.Response{ SendEntry.Inject(w, r, data)
Archive: archive,
Entry: base64.StdEncoding.EncodeToString([]byte(entry)),
}) })
if err != nil {
t.Fatal(err)
}
w.Write(data)
})
return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
}
func testDownloadArtifact(t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder { httpRequest, err := http.NewRequest("GET", "/url/path", nil)
httpRequest, err := http.NewRequest("GET", ts.URL+"/url/path", nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
response := httptest.NewRecorder() response := httptest.NewRecorder()
apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil) mux.ServeHTTP(response, httpRequest)
DownloadArtifact(apiClient).ServeHTTP(response, httpRequest)
return response return response
} }
...@@ -65,10 +53,8 @@ func TestDownloadingFromValidArchive(t *testing.T) { ...@@ -65,10 +53,8 @@ func TestDownloadingFromValidArchive(t *testing.T) {
fmt.Fprint(fileInArchive, "testtest") fmt.Fprint(fileInArchive, "testtest")
archive.Close() archive.Close()
ts := testArtifactDownloadServer(t, tempFile.Name(), "test.txt") response := testEntryServer(t, tempFile.Name(), "test.txt")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 200) testhelper.AssertResponseCode(t, response, 200)
testhelper.AssertResponseHeader(t, response, testhelper.AssertResponseHeader(t, response,
...@@ -93,25 +79,16 @@ func TestDownloadingNonExistingFile(t *testing.T) { ...@@ -93,25 +79,16 @@ func TestDownloadingNonExistingFile(t *testing.T) {
defer archive.Close() defer archive.Close()
archive.Close() archive.Close()
ts := testArtifactDownloadServer(t, tempFile.Name(), "test") response := testEntryServer(t, tempFile.Name(), "test")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 404) testhelper.AssertResponseCode(t, response, 404)
} }
func TestDownloadingFromInvalidArchive(t *testing.T) { func TestDownloadingFromInvalidArchive(t *testing.T) {
ts := testArtifactDownloadServer(t, "path/to/non/existing/file", "test") response := testEntryServer(t, "path/to/non/existing/file", "test")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 404) testhelper.AssertResponseCode(t, response, 404)
} }
func TestIncompleteApiResponse(t *testing.T) { func TestIncompleteApiResponse(t *testing.T) {
ts := testArtifactDownloadServer(t, "", "") response := testEntryServer(t, "", "")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 500) testhelper.AssertResponseCode(t, response, 500)
} }
...@@ -50,6 +50,7 @@ func (u *Upstream) configureRoutes() { ...@@ -50,6 +50,7 @@ func (u *Upstream) configureRoutes() {
git.SendBlob, git.SendBlob,
git.SendDiff, git.SendDiff,
git.SendPatch, git.SendPatch,
artifacts.SendEntry,
) )
u.Routes = []route{ u.Routes = []route{
...@@ -60,7 +61,6 @@ func (u *Upstream) configureRoutes() { ...@@ -60,7 +61,6 @@ func (u *Upstream) configureRoutes() {
route{"PUT", regexp.MustCompile(gitProjectPattern + `gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfs.PutStore(api, proxy)}, route{"PUT", regexp.MustCompile(gitProjectPattern + `gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfs.PutStore(api, proxy)},
// CI Artifacts // CI Artifacts
route{"GET", regexp.MustCompile(projectPattern + `builds/[0-9]+/artifacts/file/`), contentEncodingHandler(artifacts.DownloadArtifact(api))},
route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(artifacts.UploadArtifacts(api, proxy))}, route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(artifacts.UploadArtifacts(api, proxy))},
// Explicitly proxy API requests // Explicitly proxy API requests
......
...@@ -552,9 +552,9 @@ func TestArtifactsGetSingleFile(t *testing.T) { ...@@ -552,9 +552,9 @@ func TestArtifactsGetSingleFile(t *testing.T) {
resourcePath := `/namespace/project/builds/123/artifacts/file/` + fileName resourcePath := `/namespace/project/builds/123/artifacts/file/` + fileName
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`\A`+resourcePath+`\z`), func(w http.ResponseWriter, r *http.Request) { ts := testhelper.TestServerWithHandler(regexp.MustCompile(`\A`+resourcePath+`\z`), func(w http.ResponseWriter, r *http.Request) {
encodedFilename := base64.StdEncoding.EncodeToString([]byte(fileName)) encodedFilename := base64.StdEncoding.EncodeToString([]byte(fileName))
if _, err := fmt.Fprintf(w, `{"Archive":"%s","Entry":"%s"}`, archivePath, encodedFilename); err != nil { jsonParams := fmt.Sprintf(`{"Archive":"%s","Entry":"%s"}`, archivePath, encodedFilename)
t.Fatal(err) data := base64.URLEncoding.EncodeToString([]byte(jsonParams))
} w.Header().Set("Gitlab-Workhorse-Send-Data", "artifacts-entry:"+data)
return return
}) })
defer ts.Close() defer ts.Close()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment