Merge branch 'artifacts-metadata' into 'master'

Add support for new artifacts features

It is proof of concept implementation that misses following:

- [x] Saving metadata to file, and passing that file to rails,
- [x] Making it look nice,
- [x] Have `refactor-upstream` merged: https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/20
- [x] Add tests for metadata and single file downloads,
- [ ] Spawning separate process to verify zip archive **ONHOLD**: https://gitlab.com/gitlab-org/gitlab-workhorse/issues/17,
- [ ] Spawning separate process to download file from zip archive **ONHOLD**: https://gitlab.com/gitlab-org/gitlab-workhorse/issues/17


See merge request !28

internal/api/api.go

@@ -58,6 +58,10 @@ type Response struct {
 	// TmpPath is the path where we should store temporary files
 	// This is set by authorization middleware
 	TempPath string
+	// Archive is the path where the artifacts archive is stored
+	Archive string `json:"archive"`
+	// Entry is a filename inside the archive point to file that needs to be extracted
+	Entry string `json:"entry"`
 }
 
 // singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy

internal/artifacts/artifact_download.go

+package artifacts
+
+import (
+	"../api"
+	"../helper"
+	"archive/zip"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+func decodeFileEntry(entry string) (string, error) {
+	decoded, err := base64.StdEncoding.DecodeString(entry)
+	if err != nil {
+		return "", err
+	}
+	return string(decoded), nil
+}
+
+func detectFileContentType(fileName string) string {
+	contentType := mime.TypeByExtension(filepath.Ext(fileName))
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	return contentType
+}
+
+func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
+	for _, file := range archive.File {
+		if file.Name == fileName {
+			return file
+		}
+	}
+	return nil
+}
+
+func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error {
+	archive, err := zip.OpenReader(archiveFileName)
+	if err != nil {
+		return err
+	}
+	defer archive.Close()
+
+	file := findFileInZip(fileName, &archive.Reader)
+	if file == nil {
+		return os.ErrNotExist
+	}
+
+	// Start decompressing the file
+	reader, err := file.Open()
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+
+	basename := filepath.Base(fileName)
+
+	// Write http headers about the file
+	headers.Set("Content-Length", strconv.FormatInt(int64(file.UncompressedSize64), 10))
+	headers.Set("Content-Type", detectFileContentType(file.Name))
+	headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"")
+
+	// Copy file body to client
+	_, err = io.Copy(output, reader)
+	return err
+}
+
+// Artifacts downloader doesn't support ranges when downloading a single file
+func DownloadArtifact(myAPI *api.API) http.Handler {
+	return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
+		if a.Archive == "" || a.Entry == "" {
+			helper.Fail500(w, errors.New("DownloadArtifact: Archive or Path is empty"))
+			return
+		}
+
+		fileName, err := decodeFileEntry(a.Entry)
+		if err != nil {
+			helper.Fail500(w, err)
+			return
+		}
+
+		err = unpackFileFromZip(a.Archive, fileName, w.Header(), w)
+		if os.IsNotExist(err) {
+			http.NotFound(w, r)
+			return
+		} else if err != nil {
+			helper.Fail500(w, fmt.Errorf("DownloadArtifact: %v", err))
+		}
+	}, "")
+}

internal/artifacts/artifact_download_test.go

+package artifacts
+
+import (
+	"../api"
+	"../helper"
+	"../testhelper"
+	"archive/zip"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+)
+
+func testArtifactDownloadServer(t *testing.T, archive string, entry string) *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "GET" {
+			t.Fatal("Expected GET request")
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+
+		data, err := json.Marshal(&api.Response{
+			Archive: archive,
+			Entry:   base64.StdEncoding.EncodeToString([]byte(entry)),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		w.Write(data)
+	})
+	return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
+}
+
+func testDownloadArtifact(t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
+	httpRequest, err := http.NewRequest("GET", ts.URL+"/url/path", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	response := httptest.NewRecorder()
+	apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
+	DownloadArtifact(apiClient).ServeHTTP(response, httpRequest)
+	return response
+}
+
+func TestDownloadingFromValidArchive(t *testing.T) {
+	tempFile, err := ioutil.TempFile("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	archive := zip.NewWriter(tempFile)
+	defer archive.Close()
+	fileInArchive, err := archive.Create("test.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(fileInArchive, "testtest")
+	archive.Close()
+
+	ts := testArtifactDownloadServer(t, tempFile.Name(), "test.txt")
+	defer ts.Close()
+
+	response := testDownloadArtifact(t, ts)
+	testhelper.AssertResponseCode(t, response, 200)
+
+	testhelper.AssertResponseHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.AssertResponseHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"test.txt\"")
+
+	testhelper.AssertResponseBody(t, response, "testtest")
+}
+
+func TestDownloadingNonExistingFile(t *testing.T) {
+	tempFile, err := ioutil.TempFile("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	archive := zip.NewWriter(tempFile)
+	defer archive.Close()
+	archive.Close()
+
+	ts := testArtifactDownloadServer(t, tempFile.Name(), "test")
+	defer ts.Close()
+
+	response := testDownloadArtifact(t, ts)
+	testhelper.AssertResponseCode(t, response, 404)
+}
+
+func TestDownloadingFromInvalidArchive(t *testing.T) {
+	ts := testArtifactDownloadServer(t, "path/to/non/existing/file", "test")
+	defer ts.Close()
+
+	response := testDownloadArtifact(t, ts)
+	testhelper.AssertResponseCode(t, response, 404)
+}
+
+func TestIncompleteApiResponse(t *testing.T) {
+	ts := testArtifactDownloadServer(t, "", "")
+	defer ts.Close()
+
+	response := testDownloadArtifact(t, ts)
+	testhelper.AssertResponseCode(t, response, 500)
+}

internal/artifacts/artifacts_upload.go

+package artifacts
+
+import (
+	"../api"
+	"../helper"
+	"../upload"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"os"
+)
+
+type artifactsUploadProcessor struct {
+	TempPath     string
+	metadataFile string
+}
+
+func (a *artifactsUploadProcessor) ProcessFile(formName, fileName string, writer *multipart.Writer) error {
+	//  ProcessFile for artifacts requires file form-data field name to eq `file`
+
+	if formName != "file" {
+		return fmt.Errorf("Invalid form field: %q", formName)
+	}
+	if a.metadataFile != "" {
+		return fmt.Errorf("Artifacts request contains more than one file!")
+	}
+
+	// Create temporary file for metadata and store it's path
+	tempFile, err := ioutil.TempFile(a.TempPath, "metadata_")
+	if err != nil {
+		return err
+	}
+	defer tempFile.Close()
+	a.metadataFile = tempFile.Name()
+
+	// Generate metadata and save to file
+	err = generateZipMetadataFromFile(fileName, tempFile)
+	if err == os.ErrInvalid {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	// Pass metadata file path to Rails
+	writer.WriteField("metadata.path", a.metadataFile)
+	writer.WriteField("metadata.name", "metadata.gz")
+	return nil
+}
+
+func (a *artifactsUploadProcessor) ProcessField(formName string, writer *multipart.Writer) error {
+	return nil
+}
+
+func (a *artifactsUploadProcessor) Cleanup() {
+	if a.metadataFile != "" {
+		os.Remove(a.metadataFile)
+	}
+}
+
+func UploadArtifacts(myAPI *api.API, h http.Handler) http.Handler {
+	return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
+		if a.TempPath == "" {
+			helper.Fail500(w, errors.New("UploadArtifacts: TempPath is empty"))
+			return
+		}
+
+		mg := &artifactsUploadProcessor{TempPath: a.TempPath}
+		defer mg.Cleanup()
+
+		upload.HandleFileUploads(w, r, h, a.TempPath, mg)
+	}, "/authorize")
+}

internal/artifacts/artifacts_upload_test.go

+package artifacts
+
+import (
+	"../api"
+	"../helper"
+	"../proxy"
+	"../testhelper"
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+)
+
+func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/url/path/authorize", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "POST" {
+			t.Fatal("Expected POST request")
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+
+		data, err := json.Marshal(&api.Response{
+			TempPath: tempPath,
+		})
+		if err != nil {
+			t.Fatal("Expected to marshal")
+		}
+		w.Write(data)
+	})
+	mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "POST" {
+			t.Fatal("Expected POST request")
+		}
+		if r.FormValue("file.path") == "" {
+			w.WriteHeader(501)
+			return
+		}
+		if r.FormValue("metadata.path") == "" {
+			w.WriteHeader(502)
+			return
+		}
+
+		_, err := ioutil.ReadFile(r.FormValue("file.path"))
+		if err != nil {
+			w.WriteHeader(404)
+			return
+		}
+
+		metadata, err := ioutil.ReadFile(r.FormValue("metadata.path"))
+		if err != nil {
+			w.WriteHeader(404)
+			return
+		}
+		gz, err := gzip.NewReader(bytes.NewReader(metadata))
+		if err != nil {
+			w.WriteHeader(405)
+			return
+		}
+		defer gz.Close()
+		metadata, err = ioutil.ReadAll(gz)
+		if err != nil {
+			w.WriteHeader(404)
+			return
+		}
+		if !bytes.HasPrefix(metadata, []byte(metadataHeaderPrefix+metadataHeader)) {
+			w.WriteHeader(400)
+			return
+		}
+
+		w.WriteHeader(200)
+	})
+	return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
+}
+
+func testUploadArtifacts(contentType string, body io.Reader, t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
+	httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	httpRequest.Header.Set("Content-Type", contentType)
+	response := httptest.NewRecorder()
+	apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
+	proxyClient := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
+	UploadArtifacts(apiClient, proxyClient).ServeHTTP(response, httpRequest)
+	return response
+}
+
+func TestUploadHandlerAddingMetadata(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	ts := testArtifactsUploadServer(t, tempPath)
+	defer ts.Close()
+
+	var buffer bytes.Buffer
+	writer := multipart.NewWriter(&buffer)
+	file, err := writer.CreateFormFile("file", "my.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	archive := zip.NewWriter(file)
+	defer archive.Close()
+
+	fileInArchive, err := archive.Create("test.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(fileInArchive, "test")
+	archive.Close()
+	writer.Close()
+
+	response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
+	testhelper.AssertResponseCode(t, response, 200)
+}
+
+func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	ts := testArtifactsUploadServer(t, tempPath)
+	defer ts.Close()
+
+	var buffer bytes.Buffer
+	writer := multipart.NewWriter(&buffer)
+	file, err := writer.CreateFormFile("file", "my.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(file, "test")
+	writer.Close()
+
+	response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
+	// 502 is a custom response code from the mock server in testUploadArtifacts
+	testhelper.AssertResponseCode(t, response, 502)
+}
+
+func TestUploadFormProcessing(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	ts := testArtifactsUploadServer(t, tempPath)
+	defer ts.Close()
+
+	var buffer bytes.Buffer
+	writer := multipart.NewWriter(&buffer)
+	file, err := writer.CreateFormFile("metadata", "my.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(file, "test")
+	writer.Close()
+
+	response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
+	testhelper.AssertResponseCode(t, response, 500)
+}

internal/artifacts/escape_quotes.go

+package artifacts
+
+import "strings"
+
+// taken from mime/multipart/writer.go
+var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+
+func escapeQuotes(s string) string {
+	return quoteEscaper.Replace(s)
+}

internal/artifacts/metadata.go

+package artifacts
+
+import (
+	"archive/zip"
+	"compress/gzip"
+	"encoding/binary"
+	"encoding/json"
+	"io"
+	"os"
+	"strconv"
+)
+
+type metadata struct {
+	Modified int64  `json:"modified"`
+	Mode     string `json:"mode"`
+	CRC      uint32 `json:"crc,omitempty"`
+	Size     uint64 `json:"size,omitempty"`
+	Zipped   uint64 `json:"zipped,omitempty"`
+	Comment  string `json:"comment,omitempty"`
+}
+
+const metadataHeaderPrefix = "\x00\x00\x00&" // length of string below, encoded properly
+const metadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n"
+
+func newMetadata(file *zip.File) metadata {
+	return metadata{
+		Modified: file.ModTime().Unix(),
+		Mode:     strconv.FormatUint(uint64(file.Mode().Perm()), 8),
+		CRC:      file.CRC32,
+		Size:     file.UncompressedSize64,
+		Zipped:   file.CompressedSize64,
+		Comment:  file.Comment,
+	}
+}
+
+func (m metadata) writeEncoded(output io.Writer) error {
+	j, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+	j = append(j, byte('\n'))
+	return writeBytes(output, j)
+}
+
+func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
+	err := writeString(output, entry.Name)
+	if err != nil {
+		return err
+	}
+
+	err = newMetadata(entry).writeEncoded(output)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
+	err := writeString(output, metadataHeader)
+	if err != nil {
+		return err
+	}
+
+	// Write empty error string
+	err = writeString(output, "{}")
+	if err != nil {
+		return err
+	}
+
+	// Write all files
+	for _, entry := range archive.File {
+		err = writeZipEntryMetadata(output, entry)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func generateZipMetadataFromFile(fileName string, w io.Writer) error {
+	archive, err := zip.OpenReader(fileName)
+	if err != nil {
+		// Ignore non-zip archives
+		return os.ErrInvalid
+	}
+	defer archive.Close()
+
+	gz := gzip.NewWriter(w)
+	defer gz.Close()
+
+	return generateZipMetadata(gz, &archive.Reader)
+}
+
+func writeBytes(output io.Writer, data []byte) error {
+	err := binary.Write(output, binary.BigEndian, uint32(len(data)))
+	if err == nil {
+		_, err = output.Write(data)
+	}
+	return err
+}
+
+func writeString(output io.Writer, str string) error {
+	return writeBytes(output, []byte(str))
+}

internal/git/archive.go

@@ -23,6 +23,7 @@ import (
 func GetArchive(a *api.API) http.Handler {
 	return repoPreAuthorizeHandler(a, handleGetArchive)
 }
+
 func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
 	var format string
 	urlPath := r.URL.Path

internal/upload/artifacts.go

-package upload
-
-import (
-	"../api"
-	"net/http"
-)
-
-func Artifacts(myAPI *api.API, h http.Handler) http.Handler {
-	return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
-		r.Header.Set(tempPathHeader, a.TempPath)
-		handleFileUploads(h).ServeHTTP(w, r)
-	}, "/authorize")
-}

internal/upload/uploads.go

@@ -3,7 +3,6 @@ package upload
 import (
 	"../helper"
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,9 +11,12 @@ import (
 	"os"
 )
 
-const tempPathHeader = "Gitlab-Workhorse-Temp-Path"
+type MultipartFormProcessor interface {
+	ProcessFile(formName, fileName string, writer *multipart.Writer) error
+	ProcessField(formName string, writer *multipart.Writer) error
+}
 
-func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string) (cleanup func(), err error) {
+func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string, filter MultipartFormProcessor) (cleanup func(), err error) {
 	// Create multipart reader
 	reader, err := r.MultipartReader()
 	if err != nil {
@@ -67,10 +69,15 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
 			files = append(files, file.Name())
 
 			_, err = io.Copy(file, p)
-			file.Close()
 			if err != nil {
 				return cleanup, err
 			}
+
+			file.Close()
+
+			if err := filter.ProcessFile(name, file.Name(), writer); err != nil {
+				return cleanup, err
+			}
 		} else {
 			np, err := writer.CreatePart(p.Header)
 			if err != nil {
@@ -81,48 +88,48 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
 			if err != nil {
 				return cleanup, err
 			}
+
+			if err := filter.ProcessField(name, writer); err != nil {
+				return cleanup, err
+			}
 		}
 	}
 	return cleanup, nil
 }
 
-func handleFileUploads(h http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		tempPath := r.Header.Get(tempPathHeader)
-		if tempPath == "" {
-			helper.Fail500(w, errors.New("handleFileUploads: TempPath empty"))
-			return
-		}
-		r.Header.Del(tempPathHeader)
+func HandleFileUploads(w http.ResponseWriter, r *http.Request, h http.Handler, tempPath string, filter MultipartFormProcessor) {
+	if tempPath == "" {
+		helper.Fail500(w, fmt.Errorf("handleFileUploads: temporary path not defined"))
+		return
+	}
 
-		var body bytes.Buffer
-		writer := multipart.NewWriter(&body)
-		defer writer.Close()
+	var body bytes.Buffer
+	writer := multipart.NewWriter(&body)
+	defer writer.Close()
 
-		// Rewrite multipart form data
-		cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath)
-		if err != nil {
-			if err == http.ErrNotMultipart {
-				h.ServeHTTP(w, r)
-			} else {
-				helper.Fail500(w, fmt.Errorf("handleFileUploads: extract files from multipart: %v", err))
-			}
-			return
+	// Rewrite multipart form data
+	cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath, filter)
+	if err != nil {
+		if err == http.ErrNotMultipart {
+			h.ServeHTTP(w, r)
+		} else {
+			helper.Fail500(w, fmt.Errorf("handleFileUploads: extract files from multipart: %v", err))
 		}
+		return
+	}
 
-		if cleanup != nil {
-			defer cleanup()
-		}
+	if cleanup != nil {
+		defer cleanup()
+	}
 
-		// Close writer
-		writer.Close()
+	// Close writer
+	writer.Close()
 
-		// Hijack the request
-		r.Body = ioutil.NopCloser(&body)
-		r.ContentLength = int64(body.Len())
-		r.Header.Set("Content-Type", writer.FormDataContentType())
+	// Hijack the request
+	r.Body = ioutil.NopCloser(&body)
+	r.ContentLength = int64(body.Len())
+	r.Header.Set("Content-Type", writer.FormDataContentType())
 
-		// Proxy the request
-		h.ServeHTTP(w, r)
-	})
+	// Proxy the request
+	h.ServeHTTP(w, r)
 }

internal/upload/uploads_test.go

@@ -5,6 +5,7 @@ import (
 	"../proxy"
 	"../testhelper"
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -19,10 +20,27 @@ import (
 
 var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
 
+type testFormProcessor struct {
+}
+
+func (a *testFormProcessor) ProcessFile(formName, fileName string, writer *multipart.Writer) error {
+	if formName != "file" && fileName != "my.file" {
+		return errors.New("illegal file")
+	}
+	return nil
+}
+
+func (a *testFormProcessor) ProcessField(formName string, writer *multipart.Writer) error {
+	if formName != "token" {
+		return errors.New("illegal field")
+	}
+	return nil
+}
+
 func TestUploadTempPathRequirement(t *testing.T) {
 	response := httptest.NewRecorder()
 	request := &http.Request{}
-	handleFileUploads(nilHandler).ServeHTTP(response, request)
+	HandleFileUploads(response, request, nilHandler, "", nil)
 	testhelper.AssertResponseCode(t, response, 500)
 }
 
@@ -56,9 +74,8 @@ func TestUploadHandlerForwardingRawData(t *testing.T) {
 
 	response := httptest.NewRecorder()
 
-	httpRequest.Header.Set(tempPathHeader, tempPath)
-
-	handleFileUploads(proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)).ServeHTTP(response, httpRequest)
+	handler := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
+	HandleFileUploads(response, httpRequest, handler, tempPath, nil)
 	testhelper.AssertResponseCode(t, response, 202)
 	if response.Body.String() != "RESPONSE" {
 		t.Fatal("Expected RESPONSE in response body")
@@ -129,13 +146,65 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
 	httpRequest.Body = ioutil.NopCloser(&buffer)
 	httpRequest.ContentLength = int64(buffer.Len())
 	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
-	httpRequest.Header.Set(tempPathHeader, tempPath)
 	response := httptest.NewRecorder()
 
-	handleFileUploads(proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)).ServeHTTP(response, httpRequest)
+	handler := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
+	HandleFileUploads(response, httpRequest, handler, tempPath, &testFormProcessor{})
 	testhelper.AssertResponseCode(t, response, 202)
 
 	if _, err := os.Stat(filePath); !os.IsNotExist(err) {
 		t.Fatal("expected the file to be deleted")
 	}
 }
+
+func TestUploadProcessingField(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	var buffer bytes.Buffer
+
+	writer := multipart.NewWriter(&buffer)
+	writer.WriteField("token2", "test")
+	writer.Close()
+
+	httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
+
+	response := httptest.NewRecorder()
+	HandleFileUploads(response, httpRequest, nilHandler, tempPath, &testFormProcessor{})
+	testhelper.AssertResponseCode(t, response, 500)
+}
+
+func TestUploadProcessingFile(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	var buffer bytes.Buffer
+
+	writer := multipart.NewWriter(&buffer)
+	file, err := writer.CreateFormFile("file2", "my.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(file, "test")
+	writer.Close()
+
+	httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
+
+	response := httptest.NewRecorder()
+	HandleFileUploads(response, httpRequest, nilHandler, tempPath, &testFormProcessor{})
+	testhelper.AssertResponseCode(t, response, 500)
+}

internal/upstream/routes.go

@@ -2,11 +2,11 @@ package upstream
 
 import (
 	apipkg "../api"
+	"../artifacts"
 	"../git"
 	"../lfs"
 	proxypkg "../proxy"
 	"../staticpages"
-	"../upload"
 	"net/http"
 	"regexp"
 )
@@ -64,8 +64,9 @@ func (u *Upstream) configureRoutes() {
 		route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)},
 		route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)},
 
-		// CI Artifacts API
-		route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(upload.Artifacts(api, proxy))},
+		// CI Artifacts
+		route{"GET", regexp.MustCompile(projectPattern + `builds/[0-9]+/artifacts/file/`), contentEncodingHandler(artifacts.DownloadArtifact(api))},
+		route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(artifacts.UploadArtifacts(api, proxy))},
 
 		// Explicitly proxy API requests
 		route{"", regexp.MustCompile(apiPattern), proxy},

main_test.go

@@ -6,6 +6,7 @@ import (
 	"./internal/testhelper"
 	"./internal/upstream"
 	"bytes"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -488,6 +489,40 @@ func TestArtifactsUpload(t *testing.T) {
 	}
 }
 
+func TestArtifactsGetSingleFile(t *testing.T) {
+	// We manually created this zip file in the gitlab-workhorse Git repository
+	archivePath := `testdata/artifacts-archive.zip`
+	fileName := "myfile"
+	fileContents := "MY FILE"
+	resourcePath := `/namespace/project/builds/123/artifacts/file/` + fileName
+	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`\A`+resourcePath+`\z`), func(w http.ResponseWriter, r *http.Request) {
+		encodedFilename := base64.StdEncoding.EncodeToString([]byte(fileName))
+		if _, err := fmt.Fprintf(w, `{"Archive":"%s","Entry":"%s"}`, archivePath, encodedFilename); err != nil {
+			t.Fatal(err)
+		}
+		return
+	})
+	defer ts.Close()
+	ws := startWorkhorseServer(ts.URL)
+	defer ws.Close()
+
+	resp, err := http.Get(ws.URL + resourcePath)
+	if err != nil {
+		t.Error(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		t.Errorf("GET %q: expected 200, got %d", resourcePath, resp.StatusCode)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(body) != fileContents {
+		t.Fatalf("Expected file contents %q, got %q", fileContents, body)
+	}
+}
+
 func setupStaticFile(fpath, content string) error {
 	cwd, err := os.Getwd()
 	if err != nil {