Merge branch 'master' of https://gitlab.com/gitlab-org/gitlab-workhorse into raw-blob

.gitignore

@@ -2,3 +2,5 @@ gitlab-workhorse
 testdata/data
 testdata/scratch
 testdata/public
+gitlab-zip-cat
+gitlab-zip-metadata

.gitlab-ci.yml

 before_script:
-  - test -f /.dockerinit && apt-get update -qq && apt-get install -y curl unzip bzip2
-  - curl -O https://storage.googleapis.com/golang/go1.5.2.linux-amd64.tar.gz
-  - echo 'cae87ed095e8d94a81871281d35da7829bd1234e  go1.5.2.linux-amd64.tar.gz' | shasum -c -
-  - test -f /.dockerinit && rm -rf /usr/local/go && tar -C /usr/local -xzf go1.5.2.linux-amd64.tar.gz
+  - apt-get update -qq && apt-get install -y curl unzip bzip2
+  - curl -O https://storage.googleapis.com/golang/go1.5.3.linux-amd64.tar.gz
+  - echo '43afe0c5017e502630b1aea4d44b8a7f059bf60d7f29dfd58db454d4e4e0ae53  go1.5.3.linux-amd64.tar.gz' | shasum -c -a256 -
+  - rm -rf /usr/local/go && tar -C /usr/local -xzf go1.5.3.linux-amd64.tar.gz
   - export PATH=/usr/local/go/bin:$PATH
 
 test:

CHANGELOG

@@ -2,11 +2,20 @@
 
 Formerly known as 'gitlab-git-http-server'.
 
+0.6.1
+
+Add support for generating zip artifacts metadata and serving single
+files from zip archives.
+
+Gitlab-workhorse now consists of multiple executables. We also fixed a
+routing bug introduced by the 0.6.0 refactor that broke relative URL
+support.
+
 0.6.0
 
 Overhauled the source code organization; no user-facing changes
 (intended). The application code is now split into Go 'packages'
-(modules).
+(modules). As of 0.6.0 gitlab-workhorse requires Go 1.5 or newer.
 
 0.5.4
 

Makefile

 PREFIX=/usr/local
 VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
+GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
+
+all: gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
+
+gitlab-zip-cat:	$(shell find cmd/gitlab-zip-cat/ -name '*.go')
+	${GOBUILD} -o $@ ./cmd/$@
+	
+gitlab-zip-metadata:	$(shell find cmd/gitlab-zip-metadata/ -name '*.go')
+	${GOBUILD} -o $@ ./cmd/$@
 
 gitlab-workhorse: $(shell find . -name '*.go')
-	go build -ldflags "-X main.Version=${VERSION}" -o gitlab-workhorse
+	${GOBUILD} -o $@
 
-install: gitlab-workhorse
-	install gitlab-workhorse ${PREFIX}/bin/
+install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
+	mkdir -p $(DESTDIR)${PREFIX}/bin/
+	install gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata ${DESTDIR}${PREFIX}/bin/
 
 .PHONY: test
-test: testdata/data/group/test.git clean-workhorse gitlab-workhorse
+test: testdata/data/group/test.git clean-workhorse all
 	go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
-	go test ./...
+	support/path go test ./...
 	@echo SUCCESS
 
 coverage: testdata/data/group/test.git
@@ -30,4 +40,4 @@ clean:	clean-workhorse
 
 .PHONY:	clean-workhorse
 clean-workhorse:
-	rm -f gitlab-workhorse
+	rm -f gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata

VERSION

-0.6.0
+0.6.1

cmd/gitlab-zip-cat/main.go

+package main
+
+import (
+	"../../internal/zipartifacts"
+	"archive/zip"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+)
+
+const progName = "gitlab-zip-cat"
+
+var Version = "unknown"
+
+var printVersion = flag.Bool("version", false, "Print version and exit")
+
+func main() {
+	flag.Parse()
+
+	version := fmt.Sprintf("%s %s", progName, Version)
+	if *printVersion {
+		fmt.Println(version)
+		os.Exit(0)
+	}
+
+	if len(os.Args) != 3 {
+		fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP ENTRY", progName)
+		os.Exit(1)
+	}
+
+	archiveFileName := os.Args[1]
+
+	fileName, err := zipartifacts.DecodeFileEntry(os.Args[2])
+	if err != nil {
+		fatalError(fmt.Errorf("decode entry %q: %v", os.Args[2], err))
+	}
+
+	archive, err := zip.OpenReader(archiveFileName)
+	if err != nil {
+		notFoundError(fmt.Errorf("open %q: %v", archiveFileName, err))
+	}
+	defer archive.Close()
+
+	file := findFileInZip(fileName, &archive.Reader)
+	if file == nil {
+		notFoundError(fmt.Errorf("find %q in %q: not found", fileName, archiveFileName))
+	}
+	// Start decompressing the file
+	reader, err := file.Open()
+	if err != nil {
+		fatalError(fmt.Errorf("open %q in %q: %v", fileName, archiveFileName, err))
+	}
+	defer reader.Close()
+
+	if _, err := fmt.Printf("%d\n", file.UncompressedSize64); err != nil {
+		fatalError(fmt.Errorf("write file size: %v", err))
+	}
+
+	if _, err := io.Copy(os.Stdout, reader); err != nil {
+		fatalError(fmt.Errorf("write %q from %q to stdout: %v", fileName, archiveFileName, err))
+	}
+}
+
+func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
+	for _, file := range archive.File {
+		if file.Name == fileName {
+			return file
+		}
+	}
+	return nil
+}
+
+func printError(err error) {
+	fmt.Fprintf(os.Stderr, "%s: %v", progName, err)
+}
+
+func fatalError(err error) {
+	printError(err)
+	os.Exit(1)
+}
+
+func notFoundError(err error) {
+	printError(err)
+	os.Exit(zipartifacts.StatusEntryNotFound)
+}

cmd/gitlab-zip-metadata/main.go

+package main
+
+import (
+	"../../internal/zipartifacts"
+	"flag"
+	"fmt"
+	"os"
+)
+
+const progName = "gitlab-zip-metadata"
+
+var Version = "unknown"
+
+var printVersion = flag.Bool("version", false, "Print version and exit")
+
+func main() {
+	flag.Parse()
+
+	version := fmt.Sprintf("%s %s", progName, Version)
+	if *printVersion {
+		fmt.Println(version)
+		os.Exit(0)
+	}
+
+	if len(os.Args) != 2 {
+		fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP", progName)
+		os.Exit(1)
+	}
+	if err := zipartifacts.GenerateZipMetadataFromFile(os.Args[1], os.Stdout); err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %v\n", progName, err)
+		if err == os.ErrInvalid {
+			os.Exit(zipartifacts.StatusNotZip)
+		}
+		os.Exit(1)
+	}
+}

internal/api/api.go

@@ -57,6 +57,10 @@ type Response struct {
 	// TmpPath is the path where we should store temporary files
 	// This is set by authorization middleware
 	TempPath string
+	// Archive is the path where the artifacts archive is stored
+	Archive string `json:"archive"`
+	// Entry is a filename inside the archive point to file that needs to be extracted
+	Entry string `json:"entry"`
 }
 
 // singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy

internal/artifacts/artifact_download.go

+package artifacts
+
+import (
+	"../api"
+	"../helper"
+	"../zipartifacts"
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"syscall"
+)
+
+func detectFileContentType(fileName string) string {
+	contentType := mime.TypeByExtension(filepath.Ext(fileName))
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	return contentType
+}
+
+func unpackFileFromZip(archiveFileName, encodedFilename string, headers http.Header, output io.Writer) error {
+	fileName, err := zipartifacts.DecodeFileEntry(encodedFilename)
+	if err != nil {
+		return err
+	}
+
+	catFile := exec.Command("gitlab-zip-cat", archiveFileName, encodedFilename)
+	catFile.Stderr = os.Stderr
+	catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	stdout, err := catFile.StdoutPipe()
+	if err != nil {
+		return fmt.Errorf("create gitlab-zip-cat stdout pipe: %v", err)
+	}
+
+	if err := catFile.Start(); err != nil {
+		return fmt.Errorf("start %v: %v", catFile.Args, err)
+	}
+	defer helper.CleanUpProcessGroup(catFile)
+
+	basename := filepath.Base(fileName)
+	reader := bufio.NewReader(stdout)
+	contentLength, err := reader.ReadString('\n')
+	if err != nil {
+		if catFileErr := waitCatFile(catFile); catFileErr != nil {
+			return catFileErr
+		}
+		return fmt.Errorf("read content-length: %v", err)
+	}
+	contentLength = strings.TrimSuffix(contentLength, "\n")
+
+	// Write http headers about the file
+	headers.Set("Content-Length", contentLength)
+	headers.Set("Content-Type", detectFileContentType(fileName))
+	headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"")
+	// Copy file body to client
+	if _, err := io.Copy(output, reader); err != nil {
+		return fmt.Errorf("copy stdout of %v: %v", catFile.Args, err)
+	}
+
+	return waitCatFile(catFile)
+}
+
+func waitCatFile(cmd *exec.Cmd) error {
+	err := cmd.Wait()
+	if err == nil {
+		return nil
+	}
+
+	if st, ok := helper.ExitStatus(err); ok && st == zipartifacts.StatusEntryNotFound {
+		return os.ErrNotExist
+	}
+	return fmt.Errorf("wait for %v to finish: %v", cmd.Args, err)
+
+}
+
+// Artifacts downloader doesn't support ranges when downloading a single file
+func DownloadArtifact(myAPI *api.API) http.Handler {
+	return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
+		if a.Archive == "" || a.Entry == "" {
+			helper.Fail500(w, errors.New("DownloadArtifact: Archive or Path is empty"))
+			return
+		}
+
+		err := unpackFileFromZip(a.Archive, a.Entry, w.Header(), w)
+		if os.IsNotExist(err) {
+			http.NotFound(w, r)
+			return
+		} else if err != nil {
+			helper.Fail500(w, fmt.Errorf("DownloadArtifact: %v", err))
+		}
+	}, "")
+}

internal/artifacts/artifact_download_test.go

+package artifacts
+
+import (
+	"../api"
+	"../helper"
+	"../testhelper"
+	"archive/zip"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+)
+
+func testArtifactDownloadServer(t *testing.T, archive string, entry string) *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "GET" {
+			t.Fatal("Expected GET request")
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+
+		data, err := json.Marshal(&api.Response{
+			Archive: archive,
+			Entry:   base64.StdEncoding.EncodeToString([]byte(entry)),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		w.Write(data)
+	})
+	return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
+}
+
+func testDownloadArtifact(t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
+	httpRequest, err := http.NewRequest("GET", ts.URL+"/url/path", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	response := httptest.NewRecorder()
+	apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
+	DownloadArtifact(apiClient).ServeHTTP(response, httpRequest)
+	return response
+}
+
+func TestDownloadingFromValidArchive(t *testing.T) {
+	tempFile, err := ioutil.TempFile("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	archive := zip.NewWriter(tempFile)
+	defer archive.Close()
+	fileInArchive, err := archive.Create("test.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(fileInArchive, "testtest")
+	archive.Close()
+
+	ts := testArtifactDownloadServer(t, tempFile.Name(), "test.txt")
+	defer ts.Close()
+
+	response := testDownloadArtifact(t, ts)
+	testhelper.AssertResponseCode(t, response, 200)
+
+	testhelper.AssertResponseHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.AssertResponseHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"test.txt\"")
+
+	testhelper.AssertResponseBody(t, response, "testtest")
+}
+
+func TestDownloadingNonExistingFile(t *testing.T) {
+	tempFile, err := ioutil.TempFile("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	archive := zip.NewWriter(tempFile)
+	defer archive.Close()
+	archive.Close()
+
+	ts := testArtifactDownloadServer(t, tempFile.Name(), "test")
+	defer ts.Close()
+
+	response := testDownloadArtifact(t, ts)
+	testhelper.AssertResponseCode(t, response, 404)
+}
+
+func TestDownloadingFromInvalidArchive(t *testing.T) {
+	ts := testArtifactDownloadServer(t, "path/to/non/existing/file", "test")
+	defer ts.Close()
+
+	response := testDownloadArtifact(t, ts)
+	testhelper.AssertResponseCode(t, response, 404)
+}
+
+func TestIncompleteApiResponse(t *testing.T) {
+	ts := testArtifactDownloadServer(t, "", "")
+	defer ts.Close()
+
+	response := testDownloadArtifact(t, ts)
+	testhelper.AssertResponseCode(t, response, 500)
+}

internal/artifacts/artifacts_upload.go

+package artifacts
+
+import (
+	"../api"
+	"../helper"
+	"../upload"
+	"../zipartifacts"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"os/exec"
+	"syscall"
+)
+
+type artifactsUploadProcessor struct {
+	TempPath     string
+	metadataFile string
+}
+
+func (a *artifactsUploadProcessor) ProcessFile(formName, fileName string, writer *multipart.Writer) error {
+	//  ProcessFile for artifacts requires file form-data field name to eq `file`
+
+	if formName != "file" {
+		return fmt.Errorf("Invalid form field: %q", formName)
+	}
+	if a.metadataFile != "" {
+		return fmt.Errorf("Artifacts request contains more than one file!")
+	}
+
+	// Create temporary file for metadata and store it's path
+	tempFile, err := ioutil.TempFile(a.TempPath, "metadata_")
+	if err != nil {
+		return err
+	}
+	defer tempFile.Close()
+	a.metadataFile = tempFile.Name()
+
+	// Generate metadata and save to file
+	zipMd := exec.Command("gitlab-zip-metadata", fileName)
+	zipMd.Stderr = os.Stderr
+	zipMd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	zipMd.Stdout = tempFile
+
+	if err := zipMd.Start(); err != nil {
+		return err
+	}
+	defer helper.CleanUpProcessGroup(zipMd)
+	if err := zipMd.Wait(); err != nil {
+		if st, ok := helper.ExitStatus(err); ok && st == zipartifacts.StatusNotZip {
+			return nil
+		}
+		return err
+	}
+
+	// Pass metadata file path to Rails
+	writer.WriteField("metadata.path", a.metadataFile)
+	writer.WriteField("metadata.name", "metadata.gz")
+	return nil
+}
+
+func (a *artifactsUploadProcessor) ProcessField(formName string, writer *multipart.Writer) error {
+	return nil
+}
+
+func (a *artifactsUploadProcessor) Cleanup() {
+	if a.metadataFile != "" {
+		os.Remove(a.metadataFile)
+	}
+}
+
+func UploadArtifacts(myAPI *api.API, h http.Handler) http.Handler {
+	return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
+		if a.TempPath == "" {
+			helper.Fail500(w, errors.New("UploadArtifacts: TempPath is empty"))
+			return
+		}
+
+		mg := &artifactsUploadProcessor{TempPath: a.TempPath}
+		defer mg.Cleanup()
+
+		upload.HandleFileUploads(w, r, h, a.TempPath, mg)
+	}, "/authorize")
+}

internal/artifacts/artifacts_upload_test.go

+package artifacts
+
+import (
+	"../api"
+	"../helper"
+	"../proxy"
+	"../testhelper"
+	"../zipartifacts"
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+)
+
+func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/url/path/authorize", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "POST" {
+			t.Fatal("Expected POST request")
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+
+		data, err := json.Marshal(&api.Response{
+			TempPath: tempPath,
+		})
+		if err != nil {
+			t.Fatal("Expected to marshal")
+		}
+		w.Write(data)
+	})
+	mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "POST" {
+			t.Fatal("Expected POST request")
+		}
+		if r.FormValue("file.path") == "" {
+			w.WriteHeader(501)
+			return
+		}
+		if r.FormValue("metadata.path") == "" {
+			w.WriteHeader(502)
+			return
+		}
+
+		_, err := ioutil.ReadFile(r.FormValue("file.path"))
+		if err != nil {
+			w.WriteHeader(404)
+			return
+		}
+
+		metadata, err := ioutil.ReadFile(r.FormValue("metadata.path"))
+		if err != nil {
+			w.WriteHeader(404)
+			return
+		}
+		gz, err := gzip.NewReader(bytes.NewReader(metadata))
+		if err != nil {
+			w.WriteHeader(405)
+			return
+		}
+		defer gz.Close()
+		metadata, err = ioutil.ReadAll(gz)
+		if err != nil {
+			w.WriteHeader(404)
+			return
+		}
+		if !bytes.HasPrefix(metadata, []byte(zipartifacts.MetadataHeaderPrefix+zipartifacts.MetadataHeader)) {
+			w.WriteHeader(400)
+			return
+		}
+
+		w.WriteHeader(200)
+	})
+	return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
+}
+
+func testUploadArtifacts(contentType string, body io.Reader, t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
+	httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	httpRequest.Header.Set("Content-Type", contentType)
+	response := httptest.NewRecorder()
+	apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
+	proxyClient := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
+	UploadArtifacts(apiClient, proxyClient).ServeHTTP(response, httpRequest)
+	return response
+}
+
+func TestUploadHandlerAddingMetadata(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	ts := testArtifactsUploadServer(t, tempPath)
+	defer ts.Close()
+
+	var buffer bytes.Buffer
+	writer := multipart.NewWriter(&buffer)
+	file, err := writer.CreateFormFile("file", "my.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	archive := zip.NewWriter(file)
+	defer archive.Close()
+
+	fileInArchive, err := archive.Create("test.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(fileInArchive, "test")
+	archive.Close()
+	writer.Close()
+
+	response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
+	testhelper.AssertResponseCode(t, response, 200)
+}
+
+func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	ts := testArtifactsUploadServer(t, tempPath)
+	defer ts.Close()
+
+	var buffer bytes.Buffer
+	writer := multipart.NewWriter(&buffer)
+	file, err := writer.CreateFormFile("file", "my.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(file, "test")
+	writer.Close()
+
+	response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
+	// 502 is a custom response code from the mock server in testUploadArtifacts
+	testhelper.AssertResponseCode(t, response, 502)
+}
+
+func TestUploadFormProcessing(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	ts := testArtifactsUploadServer(t, tempPath)
+	defer ts.Close()
+
+	var buffer bytes.Buffer
+	writer := multipart.NewWriter(&buffer)
+	file, err := writer.CreateFormFile("metadata", "my.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(file, "test")
+	writer.Close()
+
+	response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
+	testhelper.AssertResponseCode(t, response, 500)
+}

internal/artifacts/escape_quotes.go

+package artifacts
+
+import "strings"
+
+// taken from mime/multipart/writer.go
+var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+
+func escapeQuotes(s string) string {
+	return quoteEscaper.Replace(s)
+}

internal/git/archive.go

@@ -23,6 +23,7 @@ import (
 func GetArchive(a *api.API) http.Handler {
 	return repoPreAuthorizeHandler(a, handleGetArchive)
 }
+
 func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
 	var format string
 	urlPath := r.URL.Path
@@ -78,7 +79,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
 		helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", archiveCmd.Args, err))
 		return
 	}
-	defer cleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up
+	defer helper.CleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up
 
 	var stdout io.ReadCloser
 	if compressCmd == nil {
@@ -98,7 +99,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
 			helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", compressCmd.Args, err))
 			return
 		}
-		defer cleanUpProcessGroup(compressCmd)
+		defer helper.CleanUpProcessGroup(compressCmd)
 
 		archiveStdout.Close()
 	}
@@ -110,7 +111,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
 	setArchiveHeaders(w, format, archiveFilename)
 	w.WriteHeader(200) // Don't bother with HTTP 500 from this point on, just return
 	if _, err := io.Copy(w, archiveReader); err != nil {
-		helper.LogError(fmt.Errorf("handleGetArchive: read: %v", err))
+		helper.LogError(fmt.Errorf("handleGetArchive: copy 'git archive' output: %v", err))
 		return
 	}
 	if err := archiveCmd.Wait(); err != nil {

internal/git/command.go

@@ -23,18 +23,3 @@ func gitCommand(gl_id string, name string, args ...string) *exec.Cmd {
 	cmd.Stderr = os.Stderr
 	return cmd
 }
-
-func cleanUpProcessGroup(cmd *exec.Cmd) {
-	if cmd == nil {
-		return
-	}
-
-	process := cmd.Process
-	if process != nil && process.Pid > 0 {
-		// Send SIGTERM to the process group of cmd
-		syscall.Kill(-process.Pid, syscall.SIGTERM)
-	}
-
-	// reap our child process
-	cmd.Wait()
-}

internal/git/git-http.go

@@ -72,7 +72,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
 		helper.Fail500(w, fmt.Errorf("handleGetInfoRefs: start %v: %v", cmd.Args, err))
 		return
 	}
-	defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
+	defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
 
 	// Start writing the response
 	w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
@@ -87,7 +87,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
 		return
 	}
 	if _, err := io.Copy(w, stdout); err != nil {
-		helper.LogError(fmt.Errorf("handleGetInfoRefs: read from %v: %v", cmd.Args, err))
+		helper.LogError(fmt.Errorf("handleGetInfoRefs: copy output of %v: %v", cmd.Args, err))
 		return
 	}
 	if err := cmd.Wait(); err != nil {
@@ -125,7 +125,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
 		helper.Fail500(w, fmt.Errorf("handlePostRPC: start %v: %v", cmd.Args, err))
 		return
 	}
-	defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
+	defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
 
 	// Write the client request body to Git's standard input
 	if _, err := io.Copy(stdin, r.Body); err != nil {
@@ -146,7 +146,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
 
 	// This io.Copy may take a long time, both for Git push and pull.
 	if _, err := io.Copy(w, stdout); err != nil {
-		helper.LogError(fmt.Errorf("handlePostRPC read from %v: %v", cmd.Args, err))
+		helper.LogError(fmt.Errorf("handlePostRPC copy output of %v: %v", cmd.Args, err))
 		return
 	}
 	if err := cmd.Wait(); err != nil {

internal/helper/helpers.go

@@ -6,6 +6,8 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
+	"syscall"
 )
 
 func Fail500(w http.ResponseWriter, err error) {
@@ -79,3 +81,32 @@ func HeaderClone(h http.Header) http.Header {
 	}
 	return h2
 }
+
+func CleanUpProcessGroup(cmd *exec.Cmd) {
+	if cmd == nil {
+		return
+	}
+
+	process := cmd.Process
+	if process != nil && process.Pid > 0 {
+		// Send SIGTERM to the process group of cmd
+		syscall.Kill(-process.Pid, syscall.SIGTERM)
+	}
+
+	// reap our child process
+	cmd.Wait()
+}
+
+func ExitStatus(err error) (int, bool) {
+	exitError, ok := err.(*exec.ExitError)
+	if !ok {
+		return 0, false
+	}
+
+	waitStatus, ok := exitError.Sys().(syscall.WaitStatus)
+	if !ok {
+		return 0, false
+	}
+
+	return waitStatus.ExitStatus(), true
+}

internal/lfs/lfs.go

@@ -61,7 +61,7 @@ func handleStoreLfsObject(h http.Handler) api.HandleFunc {
 
 		written, err := io.Copy(hw, r.Body)
 		if err != nil {
-			helper.Fail500(w, fmt.Errorf("handleStoreLfsObject: write tempfile: %v", err))
+			helper.Fail500(w, fmt.Errorf("handleStoreLfsObject: copy body to tempfile: %v", err))
 			return
 		}
 		file.Close()

internal/upload/artifacts.go

-package upload
-
-import (
-	"../api"
-	"net/http"
-)
-
-func Artifacts(myAPI *api.API, h http.Handler) http.Handler {
-	return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
-		r.Header.Set(tempPathHeader, a.TempPath)
-		handleFileUploads(h).ServeHTTP(w, r)
-	}, "/authorize")
-}

internal/upload/uploads.go

@@ -3,7 +3,6 @@ package upload
 import (
 	"../helper"
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,13 +11,20 @@ import (
 	"os"
 )
 
-const tempPathHeader = "Gitlab-Workhorse-Temp-Path"
+type MultipartFormProcessor interface {
+	ProcessFile(formName, fileName string, writer *multipart.Writer) error
+	ProcessField(formName string, writer *multipart.Writer) error
+}
 
-func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string) (cleanup func(), err error) {
+func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string, filter MultipartFormProcessor) (cleanup func(), err error) {
 	// Create multipart reader
 	reader, err := r.MultipartReader()
 	if err != nil {
-		return nil, err
+		if err == http.ErrNotMultipart {
+			// We want to be able to recognize http.ErrNotMultipart elsewhere so no fmt.Errorf
+			return nil, http.ErrNotMultipart
+		}
+		return nil, fmt.Errorf("get multipart reader: %v", err)
 	}
 
 	var files []string
@@ -51,13 +57,13 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
 		if filename := p.FileName(); filename != "" {
 			// Create temporary directory where the uploaded file will be stored
 			if err := os.MkdirAll(tempPath, 0700); err != nil {
-				return cleanup, err
+				return cleanup, fmt.Errorf("mkdir for tempfile: %v", err)
 			}
 
 			// Create temporary file in path returned by Authorization filter
 			file, err := ioutil.TempFile(tempPath, "upload_")
 			if err != nil {
-				return cleanup, err
+				return cleanup, fmt.Errorf("create tempfile: %v", err)
 			}
 			defer file.Close()
 
@@ -67,40 +73,46 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
 			files = append(files, file.Name())
 
 			_, err = io.Copy(file, p)
-			file.Close()
 			if err != nil {
+				return cleanup, fmt.Errorf("copy from multipart to tempfile: %v", err)
+			}
+
+			file.Close()
+
+			if err := filter.ProcessFile(name, file.Name(), writer); err != nil {
 				return cleanup, err
 			}
 		} else {
 			np, err := writer.CreatePart(p.Header)
 			if err != nil {
-				return cleanup, err
+				return cleanup, fmt.Errorf("create multipart field: %v", err)
 			}
 
 			_, err = io.Copy(np, p)
 			if err != nil {
-				return cleanup, err
+				return cleanup, fmt.Errorf("duplicate multipart field: %v", err)
+			}
+
+			if err := filter.ProcessField(name, writer); err != nil {
+				return cleanup, fmt.Errorf("process multipart field: %v", err)
 			}
 		}
 	}
 	return cleanup, nil
 }
 
-func handleFileUploads(h http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		tempPath := r.Header.Get(tempPathHeader)
+func HandleFileUploads(w http.ResponseWriter, r *http.Request, h http.Handler, tempPath string, filter MultipartFormProcessor) {
 	if tempPath == "" {
-			helper.Fail500(w, errors.New("handleFileUploads: TempPath empty"))
+		helper.Fail500(w, fmt.Errorf("handleFileUploads: tempPath empty"))
 		return
 	}
-		r.Header.Del(tempPathHeader)
 
 	var body bytes.Buffer
 	writer := multipart.NewWriter(&body)
 	defer writer.Close()
 
 	// Rewrite multipart form data
-		cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath)
+	cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath, filter)
 	if err != nil {
 		if err == http.ErrNotMultipart {
 			h.ServeHTTP(w, r)
@@ -124,5 +136,4 @@ func handleFileUploads(h http.Handler) http.Handler {
 
 	// Proxy the request
 	h.ServeHTTP(w, r)
-	})
 }

internal/upload/uploads_test.go

@@ -5,6 +5,7 @@ import (
 	"../proxy"
 	"../testhelper"
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -19,10 +20,27 @@ import (
 
 var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
 
+type testFormProcessor struct {
+}
+
+func (a *testFormProcessor) ProcessFile(formName, fileName string, writer *multipart.Writer) error {
+	if formName != "file" && fileName != "my.file" {
+		return errors.New("illegal file")
+	}
+	return nil
+}
+
+func (a *testFormProcessor) ProcessField(formName string, writer *multipart.Writer) error {
+	if formName != "token" {
+		return errors.New("illegal field")
+	}
+	return nil
+}
+
 func TestUploadTempPathRequirement(t *testing.T) {
 	response := httptest.NewRecorder()
 	request := &http.Request{}
-	handleFileUploads(nilHandler).ServeHTTP(response, request)
+	HandleFileUploads(response, request, nilHandler, "", nil)
 	testhelper.AssertResponseCode(t, response, 500)
 }
 
@@ -56,9 +74,8 @@ func TestUploadHandlerForwardingRawData(t *testing.T) {
 
 	response := httptest.NewRecorder()
 
-	httpRequest.Header.Set(tempPathHeader, tempPath)
-
-	handleFileUploads(proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)).ServeHTTP(response, httpRequest)
+	handler := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
+	HandleFileUploads(response, httpRequest, handler, tempPath, nil)
 	testhelper.AssertResponseCode(t, response, 202)
 	if response.Body.String() != "RESPONSE" {
 		t.Fatal("Expected RESPONSE in response body")
@@ -129,13 +146,65 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
 	httpRequest.Body = ioutil.NopCloser(&buffer)
 	httpRequest.ContentLength = int64(buffer.Len())
 	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
-	httpRequest.Header.Set(tempPathHeader, tempPath)
 	response := httptest.NewRecorder()
 
-	handleFileUploads(proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)).ServeHTTP(response, httpRequest)
+	handler := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
+	HandleFileUploads(response, httpRequest, handler, tempPath, &testFormProcessor{})
 	testhelper.AssertResponseCode(t, response, 202)
 
 	if _, err := os.Stat(filePath); !os.IsNotExist(err) {
 		t.Fatal("expected the file to be deleted")
 	}
 }
+
+func TestUploadProcessingField(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	var buffer bytes.Buffer
+
+	writer := multipart.NewWriter(&buffer)
+	writer.WriteField("token2", "test")
+	writer.Close()
+
+	httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
+
+	response := httptest.NewRecorder()
+	HandleFileUploads(response, httpRequest, nilHandler, tempPath, &testFormProcessor{})
+	testhelper.AssertResponseCode(t, response, 500)
+}
+
+func TestUploadProcessingFile(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	var buffer bytes.Buffer
+
+	writer := multipart.NewWriter(&buffer)
+	file, err := writer.CreateFormFile("file2", "my.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(file, "test")
+	writer.Close()
+
+	httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
+
+	response := httptest.NewRecorder()
+	HandleFileUploads(response, httpRequest, nilHandler, tempPath, &testFormProcessor{})
+	testhelper.AssertResponseCode(t, response, 500)
+}

internal/upstream/routes.go

@@ -2,11 +2,11 @@ package upstream
 
 import (
 	apipkg "../api"
+	"../artifacts"
 	"../git"
 	"../lfs"
 	proxypkg "../proxy"
 	"../staticpages"
-	"../upload"
 	"net/http"
 	"regexp"
 )
@@ -64,8 +64,9 @@ func (u *Upstream) configureRoutes() {
 		route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)},
 		route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)},
 
-		// CI Artifacts API
-		route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(upload.Artifacts(api, proxy))},
+		// CI Artifacts
+		route{"GET", regexp.MustCompile(projectPattern + `builds/[0-9]+/artifacts/file/`), contentEncodingHandler(artifacts.DownloadArtifact(api))},
+		route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(artifacts.UploadArtifacts(api, proxy))},
 
 		// Explicitly proxy API requests
 		route{"", regexp.MustCompile(apiPattern), proxy},

internal/upstream/upstream.go

@@ -41,8 +41,8 @@ func NewUpstream(backend *url.URL, socket string, version string, documentRoot s
 	if backend == nil {
 		up.Backend = DefaultBackend
 	}
-	up.configureRoutes()
 	up.configureURLPrefix()
+	up.configureRoutes()
 	return &up
 }
 

internal/zipartifacts/codes.go

+package zipartifacts
+
+// These are exit codes used by subprocesses in cmd/gitlab-zip-xxx
+const (
+	StatusNotZip = 10 + iota
+	StatusEntryNotFound
+)

internal/zipartifacts/entry.go

+package zipartifacts
+
+import (
+	"encoding/base64"
+)
+
+func DecodeFileEntry(entry string) (string, error) {
+	decoded, err := base64.StdEncoding.DecodeString(entry)
+	if err != nil {
+		return "", err
+	}
+	return string(decoded), nil
+}

internal/zipartifacts/metadata.go

+package zipartifacts
+
+import (
+	"archive/zip"
+	"compress/gzip"
+	"encoding/binary"
+	"encoding/json"
+	"io"
+	"os"
+	"strconv"
+)
+
+type metadata struct {
+	Modified int64  `json:"modified"`
+	Mode     string `json:"mode"`
+	CRC      uint32 `json:"crc,omitempty"`
+	Size     uint64 `json:"size,omitempty"`
+	Zipped   uint64 `json:"zipped,omitempty"`
+	Comment  string `json:"comment,omitempty"`
+}
+
+const MetadataHeaderPrefix = "\x00\x00\x00&" // length of string below, encoded properly
+const MetadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n"
+
+func newMetadata(file *zip.File) metadata {
+	return metadata{
+		Modified: file.ModTime().Unix(),
+		Mode:     strconv.FormatUint(uint64(file.Mode().Perm()), 8),
+		CRC:      file.CRC32,
+		Size:     file.UncompressedSize64,
+		Zipped:   file.CompressedSize64,
+		Comment:  file.Comment,
+	}
+}
+
+func (m metadata) writeEncoded(output io.Writer) error {
+	j, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+	j = append(j, byte('\n'))
+	return writeBytes(output, j)
+}
+
+func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
+	err := writeString(output, entry.Name)
+	if err != nil {
+		return err
+	}
+
+	err = newMetadata(entry).writeEncoded(output)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
+	err := writeString(output, MetadataHeader)
+	if err != nil {
+		return err
+	}
+
+	// Write empty error string
+	err = writeString(output, "{}")
+	if err != nil {
+		return err
+	}
+
+	// Write all files
+	for _, entry := range archive.File {
+		err = writeZipEntryMetadata(output, entry)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func GenerateZipMetadataFromFile(fileName string, w io.Writer) error {
+	archive, err := zip.OpenReader(fileName)
+	if err != nil {
+		// Ignore non-zip archives
+		return os.ErrInvalid
+	}
+	defer archive.Close()
+
+	gz := gzip.NewWriter(w)
+	defer gz.Close()
+
+	return generateZipMetadata(gz, &archive.Reader)
+}
+
+func writeBytes(output io.Writer, data []byte) error {
+	err := binary.Write(output, binary.BigEndian, uint32(len(data)))
+	if err == nil {
+		_, err = output.Write(data)
+	}
+	return err
+}
+
+func writeString(output io.Writer, str string) error {
+	return writeBytes(output, []byte(str))
+}

main_test.go

@@ -6,6 +6,7 @@ import (
 	"./internal/testhelper"
 	"./internal/upstream"
 	"bytes"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -363,6 +364,37 @@ func TestAllowedStaticFile(t *testing.T) {
 	}
 }
 
+func TestStaticFileRelativeURL(t *testing.T) {
+	content := "PUBLIC"
+	if err := setupStaticFile("static.txt", content); err != nil {
+		t.Fatalf("create public/static.txt: %v", err)
+	}
+
+	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), http.HandlerFunc(http.NotFound))
+	defer ts.Close()
+	backendURLString := ts.URL + "/my-relative-url"
+	log.Print(backendURLString)
+	ws := startWorkhorseServer(backendURLString)
+	defer ws.Close()
+
+	resource := "/my-relative-url/static.txt"
+	resp, err := http.Get(ws.URL + resource)
+	if err != nil {
+		t.Error(err)
+	}
+	defer resp.Body.Close()
+	buf := &bytes.Buffer{}
+	if _, err := io.Copy(buf, resp.Body); err != nil {
+		t.Error(err)
+	}
+	if buf.String() != content {
+		t.Errorf("GET %q: Expected %q, got %q", resource, content, buf.String())
+	}
+	if resp.StatusCode != 200 {
+		t.Errorf("GET %q: expected 200, got %d", resource, resp.StatusCode)
+	}
+}
+
 func TestAllowedPublicUploadsFile(t *testing.T) {
 	content := "PRIVATE but allowed"
 	if err := setupStaticFile("uploads/static file.txt", content); err != nil {
@@ -465,8 +497,9 @@ func TestArtifactsUpload(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		if len(r.MultipartForm.Value) != 2 { // 1 file name, 1 file path
-			t.Error("Expected to receive exactly 2 values")
+		nValues := 2 // filename + path for just the upload (no metadata because we are not POSTing a valid zip file)
+		if len(r.MultipartForm.Value) != nValues {
+			t.Errorf("Expected to receive exactly %d values", nValues)
 		}
 		if len(r.MultipartForm.File) != 0 {
 			t.Error("Expected to not receive any files")
@@ -488,6 +521,40 @@ func TestArtifactsUpload(t *testing.T) {
 	}
 }
 
+func TestArtifactsGetSingleFile(t *testing.T) {
+	// We manually created this zip file in the gitlab-workhorse Git repository
+	archivePath := `testdata/artifacts-archive.zip`
+	fileName := "myfile"
+	fileContents := "MY FILE"
+	resourcePath := `/namespace/project/builds/123/artifacts/file/` + fileName
+	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`\A`+resourcePath+`\z`), func(w http.ResponseWriter, r *http.Request) {
+		encodedFilename := base64.StdEncoding.EncodeToString([]byte(fileName))
+		if _, err := fmt.Fprintf(w, `{"Archive":"%s","Entry":"%s"}`, archivePath, encodedFilename); err != nil {
+			t.Fatal(err)
+		}
+		return
+	})
+	defer ts.Close()
+	ws := startWorkhorseServer(ts.URL)
+	defer ws.Close()
+
+	resp, err := http.Get(ws.URL + resourcePath)
+	if err != nil {
+		t.Error(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		t.Errorf("GET %q: expected 200, got %d", resourcePath, resp.StatusCode)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(body) != fileContents {
+		t.Fatalf("Expected file contents %q, got %q", fileContents, body)
+	}
+}
+
 func setupStaticFile(fpath, content string) error {
 	cwd, err := os.Getwd()
 	if err != nil {

support/path

+#!/bin/sh
+exec env PATH=$(pwd):${PATH} "$@"