Commit 51d1256c authored by Jacob Vosmaer's avatar Jacob Vosmaer

Merge branch 'master' of https://gitlab.com/gitlab-org/gitlab-workhorse into raw-blob

parents 56ac73c7 4fe063cf
......@@ -2,3 +2,5 @@ gitlab-workhorse
testdata/data
testdata/scratch
testdata/public
gitlab-zip-cat
gitlab-zip-metadata
before_script:
- test -f /.dockerinit && apt-get update -qq && apt-get install -y curl unzip bzip2
- curl -O https://storage.googleapis.com/golang/go1.5.2.linux-amd64.tar.gz
- echo 'cae87ed095e8d94a81871281d35da7829bd1234e go1.5.2.linux-amd64.tar.gz' | shasum -c -
- test -f /.dockerinit && rm -rf /usr/local/go && tar -C /usr/local -xzf go1.5.2.linux-amd64.tar.gz
- apt-get update -qq && apt-get install -y curl unzip bzip2
- curl -O https://storage.googleapis.com/golang/go1.5.3.linux-amd64.tar.gz
- echo '43afe0c5017e502630b1aea4d44b8a7f059bf60d7f29dfd58db454d4e4e0ae53 go1.5.3.linux-amd64.tar.gz' | shasum -c -a256 -
- rm -rf /usr/local/go && tar -C /usr/local -xzf go1.5.3.linux-amd64.tar.gz
- export PATH=/usr/local/go/bin:$PATH
test:
......
......@@ -2,11 +2,20 @@
Formerly known as 'gitlab-git-http-server'.
0.6.1
Add support for generating zip artifacts metadata and serving single
files from zip archives.
Gitlab-workhorse now consists of multiple executables. We also fixed a
routing bug introduced by the 0.6.0 refactor that broke relative URL
support.
0.6.0
Overhauled the source code organization; no user-facing changes
(intended). The application code is now split into Go 'packages'
(modules).
(modules). As of 0.6.0 gitlab-workhorse requires Go 1.5 or newer.
0.5.4
......
PREFIX=/usr/local
VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
all: gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
gitlab-zip-cat: $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-zip-metadata: $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-workhorse: $(shell find . -name '*.go')
go build -ldflags "-X main.Version=${VERSION}" -o gitlab-workhorse
${GOBUILD} -o $@
install: gitlab-workhorse
install gitlab-workhorse ${PREFIX}/bin/
install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
mkdir -p $(DESTDIR)${PREFIX}/bin/
install gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata ${DESTDIR}${PREFIX}/bin/
.PHONY: test
test: testdata/data/group/test.git clean-workhorse gitlab-workhorse
test: testdata/data/group/test.git clean-workhorse all
go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
go test ./...
support/path go test ./...
@echo SUCCESS
coverage: testdata/data/group/test.git
......@@ -30,4 +40,4 @@ clean: clean-workhorse
.PHONY: clean-workhorse
clean-workhorse:
rm -f gitlab-workhorse
rm -f gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
package main
import (
"../../internal/zipartifacts"
"archive/zip"
"flag"
"fmt"
"io"
"os"
)
const progName = "gitlab-zip-cat"
var Version = "unknown"
var printVersion = flag.Bool("version", false, "Print version and exit")
func main() {
flag.Parse()
version := fmt.Sprintf("%s %s", progName, Version)
if *printVersion {
fmt.Println(version)
os.Exit(0)
}
if len(os.Args) != 3 {
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP ENTRY", progName)
os.Exit(1)
}
archiveFileName := os.Args[1]
fileName, err := zipartifacts.DecodeFileEntry(os.Args[2])
if err != nil {
fatalError(fmt.Errorf("decode entry %q: %v", os.Args[2], err))
}
archive, err := zip.OpenReader(archiveFileName)
if err != nil {
notFoundError(fmt.Errorf("open %q: %v", archiveFileName, err))
}
defer archive.Close()
file := findFileInZip(fileName, &archive.Reader)
if file == nil {
notFoundError(fmt.Errorf("find %q in %q: not found", fileName, archiveFileName))
}
// Start decompressing the file
reader, err := file.Open()
if err != nil {
fatalError(fmt.Errorf("open %q in %q: %v", fileName, archiveFileName, err))
}
defer reader.Close()
if _, err := fmt.Printf("%d\n", file.UncompressedSize64); err != nil {
fatalError(fmt.Errorf("write file size: %v", err))
}
if _, err := io.Copy(os.Stdout, reader); err != nil {
fatalError(fmt.Errorf("write %q from %q to stdout: %v", fileName, archiveFileName, err))
}
}
func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
for _, file := range archive.File {
if file.Name == fileName {
return file
}
}
return nil
}
func printError(err error) {
fmt.Fprintf(os.Stderr, "%s: %v", progName, err)
}
func fatalError(err error) {
printError(err)
os.Exit(1)
}
func notFoundError(err error) {
printError(err)
os.Exit(zipartifacts.StatusEntryNotFound)
}
package main
import (
"../../internal/zipartifacts"
"flag"
"fmt"
"os"
)
const progName = "gitlab-zip-metadata"
var Version = "unknown"
var printVersion = flag.Bool("version", false, "Print version and exit")
func main() {
flag.Parse()
version := fmt.Sprintf("%s %s", progName, Version)
if *printVersion {
fmt.Println(version)
os.Exit(0)
}
if len(os.Args) != 2 {
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP", progName)
os.Exit(1)
}
if err := zipartifacts.GenerateZipMetadataFromFile(os.Args[1], os.Stdout); err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", progName, err)
if err == os.ErrInvalid {
os.Exit(zipartifacts.StatusNotZip)
}
os.Exit(1)
}
}
......@@ -57,6 +57,10 @@ type Response struct {
// TmpPath is the path where we should store temporary files
// This is set by authorization middleware
TempPath string
// Archive is the path where the artifacts archive is stored
Archive string `json:"archive"`
// Entry is a filename inside the archive point to file that needs to be extracted
Entry string `json:"entry"`
}
// singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy
......
package artifacts
import (
"../api"
"../helper"
"../zipartifacts"
"bufio"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
)
func detectFileContentType(fileName string) string {
contentType := mime.TypeByExtension(filepath.Ext(fileName))
if contentType == "" {
contentType = "application/octet-stream"
}
return contentType
}
func unpackFileFromZip(archiveFileName, encodedFilename string, headers http.Header, output io.Writer) error {
fileName, err := zipartifacts.DecodeFileEntry(encodedFilename)
if err != nil {
return err
}
catFile := exec.Command("gitlab-zip-cat", archiveFileName, encodedFilename)
catFile.Stderr = os.Stderr
catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
stdout, err := catFile.StdoutPipe()
if err != nil {
return fmt.Errorf("create gitlab-zip-cat stdout pipe: %v", err)
}
if err := catFile.Start(); err != nil {
return fmt.Errorf("start %v: %v", catFile.Args, err)
}
defer helper.CleanUpProcessGroup(catFile)
basename := filepath.Base(fileName)
reader := bufio.NewReader(stdout)
contentLength, err := reader.ReadString('\n')
if err != nil {
if catFileErr := waitCatFile(catFile); catFileErr != nil {
return catFileErr
}
return fmt.Errorf("read content-length: %v", err)
}
contentLength = strings.TrimSuffix(contentLength, "\n")
// Write http headers about the file
headers.Set("Content-Length", contentLength)
headers.Set("Content-Type", detectFileContentType(fileName))
headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"")
// Copy file body to client
if _, err := io.Copy(output, reader); err != nil {
return fmt.Errorf("copy stdout of %v: %v", catFile.Args, err)
}
return waitCatFile(catFile)
}
func waitCatFile(cmd *exec.Cmd) error {
err := cmd.Wait()
if err == nil {
return nil
}
if st, ok := helper.ExitStatus(err); ok && st == zipartifacts.StatusEntryNotFound {
return os.ErrNotExist
}
return fmt.Errorf("wait for %v to finish: %v", cmd.Args, err)
}
// Artifacts downloader doesn't support ranges when downloading a single file
func DownloadArtifact(myAPI *api.API) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
if a.Archive == "" || a.Entry == "" {
helper.Fail500(w, errors.New("DownloadArtifact: Archive or Path is empty"))
return
}
err := unpackFileFromZip(a.Archive, a.Entry, w.Header(), w)
if os.IsNotExist(err) {
http.NotFound(w, r)
return
} else if err != nil {
helper.Fail500(w, fmt.Errorf("DownloadArtifact: %v", err))
}
}, "")
}
package artifacts
import (
"../api"
"../helper"
"../testhelper"
"archive/zip"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func testArtifactDownloadServer(t *testing.T, archive string, entry string) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Fatal("Expected GET request")
}
w.Header().Set("Content-Type", "application/json")
data, err := json.Marshal(&api.Response{
Archive: archive,
Entry: base64.StdEncoding.EncodeToString([]byte(entry)),
})
if err != nil {
t.Fatal(err)
}
w.Write(data)
})
return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
}
func testDownloadArtifact(t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
httpRequest, err := http.NewRequest("GET", ts.URL+"/url/path", nil)
if err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
DownloadArtifact(apiClient).ServeHTTP(response, httpRequest)
return response
}
func TestDownloadingFromValidArchive(t *testing.T) {
tempFile, err := ioutil.TempFile("", "uploads")
if err != nil {
t.Fatal(err)
}
defer tempFile.Close()
defer os.Remove(tempFile.Name())
archive := zip.NewWriter(tempFile)
defer archive.Close()
fileInArchive, err := archive.Create("test.txt")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(fileInArchive, "testtest")
archive.Close()
ts := testArtifactDownloadServer(t, tempFile.Name(), "test.txt")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 200)
testhelper.AssertResponseHeader(t, response,
"Content-Type",
"text/plain; charset=utf-8")
testhelper.AssertResponseHeader(t, response,
"Content-Disposition",
"attachment; filename=\"test.txt\"")
testhelper.AssertResponseBody(t, response, "testtest")
}
func TestDownloadingNonExistingFile(t *testing.T) {
tempFile, err := ioutil.TempFile("", "uploads")
if err != nil {
t.Fatal(err)
}
defer tempFile.Close()
defer os.Remove(tempFile.Name())
archive := zip.NewWriter(tempFile)
defer archive.Close()
archive.Close()
ts := testArtifactDownloadServer(t, tempFile.Name(), "test")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 404)
}
func TestDownloadingFromInvalidArchive(t *testing.T) {
ts := testArtifactDownloadServer(t, "path/to/non/existing/file", "test")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 404)
}
func TestIncompleteApiResponse(t *testing.T) {
ts := testArtifactDownloadServer(t, "", "")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 500)
}
package artifacts
import (
"../api"
"../helper"
"../upload"
"../zipartifacts"
"errors"
"fmt"
"io/ioutil"
"mime/multipart"
"net/http"
"os"
"os/exec"
"syscall"
)
type artifactsUploadProcessor struct {
TempPath string
metadataFile string
}
func (a *artifactsUploadProcessor) ProcessFile(formName, fileName string, writer *multipart.Writer) error {
// ProcessFile for artifacts requires file form-data field name to eq `file`
if formName != "file" {
return fmt.Errorf("Invalid form field: %q", formName)
}
if a.metadataFile != "" {
return fmt.Errorf("Artifacts request contains more than one file!")
}
// Create temporary file for metadata and store it's path
tempFile, err := ioutil.TempFile(a.TempPath, "metadata_")
if err != nil {
return err
}
defer tempFile.Close()
a.metadataFile = tempFile.Name()
// Generate metadata and save to file
zipMd := exec.Command("gitlab-zip-metadata", fileName)
zipMd.Stderr = os.Stderr
zipMd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
zipMd.Stdout = tempFile
if err := zipMd.Start(); err != nil {
return err
}
defer helper.CleanUpProcessGroup(zipMd)
if err := zipMd.Wait(); err != nil {
if st, ok := helper.ExitStatus(err); ok && st == zipartifacts.StatusNotZip {
return nil
}
return err
}
// Pass metadata file path to Rails
writer.WriteField("metadata.path", a.metadataFile)
writer.WriteField("metadata.name", "metadata.gz")
return nil
}
func (a *artifactsUploadProcessor) ProcessField(formName string, writer *multipart.Writer) error {
return nil
}
func (a *artifactsUploadProcessor) Cleanup() {
if a.metadataFile != "" {
os.Remove(a.metadataFile)
}
}
func UploadArtifacts(myAPI *api.API, h http.Handler) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
if a.TempPath == "" {
helper.Fail500(w, errors.New("UploadArtifacts: TempPath is empty"))
return
}
mg := &artifactsUploadProcessor{TempPath: a.TempPath}
defer mg.Cleanup()
upload.HandleFileUploads(w, r, h, a.TempPath, mg)
}, "/authorize")
}
package artifacts
import (
"../api"
"../helper"
"../proxy"
"../testhelper"
"../zipartifacts"
"archive/zip"
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/url/path/authorize", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Fatal("Expected POST request")
}
w.Header().Set("Content-Type", "application/json")
data, err := json.Marshal(&api.Response{
TempPath: tempPath,
})
if err != nil {
t.Fatal("Expected to marshal")
}
w.Write(data)
})
mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Fatal("Expected POST request")
}
if r.FormValue("file.path") == "" {
w.WriteHeader(501)
return
}
if r.FormValue("metadata.path") == "" {
w.WriteHeader(502)
return
}
_, err := ioutil.ReadFile(r.FormValue("file.path"))
if err != nil {
w.WriteHeader(404)
return
}
metadata, err := ioutil.ReadFile(r.FormValue("metadata.path"))
if err != nil {
w.WriteHeader(404)
return
}
gz, err := gzip.NewReader(bytes.NewReader(metadata))
if err != nil {
w.WriteHeader(405)
return
}
defer gz.Close()
metadata, err = ioutil.ReadAll(gz)
if err != nil {
w.WriteHeader(404)
return
}
if !bytes.HasPrefix(metadata, []byte(zipartifacts.MetadataHeaderPrefix+zipartifacts.MetadataHeader)) {
w.WriteHeader(400)
return
}
w.WriteHeader(200)
})
return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
}
func testUploadArtifacts(contentType string, body io.Reader, t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", body)
if err != nil {
t.Fatal(err)
}
httpRequest.Header.Set("Content-Type", contentType)
response := httptest.NewRecorder()
apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
proxyClient := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
UploadArtifacts(apiClient, proxyClient).ServeHTTP(response, httpRequest)
return response
}
func TestUploadHandlerAddingMetadata(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testArtifactsUploadServer(t, tempPath)
defer ts.Close()
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file", "my.file")
if err != nil {
t.Fatal(err)
}
archive := zip.NewWriter(file)
defer archive.Close()
fileInArchive, err := archive.Create("test.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(fileInArchive, "test")
archive.Close()
writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
testhelper.AssertResponseCode(t, response, 200)
}
func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testArtifactsUploadServer(t, tempPath)
defer ts.Close()
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
// 502 is a custom response code from the mock server in testUploadArtifacts
testhelper.AssertResponseCode(t, response, 502)
}
func TestUploadFormProcessing(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testArtifactsUploadServer(t, tempPath)
defer ts.Close()
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("metadata", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
testhelper.AssertResponseCode(t, response, 500)
}
package artifacts
import "strings"
// taken from mime/multipart/writer.go
var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
func escapeQuotes(s string) string {
return quoteEscaper.Replace(s)
}
......@@ -23,6 +23,7 @@ import (
func GetArchive(a *api.API) http.Handler {
return repoPreAuthorizeHandler(a, handleGetArchive)
}
func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
var format string
urlPath := r.URL.Path
......@@ -78,7 +79,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", archiveCmd.Args, err))
return
}
defer cleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up
defer helper.CleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up
var stdout io.ReadCloser
if compressCmd == nil {
......@@ -98,7 +99,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", compressCmd.Args, err))
return
}
defer cleanUpProcessGroup(compressCmd)
defer helper.CleanUpProcessGroup(compressCmd)
archiveStdout.Close()
}
......@@ -110,7 +111,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
setArchiveHeaders(w, format, archiveFilename)
w.WriteHeader(200) // Don't bother with HTTP 500 from this point on, just return
if _, err := io.Copy(w, archiveReader); err != nil {
helper.LogError(fmt.Errorf("handleGetArchive: read: %v", err))
helper.LogError(fmt.Errorf("handleGetArchive: copy 'git archive' output: %v", err))
return
}
if err := archiveCmd.Wait(); err != nil {
......
......@@ -23,18 +23,3 @@ func gitCommand(gl_id string, name string, args ...string) *exec.Cmd {
cmd.Stderr = os.Stderr
return cmd
}
func cleanUpProcessGroup(cmd *exec.Cmd) {
if cmd == nil {
return
}
process := cmd.Process
if process != nil && process.Pid > 0 {
// Send SIGTERM to the process group of cmd
syscall.Kill(-process.Pid, syscall.SIGTERM)
}
// reap our child process
cmd.Wait()
}
......@@ -72,7 +72,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
helper.Fail500(w, fmt.Errorf("handleGetInfoRefs: start %v: %v", cmd.Args, err))
return
}
defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
// Start writing the response
w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
......@@ -87,7 +87,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
return
}
if _, err := io.Copy(w, stdout); err != nil {
helper.LogError(fmt.Errorf("handleGetInfoRefs: read from %v: %v", cmd.Args, err))
helper.LogError(fmt.Errorf("handleGetInfoRefs: copy output of %v: %v", cmd.Args, err))
return
}
if err := cmd.Wait(); err != nil {
......@@ -125,7 +125,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper.Fail500(w, fmt.Errorf("handlePostRPC: start %v: %v", cmd.Args, err))
return
}
defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
// Write the client request body to Git's standard input
if _, err := io.Copy(stdin, r.Body); err != nil {
......@@ -146,7 +146,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
// This io.Copy may take a long time, both for Git push and pull.
if _, err := io.Copy(w, stdout); err != nil {
helper.LogError(fmt.Errorf("handlePostRPC read from %v: %v", cmd.Args, err))
helper.LogError(fmt.Errorf("handlePostRPC copy output of %v: %v", cmd.Args, err))
return
}
if err := cmd.Wait(); err != nil {
......
......@@ -6,6 +6,8 @@ import (
"net/http"
"net/url"
"os"
"os/exec"
"syscall"
)
func Fail500(w http.ResponseWriter, err error) {
......@@ -79,3 +81,32 @@ func HeaderClone(h http.Header) http.Header {
}
return h2
}
func CleanUpProcessGroup(cmd *exec.Cmd) {
if cmd == nil {
return
}
process := cmd.Process
if process != nil && process.Pid > 0 {
// Send SIGTERM to the process group of cmd
syscall.Kill(-process.Pid, syscall.SIGTERM)
}
// reap our child process
cmd.Wait()
}
func ExitStatus(err error) (int, bool) {
exitError, ok := err.(*exec.ExitError)
if !ok {
return 0, false
}
waitStatus, ok := exitError.Sys().(syscall.WaitStatus)
if !ok {
return 0, false
}
return waitStatus.ExitStatus(), true
}
......@@ -61,7 +61,7 @@ func handleStoreLfsObject(h http.Handler) api.HandleFunc {
written, err := io.Copy(hw, r.Body)
if err != nil {
helper.Fail500(w, fmt.Errorf("handleStoreLfsObject: write tempfile: %v", err))
helper.Fail500(w, fmt.Errorf("handleStoreLfsObject: copy body to tempfile: %v", err))
return
}
file.Close()
......
package upload
import (
"../api"
"net/http"
)
func Artifacts(myAPI *api.API, h http.Handler) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
r.Header.Set(tempPathHeader, a.TempPath)
handleFileUploads(h).ServeHTTP(w, r)
}, "/authorize")
}
......@@ -3,7 +3,6 @@ package upload
import (
"../helper"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
......@@ -12,13 +11,20 @@ import (
"os"
)
const tempPathHeader = "Gitlab-Workhorse-Temp-Path"
type MultipartFormProcessor interface {
ProcessFile(formName, fileName string, writer *multipart.Writer) error
ProcessField(formName string, writer *multipart.Writer) error
}
func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string) (cleanup func(), err error) {
func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string, filter MultipartFormProcessor) (cleanup func(), err error) {
// Create multipart reader
reader, err := r.MultipartReader()
if err != nil {
return nil, err
if err == http.ErrNotMultipart {
// We want to be able to recognize http.ErrNotMultipart elsewhere so no fmt.Errorf
return nil, http.ErrNotMultipart
}
return nil, fmt.Errorf("get multipart reader: %v", err)
}
var files []string
......@@ -51,13 +57,13 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
if filename := p.FileName(); filename != "" {
// Create temporary directory where the uploaded file will be stored
if err := os.MkdirAll(tempPath, 0700); err != nil {
return cleanup, err
return cleanup, fmt.Errorf("mkdir for tempfile: %v", err)
}
// Create temporary file in path returned by Authorization filter
file, err := ioutil.TempFile(tempPath, "upload_")
if err != nil {
return cleanup, err
return cleanup, fmt.Errorf("create tempfile: %v", err)
}
defer file.Close()
......@@ -67,40 +73,46 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
files = append(files, file.Name())
_, err = io.Copy(file, p)
file.Close()
if err != nil {
return cleanup, fmt.Errorf("copy from multipart to tempfile: %v", err)
}
file.Close()
if err := filter.ProcessFile(name, file.Name(), writer); err != nil {
return cleanup, err
}
} else {
np, err := writer.CreatePart(p.Header)
if err != nil {
return cleanup, err
return cleanup, fmt.Errorf("create multipart field: %v", err)
}
_, err = io.Copy(np, p)
if err != nil {
return cleanup, err
return cleanup, fmt.Errorf("duplicate multipart field: %v", err)
}
if err := filter.ProcessField(name, writer); err != nil {
return cleanup, fmt.Errorf("process multipart field: %v", err)
}
}
}
return cleanup, nil
}
func handleFileUploads(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
tempPath := r.Header.Get(tempPathHeader)
func HandleFileUploads(w http.ResponseWriter, r *http.Request, h http.Handler, tempPath string, filter MultipartFormProcessor) {
if tempPath == "" {
helper.Fail500(w, errors.New("handleFileUploads: TempPath empty"))
helper.Fail500(w, fmt.Errorf("handleFileUploads: tempPath empty"))
return
}
r.Header.Del(tempPathHeader)
var body bytes.Buffer
writer := multipart.NewWriter(&body)
defer writer.Close()
// Rewrite multipart form data
cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath)
cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath, filter)
if err != nil {
if err == http.ErrNotMultipart {
h.ServeHTTP(w, r)
......@@ -124,5 +136,4 @@ func handleFileUploads(h http.Handler) http.Handler {
// Proxy the request
h.ServeHTTP(w, r)
})
}
......@@ -5,6 +5,7 @@ import (
"../proxy"
"../testhelper"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
......@@ -19,10 +20,27 @@ import (
var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
type testFormProcessor struct {
}
func (a *testFormProcessor) ProcessFile(formName, fileName string, writer *multipart.Writer) error {
if formName != "file" && fileName != "my.file" {
return errors.New("illegal file")
}
return nil
}
func (a *testFormProcessor) ProcessField(formName string, writer *multipart.Writer) error {
if formName != "token" {
return errors.New("illegal field")
}
return nil
}
func TestUploadTempPathRequirement(t *testing.T) {
response := httptest.NewRecorder()
request := &http.Request{}
handleFileUploads(nilHandler).ServeHTTP(response, request)
HandleFileUploads(response, request, nilHandler, "", nil)
testhelper.AssertResponseCode(t, response, 500)
}
......@@ -56,9 +74,8 @@ func TestUploadHandlerForwardingRawData(t *testing.T) {
response := httptest.NewRecorder()
httpRequest.Header.Set(tempPathHeader, tempPath)
handleFileUploads(proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)).ServeHTTP(response, httpRequest)
handler := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
HandleFileUploads(response, httpRequest, handler, tempPath, nil)
testhelper.AssertResponseCode(t, response, 202)
if response.Body.String() != "RESPONSE" {
t.Fatal("Expected RESPONSE in response body")
......@@ -129,13 +146,65 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
httpRequest.Body = ioutil.NopCloser(&buffer)
httpRequest.ContentLength = int64(buffer.Len())
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
httpRequest.Header.Set(tempPathHeader, tempPath)
response := httptest.NewRecorder()
handleFileUploads(proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)).ServeHTTP(response, httpRequest)
handler := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
HandleFileUploads(response, httpRequest, handler, tempPath, &testFormProcessor{})
testhelper.AssertResponseCode(t, response, 202)
if _, err := os.Stat(filePath); !os.IsNotExist(err) {
t.Fatal("expected the file to be deleted")
}
}
func TestUploadProcessingField(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
writer.WriteField("token2", "test")
writer.Close()
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
if err != nil {
t.Fatal(err)
}
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
HandleFileUploads(response, httpRequest, nilHandler, tempPath, &testFormProcessor{})
testhelper.AssertResponseCode(t, response, 500)
}
func TestUploadProcessingFile(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file2", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
writer.Close()
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
if err != nil {
t.Fatal(err)
}
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
HandleFileUploads(response, httpRequest, nilHandler, tempPath, &testFormProcessor{})
testhelper.AssertResponseCode(t, response, 500)
}
......@@ -2,11 +2,11 @@ package upstream
import (
apipkg "../api"
"../artifacts"
"../git"
"../lfs"
proxypkg "../proxy"
"../staticpages"
"../upload"
"net/http"
"regexp"
)
......@@ -64,8 +64,9 @@ func (u *Upstream) configureRoutes() {
route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)},
route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)},
// CI Artifacts API
route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(upload.Artifacts(api, proxy))},
// CI Artifacts
route{"GET", regexp.MustCompile(projectPattern + `builds/[0-9]+/artifacts/file/`), contentEncodingHandler(artifacts.DownloadArtifact(api))},
route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(artifacts.UploadArtifacts(api, proxy))},
// Explicitly proxy API requests
route{"", regexp.MustCompile(apiPattern), proxy},
......
......@@ -41,8 +41,8 @@ func NewUpstream(backend *url.URL, socket string, version string, documentRoot s
if backend == nil {
up.Backend = DefaultBackend
}
up.configureRoutes()
up.configureURLPrefix()
up.configureRoutes()
return &up
}
......
package zipartifacts
// These are exit codes used by subprocesses in cmd/gitlab-zip-xxx
const (
StatusNotZip = 10 + iota
StatusEntryNotFound
)
package zipartifacts
import (
"encoding/base64"
)
func DecodeFileEntry(entry string) (string, error) {
decoded, err := base64.StdEncoding.DecodeString(entry)
if err != nil {
return "", err
}
return string(decoded), nil
}
package zipartifacts
import (
"archive/zip"
"compress/gzip"
"encoding/binary"
"encoding/json"
"io"
"os"
"strconv"
)
type metadata struct {
Modified int64 `json:"modified"`
Mode string `json:"mode"`
CRC uint32 `json:"crc,omitempty"`
Size uint64 `json:"size,omitempty"`
Zipped uint64 `json:"zipped,omitempty"`
Comment string `json:"comment,omitempty"`
}
const MetadataHeaderPrefix = "\x00\x00\x00&" // length of string below, encoded properly
const MetadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n"
func newMetadata(file *zip.File) metadata {
return metadata{
Modified: file.ModTime().Unix(),
Mode: strconv.FormatUint(uint64(file.Mode().Perm()), 8),
CRC: file.CRC32,
Size: file.UncompressedSize64,
Zipped: file.CompressedSize64,
Comment: file.Comment,
}
}
func (m metadata) writeEncoded(output io.Writer) error {
j, err := json.Marshal(m)
if err != nil {
return err
}
j = append(j, byte('\n'))
return writeBytes(output, j)
}
func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
err := writeString(output, entry.Name)
if err != nil {
return err
}
err = newMetadata(entry).writeEncoded(output)
if err != nil {
return err
}
return nil
}
func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
err := writeString(output, MetadataHeader)
if err != nil {
return err
}
// Write empty error string
err = writeString(output, "{}")
if err != nil {
return err
}
// Write all files
for _, entry := range archive.File {
err = writeZipEntryMetadata(output, entry)
if err != nil {
return err
}
}
return nil
}
func GenerateZipMetadataFromFile(fileName string, w io.Writer) error {
archive, err := zip.OpenReader(fileName)
if err != nil {
// Ignore non-zip archives
return os.ErrInvalid
}
defer archive.Close()
gz := gzip.NewWriter(w)
defer gz.Close()
return generateZipMetadata(gz, &archive.Reader)
}
func writeBytes(output io.Writer, data []byte) error {
err := binary.Write(output, binary.BigEndian, uint32(len(data)))
if err == nil {
_, err = output.Write(data)
}
return err
}
func writeString(output io.Writer, str string) error {
return writeBytes(output, []byte(str))
}
......@@ -6,6 +6,7 @@ import (
"./internal/testhelper"
"./internal/upstream"
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
......@@ -363,6 +364,37 @@ func TestAllowedStaticFile(t *testing.T) {
}
}
func TestStaticFileRelativeURL(t *testing.T) {
content := "PUBLIC"
if err := setupStaticFile("static.txt", content); err != nil {
t.Fatalf("create public/static.txt: %v", err)
}
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), http.HandlerFunc(http.NotFound))
defer ts.Close()
backendURLString := ts.URL + "/my-relative-url"
log.Print(backendURLString)
ws := startWorkhorseServer(backendURLString)
defer ws.Close()
resource := "/my-relative-url/static.txt"
resp, err := http.Get(ws.URL + resource)
if err != nil {
t.Error(err)
}
defer resp.Body.Close()
buf := &bytes.Buffer{}
if _, err := io.Copy(buf, resp.Body); err != nil {
t.Error(err)
}
if buf.String() != content {
t.Errorf("GET %q: Expected %q, got %q", resource, content, buf.String())
}
if resp.StatusCode != 200 {
t.Errorf("GET %q: expected 200, got %d", resource, resp.StatusCode)
}
}
func TestAllowedPublicUploadsFile(t *testing.T) {
content := "PRIVATE but allowed"
if err := setupStaticFile("uploads/static file.txt", content); err != nil {
......@@ -465,8 +497,9 @@ func TestArtifactsUpload(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(r.MultipartForm.Value) != 2 { // 1 file name, 1 file path
t.Error("Expected to receive exactly 2 values")
nValues := 2 // filename + path for just the upload (no metadata because we are not POSTing a valid zip file)
if len(r.MultipartForm.Value) != nValues {
t.Errorf("Expected to receive exactly %d values", nValues)
}
if len(r.MultipartForm.File) != 0 {
t.Error("Expected to not receive any files")
......@@ -488,6 +521,40 @@ func TestArtifactsUpload(t *testing.T) {
}
}
func TestArtifactsGetSingleFile(t *testing.T) {
// We manually created this zip file in the gitlab-workhorse Git repository
archivePath := `testdata/artifacts-archive.zip`
fileName := "myfile"
fileContents := "MY FILE"
resourcePath := `/namespace/project/builds/123/artifacts/file/` + fileName
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`\A`+resourcePath+`\z`), func(w http.ResponseWriter, r *http.Request) {
encodedFilename := base64.StdEncoding.EncodeToString([]byte(fileName))
if _, err := fmt.Fprintf(w, `{"Archive":"%s","Entry":"%s"}`, archivePath, encodedFilename); err != nil {
t.Fatal(err)
}
return
})
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
resp, err := http.Get(ws.URL + resourcePath)
if err != nil {
t.Error(err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Errorf("GET %q: expected 200, got %d", resourcePath, resp.StatusCode)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if string(body) != fileContents {
t.Fatalf("Expected file contents %q, got %q", fileContents, body)
}
}
func setupStaticFile(fpath, content string) error {
cwd, err := os.Getwd()
if err != nil {
......
#!/bin/sh
exec env PATH=$(pwd):${PATH} "$@"
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment