Merge branch 'lfs_support' into 'master'

LFS storage api support

Part of gitlab-org/gitlab-ce#2955

See merge request !3

Makefile

 PREFIX=/usr/local
 VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
 
-gitlab-workhorse: main.go upstream.go archive.go git-http.go helpers.go xsendfile.go authorization.go
+gitlab-workhorse: main.go upstream.go archive.go git-http.go helpers.go xsendfile.go authorization.go lfs.go
 	go build -ldflags "-X main.Version ${VERSION}" -o gitlab-workhorse
 
 install: gitlab-workhorse

lfs.go

+/*
+In this file we handle git lfs objects downloads and uploads
+*/
+
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+var (
+	errHashMismatch = errors.New("Content hash does not match OID")
+	errSizeMismatch = errors.New("Content size does not match")
+)
+
+func lfsAuthorizeHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
+	return preAuthorizeHandler(func(w http.ResponseWriter, r *gitRequest) {
+
+		if r.StoreLFSPath == "" {
+			fail500(w, "lfsAuthorizeHandler", errors.New("Don't know where to store object, no store path specified."))
+			return
+		}
+
+		if r.LfsOid == "" {
+			fail500(w, "lfsAuthorizeHandler", errors.New("Lfs object oid not specified."))
+			return
+		}
+
+		if r.LfsSize == 0 {
+			fail500(w, "lfsAuthorizeHandler", errors.New("Lfs object size not specified."))
+			return
+		}
+
+		if err := os.MkdirAll(r.StoreLFSPath, 0700); err != nil {
+			fail500(w, "Couldn't create directory for storing LFS tmp objects.", err)
+			return
+		}
+
+		handleFunc(w, r)
+	}, "/authorize")
+}
+
+func handleStoreLfsObject(w http.ResponseWriter, r *gitRequest) {
+	var body io.ReadCloser
+
+	body = r.Body
+	defer body.Close()
+
+	file, err := ioutil.TempFile(r.StoreLFSPath, r.LfsOid)
+	if err != nil {
+		fail500(w, "Couldn't open tmp file for writing.", err)
+		return
+	}
+	defer os.Remove(file.Name())
+	defer file.Close()
+
+	hash := sha256.New()
+	hw := io.MultiWriter(hash, file)
+
+	written, err := io.Copy(hw, body)
+	if err != nil {
+		fail500(w, "Failed to save received LFS object.", err)
+		return
+	}
+	file.Close()
+
+	if written != r.LfsSize {
+		fail500(w, "Inconsistent size: ", errSizeMismatch)
+		return
+	}
+
+	shaStr := hex.EncodeToString(hash.Sum(nil))
+	if shaStr != r.LfsOid {
+		fail500(w, "Inconsistent size: ", errSizeMismatch)
+		return
+	}
+	r.Header.Set("X-GitLab-Lfs-Tmp", filepath.Base(file.Name()))
+
+	authReq, err := r.u.newUpstreamRequest(r.Request, nil, "")
+	if err != nil {
+		fail500(w, "newUpstreamRequestlfsCallback", err)
+		return
+	}
+
+	authResponse, err := r.u.httpClient.Do(authReq)
+	if err != nil {
+		fail500(w, "doRequestlfsCallback", err)
+		return
+	}
+	defer authResponse.Body.Close()
+
+	return
+}

main_test.go

@@ -241,69 +241,18 @@ func TestDownloadCacheCreate(t *testing.T) {
 
 func TestAllowedXSendfileDownload(t *testing.T) {
 	contentFilename := "my-content"
-	contentPath := path.Join(cacheDir, contentFilename)
+	url := fmt.Sprintf("http://%s/foo/uploads/bar", servAddr)
 	prepareDownloadDir(t)
 
-	// Prepare test server and backend
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if xSendfileType := r.Header.Get("X-Sendfile-Type"); xSendfileType != "X-Sendfile" {
-			t.Fatalf(`X-Sendfile-Type want "X-Sendfile" got %q`, xSendfileType)
-		}
-		w.Header().Set("X-Sendfile", contentPath)
-		w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, contentFilename))
-		w.WriteHeader(200)
-	}))
-	defer ts.Close()
-	defer cleanUpProcessGroup(startServerOrFail(t, ts))
-
-	if err := os.MkdirAll(cacheDir, 0755); err != nil {
-		t.Fatal(err)
-	}
-	contentBytes := []byte{'c', 'o', 'n', 't', 'e', 'n', 't'}
-	if err := ioutil.WriteFile(contentPath, contentBytes, 0644); err != nil {
-		t.Fatal(err)
-	}
-
-	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/foo/uploads/bar", servAddr))
-	downloadCmd.Dir = scratchDir
-	runOrFail(t, downloadCmd)
-
-	actual, err := ioutil.ReadFile(path.Join(scratchDir, contentFilename))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if bytes.Compare(actual, contentBytes) != 0 {
-		t.Fatal("Unexpected file contents in download")
-	}
+	allowedXSendfileDownload(t, contentFilename, url)
 }
 
 func TestDeniedXSendfileDownload(t *testing.T) {
 	contentFilename := "my-content"
+	url := fmt.Sprintf("http://%s/foo/uploads/bar", servAddr)
 	prepareDownloadDir(t)
 
-	// Prepare test server and backend
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if xSendfileType := r.Header.Get("X-Sendfile-Type"); xSendfileType != "X-Sendfile" {
-			t.Fatalf(`X-Sendfile-Type want "X-Sendfile" got %q`, xSendfileType)
-		}
-		w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, contentFilename))
-		w.WriteHeader(200)
-		fmt.Fprint(w, "Denied")
-	}))
-	defer ts.Close()
-	defer cleanUpProcessGroup(startServerOrFail(t, ts))
-
-	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/foo/uploads/bar", servAddr))
-	downloadCmd.Dir = scratchDir
-	runOrFail(t, downloadCmd)
-
-	actual, err := ioutil.ReadFile(path.Join(scratchDir, contentFilename))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if bytes.Compare(actual, []byte{'D', 'e', 'n', 'i', 'e', 'd'}) != 0 {
-		t.Fatal("Unexpected file contents in download")
-	}
+	deniedXSendfileDownload(t, contentFilename, url)
 }
 
 func prepareDownloadDir(t *testing.T) {
@@ -337,7 +286,7 @@ func testAuthServer(code int, body string) *httptest.Server {
 
 func startServerOrFail(t *testing.T, ts *httptest.Server) *exec.Cmd {
 	cmd := exec.Command("go", "run", "main.go", "upstream.go", "archive.go", "git-http.go", "helpers.go", "xsendfile.go",
-		"authorization.go", fmt.Sprintf("-authBackend=%s", ts.URL), fmt.Sprintf("-listenAddr=%s", servAddr))
+		"authorization.go", "lfs.go", fmt.Sprintf("-authBackend=%s", ts.URL), fmt.Sprintf("-listenAddr=%s", servAddr))
 	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -401,3 +350,85 @@ func repoPath(t *testing.T) string {
 	}
 	return path.Join(cwd, testRepoRoot, testRepo)
 }
+
+func TestDeniedLfsDownload(t *testing.T) {
+	contentFilename := "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80"
+	url := fmt.Sprintf("http://%s/gitlab-lfs/objects/%s", servAddr, contentFilename)
+
+	prepareDownloadDir(t)
+	deniedXSendfileDownload(t, contentFilename, url)
+}
+
+func TestAllowedLfsDownload(t *testing.T) {
+	contentFilename := "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80"
+	url := fmt.Sprintf("http://%s/gitlab-lfs/objects/%s", servAddr, contentFilename)
+
+	prepareDownloadDir(t)
+	allowedXSendfileDownload(t, contentFilename, url)
+}
+
+func allowedXSendfileDownload(t *testing.T, contentFilename string, url string) {
+	contentPath := path.Join(cacheDir, contentFilename)
+	prepareDownloadDir(t)
+
+	// Prepare test server and backend
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if xSendfileType := r.Header.Get("X-Sendfile-Type"); xSendfileType != "X-Sendfile" {
+			t.Fatalf(`X-Sendfile-Type want "X-Sendfile" got %q`, xSendfileType)
+		}
+		w.Header().Set("X-Sendfile", contentPath)
+		w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, contentFilename))
+		w.Header().Set("Content-Type", fmt.Sprintf(`application/octet-stream`))
+		w.WriteHeader(200)
+	}))
+	defer ts.Close()
+	defer cleanUpProcessGroup(startServerOrFail(t, ts))
+
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+	contentBytes := []byte{'c', 'o', 'n', 't', 'e', 'n', 't'}
+	if err := ioutil.WriteFile(contentPath, contentBytes, 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	downloadCmd := exec.Command("curl", "-J", "-O", url)
+	downloadCmd.Dir = scratchDir
+	runOrFail(t, downloadCmd)
+
+	actual, err := ioutil.ReadFile(path.Join(scratchDir, contentFilename))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Compare(actual, contentBytes) != 0 {
+		t.Fatal("Unexpected file contents in download")
+	}
+}
+
+func deniedXSendfileDownload(t *testing.T, contentFilename string, url string) {
+	prepareDownloadDir(t)
+
+	// Prepare test server and backend
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if xSendfileType := r.Header.Get("X-Sendfile-Type"); xSendfileType != "X-Sendfile" {
+			t.Fatalf(`X-Sendfile-Type want "X-Sendfile" got %q`, xSendfileType)
+		}
+		w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, contentFilename))
+		w.WriteHeader(200)
+		fmt.Fprint(w, "Denied")
+	}))
+	defer ts.Close()
+	defer cleanUpProcessGroup(startServerOrFail(t, ts))
+
+	downloadCmd := exec.Command("curl", "-J", "-O", url)
+	downloadCmd.Dir = scratchDir
+	runOrFail(t, downloadCmd)
+
+	actual, err := ioutil.ReadFile(path.Join(scratchDir, contentFilename))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Compare(actual, []byte{'D', 'e', 'n', 'i', 'e', 'd'}) != 0 {
+		t.Fatal("Unexpected file contents in download")
+	}
+}

upstream.go

@@ -44,6 +44,13 @@ type authorizationResponse struct {
 	// CommitId is used do prevent race conditions between the 'time of check'
 	// in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
 	CommitId string
+	// StoreLFSPath is provided by the GitLab Rails application
+	// to mark where the tmp file should be placed
+	StoreLFSPath string
+	// LFS object id
+	LfsOid string
+	// LFS object size
+	LfsSize int64
 }
 
 // A gitReqest is an *http.Request decorated with attributes returned by the
@@ -65,6 +72,8 @@ var gitServices = [...]gitService{
 	gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile},
+	gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfsAuthorizeHandler(handleStoreLfsObject)},
+	gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile},
 }
 
 func newUpstream(authBackend string, authTransport http.RoundTripper) *upstream {