Commit ceedac04 authored by Kamil Trzcinski's avatar Kamil Trzcinski

Introduce naive (using the same process) artifacts metadata generator and downloader

parent 9e7b612b
...@@ -58,6 +58,10 @@ type Response struct { ...@@ -58,6 +58,10 @@ type Response struct {
// TmpPath is the path where we should store temporary files // TmpPath is the path where we should store temporary files
// This is set by authorization middleware // This is set by authorization middleware
TempPath string TempPath string
// Archive is the path where the artifacts archive is stored
Archive string `json:"archive"`
// Path is the filename inside the archive to extracted file
Path string `json:"path"`
} }
// singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy // singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy
......
package artifacts
import (
"../api"
"../upload"
"net/http"
"../helper"
"errors"
"os"
"archive/zip"
"encoding/base64"
"strconv"
"mime"
"path/filepath"
"io"
)
func UploadArtifacts(myAPI *api.API, h http.Handler) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
if a.TempPath == "" {
helper.Fail500(w, errors.New("UploadArtifacts: TempPath is empty"))
return
}
upload.HandleFileUploads(w, r, h, a.TempPath, &artifactsFormFilter{})
}, "/authorize")
}
// Artifacts downloader doesn't support ranges when downloading a single file
func DownloadArtifact(myAPI *api.API) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
if a.Archive == "" || a.Path == "" {
helper.Fail500(w, errors.New("DownloadArtifact: Archive or Path is empty"))
return
}
fileNameDecoded, err := base64.StdEncoding.DecodeString(a.Path)
if err != nil {
helper.Fail500(w, err)
return
}
fileName := string(fileNameDecoded)
// TODO:
// This should be moved to sub process to reduce memory pressue on workhorse
archive, err := zip.OpenReader(a.Archive)
if os.IsNotExist(err) {
http.NotFound(w, r)
return
} else if err != nil {
helper.Fail500(w, err)
}
defer archive.Close()
var file *zip.File
for _, file = range archive.File {
if file.Name == fileName {
break
}
}
if file == nil {
http.NotFound(w, r)
return
}
contentType := mime.TypeByExtension(filepath.Ext(file.Name))
if contentType == "" {
contentType = "application/octet-stream"
}
w.Header().Set("Content-Length", strconv.FormatInt(int64(file.UncompressedSize64), 10))
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Disposition", "attachment; filename=" + filepath.Base(file.Name))
reader, err := file.Open()
if err != nil {
helper.Fail500(w, err)
}
defer reader.Close()
// Copy file body
io.Copy(w, reader)
}, "")
}
package artifacts
import (
"archive/zip"
"io"
"encoding/binary"
"encoding/json"
)
type metadata struct {
Modified uint16 `json:"modified"`
CRC uint32 `json:"crc,omitempty"`
Size uint64 `json:"size,omitempty"`
Zipped uint64 `json:"zipped,omitempty"`
Comment string `json:"comment,omitempty"`
}
func newMetadata(file *zip.File) metadata {
return metadata{
Modified: file.ModifiedDate,
CRC: file.CRC32,
Size: file.CompressedSize64,
Zipped: file.UncompressedSize64,
Comment: file.Comment,
}
}
func (m metadata) write(output io.Writer) error {
j, err := json.Marshal(m)
if err != nil {
return err
}
j = append(j, byte('\n'))
return writeBytes(output, j)
}
func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
err := writeString(output, "GitLab Build Artifacts Metadata 0.0.1\n")
if err != nil {
return err
}
err = writeString(output, "{}")
if err != nil {
return err
}
for _, entry := range archive.File {
err = writeString(output, entry.Name)
if err != nil {
return err
}
err = newMetadata(entry).write(output)
if err != nil {
return err
}
}
return nil
}
func writeBytes(output io.Writer, data []byte) error {
err := binary.Write(output, binary.BigEndian, uint32(len(data)))
if err == nil {
_, err = output.Write(data)
}
return err
}
func writeString(output io.Writer, str string) error {
return writeBytes(output, []byte(str))
}
package artifacts
import (
"mime/multipart"
"fmt"
"archive/zip"
"compress/gzip"
)
type artifactsFormFilter struct {
}
func (a *artifactsFormFilter) FilterFile(formName, fileName string, writer *multipart.Writer) error {
if formName != "file" {
return fmt.Errorf("Invalid form field: %q", formName)
}
archive, err := zip.OpenReader(fileName)
if err != nil {
// Ignore non-zip archives
return nil
}
defer archive.Close()
// TODO:
// we could create a temporary file and save to this file instead of writing to mulipart.Writer
// doing it like this is simpler, but puts more pressure on memory
metadataFile, err := writer.CreateFormFile("metadata", "metadata.gz")
if err != nil {
return err
}
defer writer.Close()
gz := gzip.NewWriter(metadataFile)
defer gz.Close()
err = generateZipMetadata(gz, &archive.Reader)
if err != nil {
return err
}
return nil
}
func (a *artifactsFormFilter) FilterField(formName string, writer *multipart.Writer) error {
return nil
}
...@@ -23,6 +23,7 @@ import ( ...@@ -23,6 +23,7 @@ import (
func GetArchive(a *api.API) http.Handler { func GetArchive(a *api.API) http.Handler {
return repoPreAuthorizeHandler(a, handleGetArchive) return repoPreAuthorizeHandler(a, handleGetArchive)
} }
func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) { func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
var format string var format string
urlPath := r.URL.Path urlPath := r.URL.Path
......
...@@ -3,7 +3,6 @@ package upload ...@@ -3,7 +3,6 @@ package upload
import ( import (
"../helper" "../helper"
"bytes" "bytes"
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
...@@ -14,7 +13,12 @@ import ( ...@@ -14,7 +13,12 @@ import (
const tempPathHeader = "Gitlab-Workhorse-Temp-Path" const tempPathHeader = "Gitlab-Workhorse-Temp-Path"
func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string) (cleanup func(), err error) { type MultipartFormFilter interface {
FilterFile(formName, fileName string, writer *multipart.Writer) error
FilterField(formName string, writer *multipart.Writer) error
}
func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string, filter MultipartFormFilter) (cleanup func(), err error) {
// Create multipart reader // Create multipart reader
reader, err := r.MultipartReader() reader, err := r.MultipartReader()
if err != nil { if err != nil {
...@@ -67,10 +71,18 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te ...@@ -67,10 +71,18 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
files = append(files, file.Name()) files = append(files, file.Name())
_, err = io.Copy(file, p) _, err = io.Copy(file, p)
if err != nil {
return cleanup, err
}
file.Close() file.Close()
if filter != nil {
err = filter.FilterFile(name, filename, writer)
if err != nil { if err != nil {
return cleanup, err return cleanup, err
} }
}
} else { } else {
np, err := writer.CreatePart(p.Header) np, err := writer.CreatePart(p.Header)
if err != nil { if err != nil {
...@@ -81,26 +93,25 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te ...@@ -81,26 +93,25 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
if err != nil { if err != nil {
return cleanup, err return cleanup, err
} }
if filter != nil {
err = filter.FilterField(name, writer)
if err != nil {
return cleanup, err
}
}
} }
} }
return cleanup, nil return cleanup, nil
} }
func handleFileUploads(h http.Handler) http.Handler { func HandleFileUploads(w http.ResponseWriter, r *http.Request, h http.Handler, tempPath string, filter MultipartFormFilter) {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
tempPath := r.Header.Get(tempPathHeader)
if tempPath == "" {
helper.Fail500(w, errors.New("handleFileUploads: TempPath empty"))
return
}
r.Header.Del(tempPathHeader)
var body bytes.Buffer var body bytes.Buffer
writer := multipart.NewWriter(&body) writer := multipart.NewWriter(&body)
defer writer.Close() defer writer.Close()
// Rewrite multipart form data // Rewrite multipart form data
cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath) cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath, filter)
if err != nil { if err != nil {
if err == http.ErrNotMultipart { if err == http.ErrNotMultipart {
h.ServeHTTP(w, r) h.ServeHTTP(w, r)
...@@ -124,5 +135,4 @@ func handleFileUploads(h http.Handler) http.Handler { ...@@ -124,5 +135,4 @@ func handleFileUploads(h http.Handler) http.Handler {
// Proxy the request // Proxy the request
h.ServeHTTP(w, r) h.ServeHTTP(w, r)
})
} }
...@@ -2,11 +2,11 @@ package upstream ...@@ -2,11 +2,11 @@ package upstream
import ( import (
apipkg "../api" apipkg "../api"
"../artifacts"
"../git" "../git"
"../lfs" "../lfs"
proxypkg "../proxy" proxypkg "../proxy"
"../staticpages" "../staticpages"
"../upload"
"net/http" "net/http"
"regexp" "regexp"
) )
...@@ -23,7 +23,7 @@ const gitProjectPattern = `^/[^/]+/[^/]+\.git/` ...@@ -23,7 +23,7 @@ const gitProjectPattern = `^/[^/]+/[^/]+\.git/`
const apiPattern = `^/api/` const apiPattern = `^/api/`
// A project ID in an API request is either a number or two strings 'namespace/project' // A project ID in an API request is either a number or two strings 'namespace/project'
const projectsAPIPattern = `^/api/v3/projects/((\d+)|([^/]+/[^/]+))/` const projectsAPIPattern = `^/api/v3/projects/(\d+)|([^/]+/[^/]+)/`
const ciAPIPattern = `^/ci/api/` const ciAPIPattern = `^/ci/api/`
// Routing table // Routing table
...@@ -64,8 +64,9 @@ func (u *Upstream) configureRoutes() { ...@@ -64,8 +64,9 @@ func (u *Upstream) configureRoutes() {
route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)}, route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)},
route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)}, route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)},
// CI Artifacts API // CI Artifacts
route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(upload.Artifacts(api, proxy))}, route{"GET", regexp.MustCompile(projectPattern + `builds/[0-9]+/file/`), contentEncodingHandler(artifacts.DownloadArtifact(api))},
route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(artifacts.UploadArtifacts(api, proxy))},
// Explicitly proxy API requests // Explicitly proxy API requests
route{"", regexp.MustCompile(apiPattern), proxy}, route{"", regexp.MustCompile(apiPattern), proxy},
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment