rewrite.go 5.22 KB
Newer Older
1 2 3
package upload

import (
4
	"context"
5
	"errors"
6 7
	"fmt"
	"io"
8
	"io/ioutil"
9 10 11 12 13
	"mime/multipart"
	"net/http"
	"strings"

	"github.com/prometheus/client_golang/prometheus"
14
	"gitlab.com/gitlab-org/labkit/log"
15

16
	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
17
	"gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore"
18
	"gitlab.com/gitlab-org/gitlab-workhorse/internal/lsif_transformer/parser"
19
	"gitlab.com/gitlab-org/gitlab-workhorse/internal/upload/exif"
20 21
)

22 23 24
// ErrInjectedClientParam means that the client sent a parameter that overrides one of our own fields
var ErrInjectedClientParam = errors.New("injected client parameter")

25
var (
26
	multipartUploadRequests = prometheus.NewCounterVec(
27 28 29
		prometheus.CounterOpts{

			Name: "gitlab_workhorse_multipart_upload_requests",
30
			Help: "How many multipart upload requests have been processed by gitlab-workhorse. Partitioned by type.",
31
		},
32
		[]string{"type"},
33 34
	)

35
	multipartFileUploadBytes = prometheus.NewCounterVec(
36 37
		prometheus.CounterOpts{
			Name: "gitlab_workhorse_multipart_upload_bytes",
38
			Help: "How many disk bytes of multipart file parts have been successfully written by gitlab-workhorse. Partitioned by type.",
39
		},
40
		[]string{"type"},
41 42
	)

43
	multipartFiles = prometheus.NewCounterVec(
44 45
		prometheus.CounterOpts{
			Name: "gitlab_workhorse_multipart_upload_files",
46
			Help: "How many multipart file parts have been processed by gitlab-workhorse. Partitioned by type.",
47
		},
48
		[]string{"type"},
49 50 51
	)
)

52
type rewriter struct {
53 54 55 56
	writer          *multipart.Writer
	preauth         *api.Response
	filter          MultipartFormProcessor
	finalizedFields map[string]bool
57 58
}

59 60 61 62 63 64
func init() {
	prometheus.MustRegister(multipartUploadRequests)
	prometheus.MustRegister(multipartFileUploadBytes)
	prometheus.MustRegister(multipartFiles)
}

65
func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, preauth *api.Response, filter MultipartFormProcessor, opts *filestore.SaveFileOpts) error {
66 67 68 69 70
	// Create multipart reader
	reader, err := r.MultipartReader()
	if err != nil {
		if err == http.ErrNotMultipart {
			// We want to be able to recognize http.ErrNotMultipart elsewhere so no fmt.Errorf
71
			return http.ErrNotMultipart
72
		}
73
		return fmt.Errorf("get multipart reader: %v", err)
74 75
	}

76
	multipartUploadRequests.WithLabelValues(filter.Name()).Inc()
77

78
	rew := &rewriter{
79 80 81 82
		writer:          writer,
		preauth:         preauth,
		filter:          filter,
		finalizedFields: make(map[string]bool),
83
	}
84 85 86

	for {
		p, err := reader.NextPart()
87 88 89 90
		if err != nil {
			if err == io.EOF {
				break
			}
91
			return err
92 93 94 95 96 97 98
		}

		name := p.FormName()
		if name == "" {
			continue
		}

99 100 101 102
		if rew.finalizedFields[name] {
			return ErrInjectedClientParam
		}

103
		if p.FileName() != "" {
104
			err = rew.handleFilePart(r.Context(), name, p, opts)
105
		} else {
106
			err = rew.copyPart(r.Context(), name, p)
107 108 109
		}

		if err != nil {
110
			return err
111 112
		}
	}
113

114
	return nil
115
}
116

117
func (rew *rewriter) handleFilePart(ctx context.Context, name string, p *multipart.Part, opts *filestore.SaveFileOpts) error {
118 119
	multipartFiles.WithLabelValues(rew.filter.Name()).Inc()

120 121 122 123 124 125
	filename := p.FileName()

	if strings.Contains(filename, "/") || filename == "." || filename == ".." {
		return fmt.Errorf("illegal filename: %q", filename)
	}

126
	opts.TempFilePrefix = filename
127

128
	var inputReader io.ReadCloser
129 130 131 132
	var err error
	switch {
	case exif.IsExifFile(filename):
		inputReader, err = handleExifUpload(ctx, p, filename)
133
		if err != nil {
134
			return err
135
		}
136
	case rew.preauth.ProcessLsif:
137
		inputReader, err = handleLsifUpload(ctx, p, opts.LocalTempPath, filename, rew.preauth)
138 139 140 141
		if err != nil {
			return err
		}
	default:
142
		inputReader = ioutil.NopCloser(p)
143 144
	}

145 146
	defer inputReader.Close()

147
	fh, err := filestore.SaveFileFromReader(ctx, inputReader, -1, opts)
148
	if err != nil {
149 150
		switch err {
		case filestore.ErrEntityTooLarge, exif.ErrRemovingExif:
Alessio Caiazza's avatar
Alessio Caiazza committed
151
			return err
152 153
		default:
			return fmt.Errorf("persisting multipart file: %v", err)
Alessio Caiazza's avatar
Alessio Caiazza committed
154
		}
155 156
	}

157 158 159 160 161 162
	fields, err := fh.GitLabFinalizeFields(name)
	if err != nil {
		return fmt.Errorf("failed to finalize fields: %v", err)
	}

	for key, value := range fields {
163
		rew.writer.WriteField(key, value)
164
		rew.finalizedFields[key] = true
165 166
	}

167
	multipartFileUploadBytes.WithLabelValues(rew.filter.Name()).Add(float64(fh.Size))
168

169
	return rew.filter.ProcessFile(ctx, name, fh, rew.writer)
170 171
}

172
func handleExifUpload(ctx context.Context, r io.Reader, filename string) (io.ReadCloser, error) {
173 174 175
	log.WithContextFields(ctx, log.Fields{
		"filename": filename,
	}).Print("running exiftool to remove any metadata")
176

Alessio Caiazza's avatar
Alessio Caiazza committed
177
	cleaner, err := exif.NewCleaner(ctx, r)
178 179 180 181
	if err != nil {
		return nil, err
	}

Alessio Caiazza's avatar
Alessio Caiazza committed
182
	return cleaner, nil
183
}
184

185 186 187
func handleLsifUpload(ctx context.Context, reader io.Reader, tempPath, filename string, preauth *api.Response) (io.ReadCloser, error) {
	parserConfig := parser.Config{
		TempPath: tempPath,
188 189
	}

190
	return parser.NewParser(ctx, reader, parserConfig)
191 192
}

193
func (rew *rewriter) copyPart(ctx context.Context, name string, p *multipart.Part) error {
194 195 196 197 198 199 200 201 202
	np, err := rew.writer.CreatePart(p.Header)
	if err != nil {
		return fmt.Errorf("create multipart field: %v", err)
	}

	if _, err := io.Copy(np, p); err != nil {
		return fmt.Errorf("duplicate multipart field: %v", err)
	}

203
	if err := rew.filter.ProcessField(ctx, name, rew.writer); err != nil {
204 205 206 207 208
		return fmt.Errorf("process multipart field: %v", err)
	}

	return nil
}