Commit 29e255d6 authored by Jacob Vosmaer's avatar Jacob Vosmaer

Merge branch 'ac-ignore-multipart-etag' into 'master'

Ignore CompleteMultipartUpload ETag

See merge request gitlab-org/gitlab-workhorse!438
parents 74718661 af421a45
......@@ -156,16 +156,7 @@ func (m *Multipart) complete(cmu *CompleteMultipartUpload) error {
m.extractETag(result.ETag)
return m.verifyETag(cmu)
}
func (m *Multipart) verifyETag(cmu *CompleteMultipartUpload) error {
expectedChecksum, err := cmu.BuildMultipartUploadETag()
if err != nil {
return err
}
return compareMD5(expectedChecksum, m.etag)
return nil
}
func (m *Multipart) readAndUploadOnePart(partURL string, putHeaders map[string]string, src io.Reader, partNumber int) (*completeMultipartUploadPart, error) {
......
......@@ -2,7 +2,6 @@ package objectstore_test
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
......@@ -36,12 +35,10 @@ func TestMultipartUploadWithUpcaseETags(t *testing.T) {
// POST with CompleteMultipartUpload request
if r.Method == "POST" {
expectedETag := "6e6b164c392b04bfbb82368179d9ade2-1"
completeBody := fmt.Sprintf(`<CompleteMultipartUploadResult>
completeBody := `<CompleteMultipartUploadResult>
<Bucket>test-bucket</Bucket>
<ETag>%s</ETag>
</CompleteMultipartUploadResult>`,
strings.ToUpper(expectedETag))
<ETag>No Longer Checked</ETag>
</CompleteMultipartUploadResult>`
postCnt++
w.Write([]byte(completeBody))
......
package objectstore
import (
"crypto/md5"
"encoding/hex"
"encoding/xml"
"fmt"
)
......@@ -51,25 +49,3 @@ type compoundCompleteMultipartUploadResult struct {
func (c *compoundCompleteMultipartUploadResult) isError() bool {
return c.CompleteMultipartUploadError != nil
}
// BuildMultipartUploadETag creates an S3 compatible ETag for MultipartUpload
// Given the MD5 hash for each uploaded part of the file, concatenate
// the hashes into a single binary string and calculate the MD5 hash of that result,
// the append "-len(etags)"
// http://permalink.gmane.org/gmane.comp.file-systems.s3.s3tools/583
func (cmu *CompleteMultipartUpload) BuildMultipartUploadETag() (string, error) {
hasher := md5.New()
for _, part := range cmu.Part {
checksum, err := hex.DecodeString(part.ETag)
if err != nil {
return "", err
}
_, err = hasher.Write(checksum)
if err != nil {
return "", err
}
}
multipartChecksum := hasher.Sum(nil)
return fmt.Sprintf("%s-%d", hex.EncodeToString(multipartChecksum), len(cmu.Part)), nil
}
package objectstore
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestMultipartUploadETag(t *testing.T) {
cmu := CompleteMultipartUpload{
Part: []*completeMultipartUploadPart{
{PartNumber: 1, ETag: "550cf6b6e60f65a0e3104a26e70fea42"},
{PartNumber: 2, ETag: "920b914bca0a70780b40881b8f376135"},
{PartNumber: 3, ETag: "175719e13d23c021058bc7376696f26f"},
},
}
expectedETag := "1dc6ab8f946f699770f14f46a8739671-3"
etag, err := cmu.BuildMultipartUploadETag()
require.NoError(t, err)
require.Equal(t, expectedETag, etag)
}
......@@ -220,11 +220,7 @@ func (o *ObjectstoreStub) completeMultipartUpload(w http.ResponseWriter, r *http
etag, overwritten := o.overwriteMD5[objectPath]
if !overwritten {
etag, err = msg.BuildMultipartUploadETag()
if err != nil {
http.Error(w, "cannot compute ETag", 400)
return
}
etag = "CompleteMultipartUploadETag"
}
o.bucket[objectPath] = etag
......
......@@ -106,7 +106,6 @@ func TestObjectStoreCompleteMultipartUpload(t *testing.T) {
contentMD5: "920b914bca0a70780b40881b8f376135",
},
}
expectedETag := "2f2f82eceacf5bd0ac5d7c3d3d388849-2"
stub.InitiateMultipartUpload(ObjectPath)
......@@ -149,7 +148,6 @@ func TestObjectStoreCompleteMultipartUpload(t *testing.T) {
assert.Equal(len(parts), stub.PutsCnt())
assert.Equal(0, stub.DeletesCnt())
assert.Equal(expectedETag, stub.GetObjectMD5(ObjectPath))
assert.False(stub.IsMultipartUpload(ObjectPath), "MultipartUpload is still in progress")
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment