Commit af421a45 authored by Alessio Caiazza's avatar Alessio Caiazza Committed by Jacob Vosmaer

Ignore CompleteMultipart ETag

Our ETag comparison on CompleteMultipartUpload was based on a reverse
engineered protocol.

It is not part of the S3 API specification and not every providers
compute it in the same way.
parent 74718661
...@@ -156,16 +156,7 @@ func (m *Multipart) complete(cmu *CompleteMultipartUpload) error { ...@@ -156,16 +156,7 @@ func (m *Multipart) complete(cmu *CompleteMultipartUpload) error {
m.extractETag(result.ETag) m.extractETag(result.ETag)
return m.verifyETag(cmu) return nil
}
func (m *Multipart) verifyETag(cmu *CompleteMultipartUpload) error {
expectedChecksum, err := cmu.BuildMultipartUploadETag()
if err != nil {
return err
}
return compareMD5(expectedChecksum, m.etag)
} }
func (m *Multipart) readAndUploadOnePart(partURL string, putHeaders map[string]string, src io.Reader, partNumber int) (*completeMultipartUploadPart, error) { func (m *Multipart) readAndUploadOnePart(partURL string, putHeaders map[string]string, src io.Reader, partNumber int) (*completeMultipartUploadPart, error) {
......
...@@ -2,7 +2,6 @@ package objectstore_test ...@@ -2,7 +2,6 @@ package objectstore_test
import ( import (
"context" "context"
"fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
...@@ -36,12 +35,10 @@ func TestMultipartUploadWithUpcaseETags(t *testing.T) { ...@@ -36,12 +35,10 @@ func TestMultipartUploadWithUpcaseETags(t *testing.T) {
// POST with CompleteMultipartUpload request // POST with CompleteMultipartUpload request
if r.Method == "POST" { if r.Method == "POST" {
expectedETag := "6e6b164c392b04bfbb82368179d9ade2-1" completeBody := `<CompleteMultipartUploadResult>
completeBody := fmt.Sprintf(`<CompleteMultipartUploadResult>
<Bucket>test-bucket</Bucket> <Bucket>test-bucket</Bucket>
<ETag>%s</ETag> <ETag>No Longer Checked</ETag>
</CompleteMultipartUploadResult>`, </CompleteMultipartUploadResult>`
strings.ToUpper(expectedETag))
postCnt++ postCnt++
w.Write([]byte(completeBody)) w.Write([]byte(completeBody))
......
package objectstore package objectstore
import ( import (
"crypto/md5"
"encoding/hex"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
) )
...@@ -51,25 +49,3 @@ type compoundCompleteMultipartUploadResult struct { ...@@ -51,25 +49,3 @@ type compoundCompleteMultipartUploadResult struct {
func (c *compoundCompleteMultipartUploadResult) isError() bool { func (c *compoundCompleteMultipartUploadResult) isError() bool {
return c.CompleteMultipartUploadError != nil return c.CompleteMultipartUploadError != nil
} }
// BuildMultipartUploadETag creates an S3 compatible ETag for MultipartUpload
// Given the MD5 hash for each uploaded part of the file, concatenate
// the hashes into a single binary string and calculate the MD5 hash of that result,
// the append "-len(etags)"
// http://permalink.gmane.org/gmane.comp.file-systems.s3.s3tools/583
func (cmu *CompleteMultipartUpload) BuildMultipartUploadETag() (string, error) {
hasher := md5.New()
for _, part := range cmu.Part {
checksum, err := hex.DecodeString(part.ETag)
if err != nil {
return "", err
}
_, err = hasher.Write(checksum)
if err != nil {
return "", err
}
}
multipartChecksum := hasher.Sum(nil)
return fmt.Sprintf("%s-%d", hex.EncodeToString(multipartChecksum), len(cmu.Part)), nil
}
package objectstore
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestMultipartUploadETag(t *testing.T) {
cmu := CompleteMultipartUpload{
Part: []*completeMultipartUploadPart{
{PartNumber: 1, ETag: "550cf6b6e60f65a0e3104a26e70fea42"},
{PartNumber: 2, ETag: "920b914bca0a70780b40881b8f376135"},
{PartNumber: 3, ETag: "175719e13d23c021058bc7376696f26f"},
},
}
expectedETag := "1dc6ab8f946f699770f14f46a8739671-3"
etag, err := cmu.BuildMultipartUploadETag()
require.NoError(t, err)
require.Equal(t, expectedETag, etag)
}
...@@ -220,11 +220,7 @@ func (o *ObjectstoreStub) completeMultipartUpload(w http.ResponseWriter, r *http ...@@ -220,11 +220,7 @@ func (o *ObjectstoreStub) completeMultipartUpload(w http.ResponseWriter, r *http
etag, overwritten := o.overwriteMD5[objectPath] etag, overwritten := o.overwriteMD5[objectPath]
if !overwritten { if !overwritten {
etag, err = msg.BuildMultipartUploadETag() etag = "CompleteMultipartUploadETag"
if err != nil {
http.Error(w, "cannot compute ETag", 400)
return
}
} }
o.bucket[objectPath] = etag o.bucket[objectPath] = etag
......
...@@ -106,7 +106,6 @@ func TestObjectStoreCompleteMultipartUpload(t *testing.T) { ...@@ -106,7 +106,6 @@ func TestObjectStoreCompleteMultipartUpload(t *testing.T) {
contentMD5: "920b914bca0a70780b40881b8f376135", contentMD5: "920b914bca0a70780b40881b8f376135",
}, },
} }
expectedETag := "2f2f82eceacf5bd0ac5d7c3d3d388849-2"
stub.InitiateMultipartUpload(ObjectPath) stub.InitiateMultipartUpload(ObjectPath)
...@@ -149,7 +148,6 @@ func TestObjectStoreCompleteMultipartUpload(t *testing.T) { ...@@ -149,7 +148,6 @@ func TestObjectStoreCompleteMultipartUpload(t *testing.T) {
assert.Equal(len(parts), stub.PutsCnt()) assert.Equal(len(parts), stub.PutsCnt())
assert.Equal(0, stub.DeletesCnt()) assert.Equal(0, stub.DeletesCnt())
assert.Equal(expectedETag, stub.GetObjectMD5(ObjectPath))
assert.False(stub.IsMultipartUpload(ObjectPath), "MultipartUpload is still in progress") assert.False(stub.IsMultipartUpload(ObjectPath), "MultipartUpload is still in progress")
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment