Merge branch 'fix/metadata-full-structure' into 'master'

Add missing entries in build artifacts archive metadata

We need full directory structure, but since ZIP does not require it, we
need to calculate missing entries for directories when we generate
archive metadata.

Closes gitlab-org/gitlab-ce#12634

cc @jacobvosmaer @ayufan 

See merge request !34

cmd/gitlab-zip-metadata/main.go

@@ -23,7 +23,7 @@ func main() {
 	}
 
 	if len(os.Args) != 2 {
-		fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP", progName)
+		fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP\n", progName)
 		os.Exit(1)
 	}
 	if err := zipartifacts.GenerateZipMetadataFromFile(os.Args[1], os.Stdout); err != nil {

internal/zipartifacts/metadata.go

@@ -7,12 +7,14 @@ import (
 	"encoding/json"
 	"io"
 	"os"
+	"path"
+	"sort"
 	"strconv"
 )
 
 type metadata struct {
-	Modified int64  `json:"modified"`
-	Mode     string `json:"mode"`
+	Modified int64  `json:"modified,omitempty"`
+	Mode     string `json:"mode,omitempty"`
 	CRC      uint32 `json:"crc,omitempty"`
 	Size     uint64 `json:"size,omitempty"`
 	Zipped   uint64 `json:"zipped,omitempty"`
@@ -23,6 +25,10 @@ const MetadataHeaderPrefix = "\x00\x00\x00&" // length of string below, encoded
 const MetadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n"
 
 func newMetadata(file *zip.File) metadata {
+	if file == nil {
+		return metadata{}
+	}
+
 	return metadata{
 		Modified: file.ModTime().Unix(),
 		Mode:     strconv.FormatUint(uint64(file.Mode().Perm()), 8),
@@ -42,35 +48,53 @@ func (m metadata) writeEncoded(output io.Writer) error {
 	return writeBytes(output, j)
 }
 
-func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
-	err := writeString(output, entry.Name)
-	if err != nil {
+func writeZipEntryMetadata(output io.Writer, path string, entry *zip.File) error {
+	if err := writeString(output, path); err != nil {
 		return err
 	}
 
-	err = newMetadata(entry).writeEncoded(output)
-	if err != nil {
+	if err := newMetadata(entry).writeEncoded(output); err != nil {
 		return err
 	}
+
 	return nil
 }
 
 func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
-	err := writeString(output, MetadataHeader)
-	if err != nil {
+	if err := writeString(output, MetadataHeader); err != nil {
 		return err
 	}
 
-	// Write empty error string
-	err = writeString(output, "{}")
-	if err != nil {
+	// Write empty error header that we may need in the future
+	if err := writeString(output, "{}"); err != nil {
 		return err
 	}
 
-	// Write all files
+	// Create map of files in zip archive
+	zipMap := make(map[string]*zip.File, len(archive.File))
+
+	// Add missing entries
 	for _, entry := range archive.File {
-		err = writeZipEntryMetadata(output, entry)
-		if err != nil {
+		zipMap[entry.Name] = entry
+
+		for d := path.Dir(entry.Name); d != "." && d != "/"; d = path.Dir(d) {
+			entryDir := d + "/"
+			if _, ok := zipMap[entryDir]; !ok {
+				zipMap[entryDir] = nil
+			}
+		}
+	}
+
+	// Sort paths
+	sortedPaths := make([]string, 0, len(zipMap))
+	for path, _ := range zipMap {
+		sortedPaths = append(sortedPaths, path)
+	}
+	sort.Strings(sortedPaths)
+
+	// Write all files
+	for _, path := range sortedPaths {
+		if err := writeZipEntryMetadata(output, path, zipMap[path]); err != nil {
 			return err
 		}
 	}

internal/zipartifacts/metadata_test.go

+package zipartifacts
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"testing"
+)
+
+func TestMissingMetadataEntries(t *testing.T) {
+	var zipBuffer, metaBuffer bytes.Buffer
+
+	archive := zip.NewWriter(&zipBuffer)
+
+	// non-POSIX paths are here just to test if we never enter infinite loop
+	files := []string{"file1", "some/file/dir/", "some/file/dir/file2", "../../test12/test",
+		"/usr/bin/test", `c:\windows\win32.exe`, `c:/windows/win.dll`, "./f/asd", "/"}
+
+	for _, file := range files {
+		archiveFile, err := archive.Create(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		fmt.Fprint(archiveFile, file)
+	}
+
+	archive.Close()
+
+	zipReader := bytes.NewReader(zipBuffer.Bytes())
+	zipArchiveReader, _ := zip.NewReader(zipReader, int64(binary.Size(zipBuffer.Bytes())))
+	if err := generateZipMetadata(&metaBuffer, zipArchiveReader); err != nil {
+		t.Fatal("zipartifacts: generateZipMetadata failed", err)
+	}
+
+	paths := []string{"file1", "some/", "some/file/", "some/file/dir/", "some/file/dir/file2"}
+	for _, path := range paths {
+		if !bytes.Contains(metaBuffer.Bytes(), []byte(path+"\x00")) {
+			t.Fatal("zipartifacts: metadata for path", path, "not found")
+		}
+	}
+}