cmd/dist: support GOROOT vendoring

In the second step of make.bash, cmd/dist builds cmd/go by invoking
the compiler, linker, and other tools directly on transitive
dependencies of cmd/go. Essentially, cmd/dist acts as a minimal
version of 'go install' when building go_toolchain.

Until now, cmd/go has had no transitive dependencies in vendor
directories. This changes in CL 202698, where several packages are
deleted and equivalent versions in golang.org/x/mod are used
instead. So this CL adds support to cmd/dist for vendor directories.

Updates #31761

Change-Id: Iab4cdc7e505069a8df296287d16fbaa871944955
Reviewed-on: https://go-review.googlesource.com/c/go/+/203537
Run-TryBot: Jay Conrod <jayconrod@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

src/cmd/dist/build.go

@@ -605,26 +605,26 @@ func startInstall(dir string) chan struct{} {
 
 // runInstall installs the library, package, or binary associated with dir,
 // which is relative to $GOROOT/src.
-func runInstall(dir string, ch chan struct{}) {
-	if dir == "net" || dir == "os/user" || dir == "crypto/x509" {
-		fatalf("go_bootstrap cannot depend on cgo package %s", dir)
+func runInstall(pkg string, ch chan struct{}) {
+	if pkg == "net" || pkg == "os/user" || pkg == "crypto/x509" {
+		fatalf("go_bootstrap cannot depend on cgo package %s", pkg)
 	}
 
 	defer close(ch)
 
-	if dir == "unsafe" {
+	if pkg == "unsafe" {
 		return
 	}
 
 	if vflag > 0 {
 		if goos != gohostos || goarch != gohostarch {
-			errprintf("%s (%s/%s)\n", dir, goos, goarch)
+			errprintf("%s (%s/%s)\n", pkg, goos, goarch)
 		} else {
-			errprintf("%s\n", dir)
+			errprintf("%s\n", pkg)
 		}
 	}
 
-	workdir := pathf("%s/%s", workdir, dir)
+	workdir := pathf("%s/%s", workdir, pkg)
 	xmkdirall(workdir)
 
 	var clean []string
@@ -634,11 +634,14 @@ func runInstall(dir string, ch chan struct{}) {
 		}
 	}()
 
-	// path = full path to dir.
-	path := pathf("%s/src/%s", goroot, dir)
+	// dir = full path to pkg.
+	dir := pathf("%s/src/%s", goroot, pkg)
 	name := filepath.Base(dir)
 
-	ispkg := !strings.HasPrefix(dir, "cmd/") || strings.Contains(dir, "/internal/")
+	// ispkg predicts whether the package should be linked as a binary, based
+	// on the name. There should be no "main" packages in vendor, since
+	// 'go mod vendor' will only copy imported packages there.
+	ispkg := !strings.HasPrefix(pkg, "cmd/") || strings.Contains(pkg, "/internal/") || strings.Contains(pkg, "/vendor/")
 
 	// Start final link command line.
 	// Note: code below knows that link.p[targ] is the target.
@@ -650,7 +653,7 @@ func runInstall(dir string, ch chan struct{}) {
 	if ispkg {
 		// Go library (package).
 		ispackcmd = true
-		link = []string{"pack", pathf("%s/pkg/%s_%s/%s.a", goroot, goos, goarch, dir)}
+		link = []string{"pack", packagefile(pkg)}
 		targ = len(link) - 1
 		xmkdirall(filepath.Dir(link[targ]))
 	} else {
@@ -675,7 +678,7 @@ func runInstall(dir string, ch chan struct{}) {
 	// Gather files that are sources for this target.
 	// Everything in that directory, and any target-specific
 	// additions.
-	files := xreaddir(path)
+	files := xreaddir(dir)
 
 	// Remove files beginning with . or _,
 	// which are likely to be editor temporary files.
@@ -687,7 +690,7 @@ func runInstall(dir string, ch chan struct{}) {
 	})
 
 	for _, dt := range deptab {
-		if dir == dt.prefix || strings.HasSuffix(dt.prefix, "/") && strings.HasPrefix(dir, dt.prefix) {
+		if pkg == dt.prefix || strings.HasSuffix(dt.prefix, "/") && strings.HasPrefix(pkg, dt.prefix) {
 			for _, p := range dt.dep {
 				p = os.ExpandEnv(p)
 				files = append(files, p)
@@ -699,7 +702,7 @@ func runInstall(dir string, ch chan struct{}) {
 	// Convert to absolute paths.
 	for i, p := range files {
 		if !filepath.IsAbs(p) {
-			files[i] = pathf("%s/%s", path, p)
+			files[i] = pathf("%s/%s", dir, p)
 		}
 	}
 
@@ -715,7 +718,7 @@ func runInstall(dir string, ch chan struct{}) {
 		return false
 	ok:
 		t := mtime(p)
-		if !t.IsZero() && !strings.HasSuffix(p, ".a") && !shouldbuild(p, dir) {
+		if !t.IsZero() && !strings.HasSuffix(p, ".a") && !shouldbuild(p, pkg) {
 			return false
 		}
 		if strings.HasSuffix(p, ".go") {
@@ -742,7 +745,7 @@ func runInstall(dir string, ch chan struct{}) {
 	}
 
 	// For package runtime, copy some files into the work space.
-	if dir == "runtime" {
+	if pkg == "runtime" {
 		xmkdirall(pathf("%s/pkg/include", goroot))
 		// For use by assembly and C files.
 		copyfile(pathf("%s/pkg/include/textflag.h", goroot),
@@ -764,7 +767,7 @@ func runInstall(dir string, ch chan struct{}) {
 				if vflag > 1 {
 					errprintf("generate %s\n", p)
 				}
-				gt.gen(path, p)
+				gt.gen(dir, p)
 				// Do not add generated file to clean list.
 				// In runtime, we want to be able to
 				// build the package with the go tool,
@@ -782,22 +785,31 @@ func runInstall(dir string, ch chan struct{}) {
 	built:
 	}
 
-	// Make sure dependencies are installed.
-	var deps []string
+	// Resolve imported packages to actual package paths.
+	// Make sure they're installed.
+	importMap := make(map[string]string)
 	for _, p := range gofiles {
-		deps = append(deps, readimports(p)...)
+		for _, imp := range readimports(p) {
+			importMap[imp] = resolveVendor(imp, dir)
 		}
-	for _, dir1 := range deps {
-		startInstall(dir1)
 	}
-	for _, dir1 := range deps {
-		install(dir1)
+	sortedImports := make([]string, 0, len(importMap))
+	for imp := range importMap {
+		sortedImports = append(sortedImports, imp)
+	}
+	sort.Strings(sortedImports)
+
+	for _, dep := range importMap {
+		startInstall(dep)
+	}
+	for _, dep := range importMap {
+		install(dep)
 	}
 
 	if goos != gohostos || goarch != gohostarch {
 		// We've generated the right files; the go command can do the build.
 		if vflag > 1 {
-			errprintf("skip build for cross-compile %s\n", dir)
+			errprintf("skip build for cross-compile %s\n", pkg)
 		}
 		return
 	}
@@ -830,18 +842,35 @@ func runInstall(dir string, ch chan struct{}) {
 		if err := ioutil.WriteFile(goasmh, nil, 0666); err != nil {
 			fatalf("cannot write empty go_asm.h: %s", err)
 		}
-		bgrun(&wg, path, asmabis...)
+		bgrun(&wg, dir, asmabis...)
 		bgwait(&wg)
 	}
 
+	// Build an importcfg file for the compiler.
+	buf := &bytes.Buffer{}
+	for _, imp := range sortedImports {
+		if imp == "unsafe" {
+			continue
+		}
+		dep := importMap[imp]
+		if imp != dep {
+			fmt.Fprintf(buf, "importmap %s=%s\n", imp, dep)
+		}
+		fmt.Fprintf(buf, "packagefile %s=%s\n", dep, packagefile(dep))
+	}
+	importcfg := pathf("%s/importcfg", workdir)
+	if err := ioutil.WriteFile(importcfg, buf.Bytes(), 0666); err != nil {
+		fatalf("cannot write importcfg file: %v", err)
+	}
+
 	var archive string
 	// The next loop will compile individual non-Go files.
 	// Hand the Go files to the compiler en masse.
 	// For packages containing assembly, this writes go_asm.h, which
 	// the assembly files will need.
-	pkg := dir
-	if strings.HasPrefix(dir, "cmd/") && strings.Count(dir, "/") == 1 {
-		pkg = "main"
+	pkgName := pkg
+	if strings.HasPrefix(pkg, "cmd/") && strings.Count(pkg, "/") == 1 {
+		pkgName = "main"
 	}
 	b := pathf("%s/_go_.a", workdir)
 	clean = append(clean, b)
@@ -852,11 +881,11 @@ func runInstall(dir string, ch chan struct{}) {
 	}
 
 	// Compile Go code.
-	compile := []string{pathf("%s/compile", tooldir), "-std", "-pack", "-o", b, "-p", pkg}
+	compile := []string{pathf("%s/compile", tooldir), "-std", "-pack", "-o", b, "-p", pkgName, "-importcfg", importcfg}
 	if gogcflags != "" {
 		compile = append(compile, strings.Fields(gogcflags)...)
 	}
-	if dir == "runtime" {
+	if pkg == "runtime" {
 		compile = append(compile, "-+")
 	}
 	if len(sfiles) > 0 {
@@ -874,7 +903,7 @@ func runInstall(dir string, ch chan struct{}) {
 	// We use bgrun and immediately wait for it instead of calling run() synchronously.
 	// This executes all jobs through the bgwork channel and allows the process
 	// to exit cleanly in case an error occurs.
-	bgrun(&wg, path, compile...)
+	bgrun(&wg, dir, compile...)
 	bgwait(&wg)
 
 	// Compile the files.
@@ -888,7 +917,7 @@ func runInstall(dir string, ch chan struct{}) {
 		// Change the last character of the output file (which was c or s).
 		b = b[:len(b)-1] + "o"
 		compile = append(compile, "-o", b, p)
-		bgrun(&wg, path, compile...)
+		bgrun(&wg, dir, compile...)
 
 		link = append(link, b)
 		if doclean {
@@ -909,6 +938,12 @@ func runInstall(dir string, ch chan struct{}) {
 	bgwait(&wg)
 }
 
+// packagefile returns the path to a compiled .a file for the given package
+// path. Paths may need to be resolved with resolveVendor first.
+func packagefile(pkg string) string {
+	return pathf("%s/pkg/%s_%s/%s.a", goroot, goos, goarch, pkg)
+}
+
 // matchfield reports whether the field (x,y,z) matches this build.
 // all the elements in the field must be satisfied.
 func matchfield(f string) bool {
@@ -940,7 +975,7 @@ func matchtag(tag string) bool {
 // of GOOS and GOARCH.
 // We also allow the special tag cmd_go_bootstrap.
 // See ../go/bootstrap.go and package go/build.
-func shouldbuild(file, dir string) bool {
+func shouldbuild(file, pkg string) bool {
 	// Check file name for GOOS or GOARCH.
 	name := filepath.Base(file)
 	excluded := func(list []string, ok string) bool {
@@ -982,7 +1017,7 @@ func shouldbuild(file, dir string) bool {
 		if code == "package documentation" {
 			return false
 		}
-		if code == "package main" && dir != "cmd/go" && dir != "cmd/cgo" {
+		if code == "package main" && pkg != "cmd/go" && pkg != "cmd/cgo" {
 			return false
 		}
 		if !strings.HasPrefix(p, "//") {

src/cmd/dist/imports.go

@@ -11,7 +11,10 @@ package main
 import (
 	"bufio"
 	"errors"
+	"fmt"
 	"io"
+	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"unicode/utf8"
@@ -243,3 +246,31 @@ func readimports(file string) []string {
 
 	return imports
 }
+
+// resolveVendor returns a unique package path imported with the given import
+// path from srcDir.
+//
+// resolveVendor assumes that a package is vendored if and only if its first
+// path component contains a dot. If a package is vendored, its import path
+// is returned with a "vendor" or "cmd/vendor" prefix, depending on srcDir.
+// Otherwise, the import path is returned verbatim.
+func resolveVendor(imp, srcDir string) string {
+	var first string
+	if i := strings.Index(imp, "/"); i < 0 {
+		first = imp
+	} else {
+		first = imp[:i]
+	}
+	isStandard := !strings.Contains(first, ".")
+	if isStandard {
+		return imp
+	}
+
+	if strings.HasPrefix(srcDir, filepath.Join(goroot, "src", "cmd")) {
+		return path.Join("cmd", "vendor", imp)
+	} else if strings.HasPrefix(srcDir, filepath.Join(goroot, "src")) {
+		return path.Join("vendor", imp)
+	} else {
+		panic(fmt.Sprintf("srcDir %q not in GOOROT/src", srcDir))
+	}
+}