cmd/go: merge module support from x/vgo repo

This CL corresponds to CL 123361, the final manual CL in that repo,
making this the final manual sync.

All future commits will happen in this repo (the main Go repo),
and we'll update x/vgo automatically with a fixed patch+script.

Change-Id: I572243309c1809727604fd704705a23c30e85d1a
Reviewed-on: https://go-review.googlesource.com/123576
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

src/cmd/go/go_test.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"debug/elf"
 	"debug/macho"
+	"flag"
 	"fmt"
 	"go/format"
 	"internal/race"
@@ -112,6 +113,12 @@ func TestMain(m *testing.M) {
 	}
 	os.Unsetenv("GOROOT_FINAL")
 
+	flag.Parse()
+	if *proxyAddr != "" {
+		StartProxy()
+		select {}
+	}
+
 	if canRun {
 		args := []string{"build", "-tags", "testgo", "-o", "testgo" + exeSuffix}
 		if race.Enabled {
@@ -417,7 +424,8 @@ func (tg *testgoData) doRun(args []string) error {
 func (tg *testgoData) run(args ...string) {
 	tg.t.Helper()
 	if status := tg.doRun(args); status != nil {
-		tg.t.Logf("go %v failed unexpectedly: %v", args, status)
+		wd, _ := os.Getwd()
+		tg.t.Logf("go %v failed unexpectedly in %s: %v", args, wd, status)
 		tg.t.FailNow()
 	}
 }
@@ -760,24 +768,51 @@ func (tg *testgoData) wantNotStale(pkg, reason, msg string) {
 	}
 }
 
+// If -testwork is specified, the test prints the name of the temp directory
+// and does not remove it when done, so that a programmer can
+// poke at the test file tree afterward.
+var testWork = flag.Bool("testwork", false, "")
+
 // cleanup cleans up a test that runs testgo.
 func (tg *testgoData) cleanup() {
 	tg.t.Helper()
 	if tg.wd != "" {
+		wd, _ := os.Getwd()
+		tg.t.Logf("ended in %s", wd)
+
 		if err := os.Chdir(tg.wd); err != nil {
 			// We are unlikely to be able to continue.
 			fmt.Fprintln(os.Stderr, "could not restore working directory, crashing:", err)
 			os.Exit(2)
 		}
 	}
+	if *testWork {
+		tg.t.Logf("TESTWORK=%s\n", tg.path("."))
+		return
+	}
 	for _, path := range tg.temps {
-		tg.check(os.RemoveAll(path))
+		tg.check(removeAll(path))
 	}
 	if tg.tempdir != "" {
-		tg.check(os.RemoveAll(tg.tempdir))
+		tg.check(removeAll(tg.tempdir))
 	}
 }
 
+func removeAll(dir string) error {
+	// module cache has 0444 directories;
+	// make them writable in order to remove content.
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil // ignore errors walking in file system
+		}
+		if info.IsDir() {
+			os.Chmod(path, 0777)
+		}
+		return nil
+	})
+	return os.RemoveAll(dir)
+}
+
 // failSSH puts an ssh executable in the PATH that always fails.
 // This is to stub out uses of ssh by go get.
 func (tg *testgoData) failSSH() {
@@ -1745,8 +1780,8 @@ func TestGoGetTestOnlyPkg(t *testing.T) {
 	defer tg.cleanup()
 	tg.tempDir("gopath")
 	tg.setenv("GOPATH", tg.path("gopath"))
-	tg.run("get", "golang.org/x/tour/content")
-	tg.run("get", "-t", "golang.org/x/tour/content")
+	tg.run("get", "golang.org/x/tour/content...")
+	tg.run("get", "-t", "golang.org/x/tour/content...")
 }
 
 func TestInstalls(t *testing.T) {
@@ -3165,11 +3200,25 @@ func TestBuildDashIInstallsDependencies(t *testing.T) {
 	checkbar("cmd")
 }
 
-func TestGoBuildInTestOnlyDirectoryFailsWithAGoodError(t *testing.T) {
+func TestGoBuildTestOnly(t *testing.T) {
 	tg := testgo(t)
 	defer tg.cleanup()
-	tg.runFail("build", "./testdata/testonly")
-	tg.grepStderr("no non-test Go files in", "go build ./testdata/testonly produced unexpected error")
+	tg.makeTempdir()
+	tg.setenv("GOPATH", tg.path("."))
+	tg.tempFile("src/testonly/t_test.go", `package testonly`)
+	tg.tempFile("src/testonly2/t.go", `package testonly2`)
+	tg.cd(tg.path("src"))
+
+	// Named explicitly, test-only packages should be reported as unbuildable/uninstallable,
+	// even if there is a wildcard also matching.
+	tg.runFail("build", "testonly", "testonly...")
+	tg.grepStderr("no non-test Go files in", "go build ./xtestonly produced unexpected error")
+	tg.runFail("install", "./testonly")
+	tg.grepStderr("no non-test Go files in", "go install ./testonly produced unexpected error")
+
+	// Named through a wildcards, the test-only packages should be silently ignored.
+	tg.run("build", "testonly...")
+	tg.run("install", "./testonly...")
 }
 
 func TestGoTestDetectsTestOnlyImportCycles(t *testing.T) {
@@ -6115,12 +6164,12 @@ func TestBadCommandLines(t *testing.T) {
 	tg.tempFile("src/@x/x.go", "package x\n")
 	tg.setenv("GOPATH", tg.path("."))
 	tg.runFail("build", "@x")
-	tg.grepStderr("invalid input directory name \"@x\"", "did not reject @x directory")
+	tg.grepStderr("invalid input directory name \"@x\"|cannot use path@version syntax", "did not reject @x directory")
 
 	tg.tempFile("src/@x/y/y.go", "package y\n")
 	tg.setenv("GOPATH", tg.path("."))
 	tg.runFail("build", "@x/y")
-	tg.grepStderr("invalid import path \"@x/y\"", "did not reject @x/y import path")
+	tg.grepStderr("invalid import path \"@x/y\"|cannot use path@version syntax", "did not reject @x/y import path")
 
 	tg.tempFile("src/-x/x.go", "package x\n")
 	tg.setenv("GOPATH", tg.path("."))

src/cmd/go/internal/cfg/cfg.go

@@ -21,6 +21,7 @@ var (
 	BuildA                 bool   // -a flag
 	BuildBuildmode         string // -buildmode flag
 	BuildContext           = build.Default
+	BuildGetmode           string             // -getmode flag
 	BuildI                 bool               // -i flag
 	BuildLinkshared        bool               // -linkshared flag
 	BuildMSan              bool               // -msan flag
@@ -67,6 +68,11 @@ var (
 	Goos      = BuildContext.GOOS
 	ExeSuffix string
 	Gopath    = filepath.SplitList(BuildContext.GOPATH)
+
+	// ModulesEnabled specifies whether the go command is running
+	// in module-aware mode (as opposed to GOPATH mode).
+	// It is equal to modload.Enabled, but not all packages can import modload.
+	ModulesEnabled bool
 )
 
 func init() {

src/cmd/go/internal/envcmd/env.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strings"
 
@@ -16,7 +17,7 @@ import (
 	"cmd/go/internal/cache"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
-	"cmd/go/internal/vgo"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/work"
 )
 
@@ -112,6 +113,18 @@ func findEnv(env []cfg.EnvVar, name string) string {
 
 // ExtraEnvVars returns environment variables that should not leak into child processes.
 func ExtraEnvVars() []cfg.EnvVar {
+	gomod := ""
+	if modload.Init(); modload.ModRoot != "" {
+		gomod = filepath.Join(modload.ModRoot, "go.mod")
+	}
+	return []cfg.EnvVar{
+		{Name: "GOMOD", Value: gomod},
+	}
+}
+
+// ExtraEnvVarsCostly returns environment variables that should not leak into child processes
+// but are costly to evaluate.
+func ExtraEnvVarsCostly() []cfg.EnvVar {
 	var b work.Builder
 	b.Init()
 	cppflags, cflags, cxxflags, fflags, ldflags, err := b.CFlags(&load.Package{})
@@ -121,6 +134,7 @@ func ExtraEnvVars() []cfg.EnvVar {
 		return nil
 	}
 	cmd := b.GccCmd(".", "")
+
 	return []cfg.EnvVar{
 		// Note: Update the switch in runEnv below when adding to this list.
 		{Name: "CGO_CFLAGS", Value: strings.Join(cflags, " ")},
@@ -130,19 +144,19 @@ func ExtraEnvVars() []cfg.EnvVar {
 		{Name: "CGO_LDFLAGS", Value: strings.Join(ldflags, " ")},
 		{Name: "PKG_CONFIG", Value: b.PkgconfigCmd()},
 		{Name: "GOGCCFLAGS", Value: strings.Join(cmd[3:], " ")},
-		{Name: "VGOMODROOT", Value: vgo.ModRoot},
 	}
 }
 
 func runEnv(cmd *base.Command, args []string) {
 	env := cfg.CmdEnv
+	env = append(env, ExtraEnvVars()...)
 
-	// Do we need to call ExtraEnvVars, which is a bit expensive?
+	// Do we need to call ExtraEnvVarsCostly, which is a bit expensive?
 	// Only if we're listing all environment variables ("go env")
 	// or the variables being requested are in the extra list.
-	needExtra := true
+	needCostly := true
 	if len(args) > 0 {
-		needExtra = false
+		needCostly = false
 		for _, arg := range args {
 			switch arg {
 			case "CGO_CFLAGS",
@@ -152,12 +166,12 @@ func runEnv(cmd *base.Command, args []string) {
 				"CGO_LDFLAGS",
 				"PKG_CONFIG",
 				"GOGCCFLAGS":
-				needExtra = true
+				needCostly = true
 			}
 		}
 	}
-	if needExtra {
-		env = append(env, ExtraEnvVars()...)
+	if needCostly {
+		env = append(env, ExtraEnvVarsCostly()...)
 	}
 
 	if len(args) > 0 {

src/cmd/go/internal/fix/fix.go

@@ -9,8 +9,8 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/str"
-	"cmd/go/internal/vgo"
 	"fmt"
 	"os"
 )
@@ -34,9 +34,9 @@ See also: go fmt, go vet.
 func runFix(cmd *base.Command, args []string) {
 	printed := false
 	for _, pkg := range load.Packages(args) {
-		if vgo.Enabled() && !pkg.Module.Top {
+		if modload.Enabled() && !pkg.Module.Main {
 			if !printed {
-				fmt.Fprintf(os.Stderr, "vgo: not fixing packages in dependency modules\n")
+				fmt.Fprintf(os.Stderr, "go: not fixing packages in dependency modules\n")
 				printed = true
 			}
 			continue

src/cmd/go/internal/fmtcmd/fmt.go

@@ -16,8 +16,8 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/str"
-	"cmd/go/internal/vgo"
 )
 
 func init() {
@@ -60,9 +60,9 @@ func runFmt(cmd *base.Command, args []string) {
 		}()
 	}
 	for _, pkg := range load.PackagesAndErrors(args) {
-		if vgo.Enabled() && !pkg.Module.Top {
+		if modload.Enabled() && !pkg.Module.Main {
 			if !printed {
-				fmt.Fprintf(os.Stderr, "vgo: not formatting packages in dependency modules\n")
+				fmt.Fprintf(os.Stderr, "go: not formatting packages in dependency modules\n")
 				printed = true
 			}
 			continue

src/cmd/go/internal/generate/generate.go

@@ -21,7 +21,7 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
-	"cmd/go/internal/vgo"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/work"
 )
 
@@ -161,9 +161,9 @@ func runGenerate(cmd *base.Command, args []string) {
 	// Even if the arguments are .go files, this loop suffices.
 	printed := false
 	for _, pkg := range load.Packages(args) {
-		if vgo.Enabled() && !pkg.Module.Top {
+		if modload.Enabled() && !pkg.Module.Main {
 			if !printed {
-				fmt.Fprintf(os.Stderr, "vgo: not generating in packages in dependency modules\n")
+				fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
 				printed = true
 			}
 			continue

src/cmd/go/internal/get/discovery.go

@@ -28,7 +28,7 @@ func charsetReader(charset string, input io.Reader) (io.Reader, error) {
 
 // parseMetaGoImports returns meta imports from the HTML in r.
 // Parsing ends at the end of the <head> section or the beginning of the <body>.
-func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
+func parseMetaGoImports(r io.Reader, mod ModuleMode) (imports []metaImport, err error) {
 	d := xml.NewDecoder(r)
 	d.CharsetReader = charsetReader
 	d.Strict = false
@@ -39,13 +39,13 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
 			if err == io.EOF || len(imports) > 0 {
 				err = nil
 			}
-			return
+			break
 		}
 		if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
-			return
+			break
 		}
 		if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
-			return
+			break
 		}
 		e, ok := t.(xml.StartElement)
 		if !ok || !strings.EqualFold(e.Name.Local, "meta") {
@@ -55,13 +55,6 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
 			continue
 		}
 		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
-			// Ignore VCS type "mod", which is new Go modules.
-			// This code is for old go get and must ignore the new mod lines.
-			// Otherwise matchGoImport will complain about two
-			// different metaImport lines for the same Prefix.
-			if f[1] == "mod" {
-				continue
-			}
 			imports = append(imports, metaImport{
 				Prefix:   f[0],
 				VCS:      f[1],
@@ -69,6 +62,27 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
 			})
 		}
 	}
+
+	// Extract mod entries if we are paying attention to them.
+	var list []metaImport
+	var have map[string]bool
+	if mod == PreferMod {
+		have = make(map[string]bool)
+		for _, m := range imports {
+			if m.VCS == "mod" {
+				have[m.Prefix] = true
+				list = append(list, m)
+			}
+		}
+	}
+
+	// Append non-mod entries, ignoring those superseded by a mod entry.
+	for _, m := range imports {
+		if m.VCS != "mod" && !have[m.Prefix] {
+			list = append(list, m)
+		}
+	}
+	return list, nil
 }
 
 // attrValue returns the attribute value for the case-insensitive key

src/cmd/go/internal/get/get.go

@@ -18,13 +18,12 @@ import (
 	"cmd/go/internal/load"
 	"cmd/go/internal/search"
 	"cmd/go/internal/str"
-	"cmd/go/internal/vgo"
 	"cmd/go/internal/web"
 	"cmd/go/internal/work"
 )
 
 var CmdGet = &base.Command{
-	UsageLine: "get [-d] [-f] [-fix] [-insecure] [-t] [-u] [-v] [build flags] [packages]",
+	UsageLine: "get [-d] [-f] [-t] [-u] [-v] [-fix] [-insecure] [build flags] [packages]",
 	Short:     "download and install packages and dependencies",
 	Long: `
 Get downloads the packages named by the import paths, along with their
@@ -75,25 +74,49 @@ For more about specifying packages, see 'go help packages'.
 For more about how 'go get' finds source code to
 download, see 'go help importpath'.
 
+This text describes the behavior of get when using GOPATH
+to manage source code and dependencies.
+If instead the go command is running in module-aware mode,
+the details of get's flags and effects change, as does 'go help get'.
+See 'go help modules' and 'go help module-get'.
+
 See also: go build, go install, go clean.
 	`,
 }
 
-var getD = CmdGet.Flag.Bool("d", false, "")
-var getF = CmdGet.Flag.Bool("f", false, "")
-var getT = CmdGet.Flag.Bool("t", false, "")
-var getU = CmdGet.Flag.Bool("u", false, "")
-var getFix = CmdGet.Flag.Bool("fix", false, "")
-var getInsecure = CmdGet.Flag.Bool("insecure", false, "")
+var HelpGopathGet = &base.Command{
+	UsageLine: "gopath-get",
+	Short:     "legacy GOPATH go get",
+	Long: `
+The 'go get' command changes behavior depending on whether the
+go command is running in module-aware mode or legacy GOPATH mode.
+This help text, accessible as 'go help gopath-get' even in module-aware mode,
+describes 'go get' as it operates in legacy GOPATH mode.
+
+Usage: ` + CmdGet.UsageLine + `
+` + CmdGet.Long,
+}
+
+var (
+	getD   = CmdGet.Flag.Bool("d", false, "")
+	getF   = CmdGet.Flag.Bool("f", false, "")
+	getT   = CmdGet.Flag.Bool("t", false, "")
+	getU   = CmdGet.Flag.Bool("u", false, "")
+	getFix = CmdGet.Flag.Bool("fix", false, "")
+
+	Insecure bool
+)
 
 func init() {
 	work.AddBuildFlags(CmdGet)
 	CmdGet.Run = runGet // break init loop
+	CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "")
 }
 
 func runGet(cmd *base.Command, args []string) {
-	if vgo.Enabled() {
-		base.Fatalf("go get: vgo not implemented")
+	if cfg.ModulesEnabled {
+		// Should not happen: main.go should install the separate module-enabled get code.
+		base.Fatalf("go get: modules not implemented")
 	}
 
 	work.BuildInit()
@@ -167,7 +190,7 @@ func runGet(cmd *base.Command, args []string) {
 		return
 	}
 
-	work.InstallPackages(args, true)
+	work.InstallPackages(args)
 }
 
 // downloadPaths prepares the list of paths to pass to download.
@@ -176,6 +199,12 @@ func runGet(cmd *base.Command, args []string) {
 // in the hope that we can figure out the repository from the
 // initial ...-free prefix.
 func downloadPaths(args []string) []string {
+	for _, arg := range args {
+		if strings.Contains(arg, "@") {
+			base.Fatalf("go: cannot use path@version syntax in GOPATH mode")
+		}
+	}
+
 	args = load.ImportPathsForGoGet(args)
 	var out []string
 	for _, a := range args {
@@ -379,7 +408,7 @@ func downloadPackage(p *load.Package) error {
 	)
 
 	security := web.Secure
-	if *getInsecure {
+	if Insecure {
 		security = web.Insecure
 	}
 
@@ -402,16 +431,16 @@ func downloadPackage(p *load.Package) error {
 			}
 			repo = remote
 			if !*getF && err == nil {
-				if rr, err := repoRootForImportPath(p.ImportPath, security); err == nil {
-					repo := rr.repo
+				if rr, err := RepoRootForImportPath(p.ImportPath, IgnoreMod, security); err == nil {
+					repo := rr.Repo
 					if rr.vcs.resolveRepo != nil {
 						resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo)
 						if err == nil {
 							repo = resolved
 						}
 					}
-					if remote != repo && rr.isCustom {
-						return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.root, repo, dir, remote)
+					if remote != repo && rr.IsCustom {
+						return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.Root, repo, dir, remote)
 					}
 				}
 			}
@@ -419,13 +448,13 @@ func downloadPackage(p *load.Package) error {
 	} else {
 		// Analyze the import path to determine the version control system,
 		// repository, and the import path for the root of the repository.
-		rr, err := repoRootForImportPath(p.ImportPath, security)
+		rr, err := RepoRootForImportPath(p.ImportPath, IgnoreMod, security)
 		if err != nil {
 			return err
 		}
-		vcs, repo, rootPath = rr.vcs, rr.repo, rr.root
+		vcs, repo, rootPath = rr.vcs, rr.Repo, rr.Root
 	}
-	if !blindRepo && !vcs.isSecure(repo) && !*getInsecure {
+	if !blindRepo && !vcs.isSecure(repo) && !Insecure {
 		return fmt.Errorf("cannot download, %v uses insecure protocol", repo)
 	}
 

src/cmd/go/internal/get/pkg_test.go

@@ -33,15 +33,18 @@ func TestFoldDup(t *testing.T) {
 
 var parseMetaGoImportsTests = []struct {
 	in  string
+	mod ModuleMode
 	out []metaImport
 }{
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
+		IgnoreMod,
 		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
 	},
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		<meta name="go-import" content="baz/quux git http://github.com/rsc/baz/quux">`,
+		IgnoreMod,
 		[]metaImport{
 			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
 			{"baz/quux", "git", "http://github.com/rsc/baz/quux"},
@@ -50,6 +53,7 @@ var parseMetaGoImportsTests = []struct {
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">`,
+		IgnoreMod,
 		[]metaImport{
 			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
 		},
@@ -57,35 +61,48 @@ var parseMetaGoImportsTests = []struct {
 	{
 		`<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">
 		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
+		IgnoreMod,
 		[]metaImport{
 			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
 		},
 	},
+	{
+		`<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">
+		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
+		PreferMod,
+		[]metaImport{
+			{"foo/bar", "mod", "http://github.com/rsc/baz/quux"},
+		},
+	},
 	{
 		`<head>
 		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		</head>`,
+		IgnoreMod,
 		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
 	},
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		<body>`,
+		IgnoreMod,
 		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
 	},
 	{
 		`<!doctype html><meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
+		IgnoreMod,
 		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
 	},
 	{
 		// XML doesn't like <div style=position:relative>.
 		`<!doctype html><title>Page Not Found</title><meta name=go-import content="chitin.io/chitin git https://github.com/chitin-io/chitin"><div style=position:relative>DRAFT</div>`,
+		IgnoreMod,
 		[]metaImport{{"chitin.io/chitin", "git", "https://github.com/chitin-io/chitin"}},
 	},
 }
 
 func TestParseMetaGoImports(t *testing.T) {
 	for i, tt := range parseMetaGoImportsTests {
-		out, err := parseMetaGoImports(strings.NewReader(tt.in))
+		out, err := parseMetaGoImports(strings.NewReader(tt.in), tt.mod)
 		if err != nil {
 			t.Errorf("test#%d: %v", i, err)
 			continue

src/cmd/go/internal/get/vcs.go

@@ -624,27 +624,29 @@ func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error {
 	return nil
 }
 
-// repoRoot represents a version control system, a repo, and a root of
-// where to put it on disk.
-type repoRoot struct {
-	vcs *vcsCmd
-
-	// repo is the repository URL, including scheme
-	repo string
-
-	// root is the import path corresponding to the root of the
-	// repository
-	root string
-
-	// isCustom is true for custom import paths (those defined by HTML meta tags)
-	isCustom bool
+// RepoRoot describes the repository root for a tree of source code.
+type RepoRoot struct {
+	Repo     string // repository URL, including scheme
+	Root     string // import path corresponding to root of repo
+	IsCustom bool   // defined by served <meta> tags (as opposed to hard-coded pattern)
+	VCS      string // vcs type ("mod", "git", ...)
+
+	vcs *vcsCmd // internal: vcs command access
 }
 
 var httpPrefixRE = regexp.MustCompile(`^https?:`)
 
-// repoRootForImportPath analyzes importPath to determine the
+// ModuleMode specifies whether to prefer modules when looking up code sources.
+type ModuleMode int
+
+const (
+	IgnoreMod ModuleMode = iota
+	PreferMod
+)
+
+// RepoRootForImportPath analyzes importPath to determine the
 // version control system, and code repository to use.
-func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoRoot, error) {
+func RepoRootForImportPath(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
 	rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths)
 	if err == errUnknownSite {
 		// If there are wildcards, look up the thing before the wildcard,
@@ -654,7 +656,7 @@ func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoR
 		if i := strings.Index(lookup, "/.../"); i >= 0 {
 			lookup = lookup[:i]
 		}
-		rr, err = repoRootForImportDynamic(lookup, security)
+		rr, err = repoRootForImportDynamic(lookup, mod, security)
 		if err != nil {
 			err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err)
 		}
@@ -667,7 +669,7 @@ func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoR
 		}
 	}
 
-	if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.root, "...") {
+	if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") {
 		// Do not allow wildcards in the repo root.
 		rr = nil
 		err = fmt.Errorf("cannot expand ... in %q", importPath)
@@ -680,7 +682,7 @@ var errUnknownSite = errors.New("dynamic lookup required to find mapping")
 // repoRootFromVCSPaths attempts to map importPath to a repoRoot
 // using the mappings defined in vcsPaths.
 // If scheme is non-empty, that scheme is forced.
-func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, vcsPaths []*vcsPath) (*repoRoot, error) {
+func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) {
 	// A common error is to use https://packagepath because that's what
 	// hg and git require. Diagnose this helpfully.
 	if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil {
@@ -733,28 +735,32 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode,
 					if security == web.Secure && !vcs.isSecureScheme(scheme) {
 						continue
 					}
-					if vcs.ping(scheme, match["repo"]) == nil {
+					if vcs.pingCmd != "" && vcs.ping(scheme, match["repo"]) == nil {
 						match["repo"] = scheme + "://" + match["repo"]
-						break
+						goto Found
 					}
 				}
+				// No scheme found. Fall back to the first one.
+				match["repo"] = vcs.scheme[0] + "://" + match["repo"]
+			Found:
 			}
 		}
-		rr := &repoRoot{
+		rr := &RepoRoot{
+			Repo: match["repo"],
+			Root: match["root"],
+			VCS:  vcs.cmd,
 			vcs:  vcs,
-			repo: match["repo"],
-			root: match["root"],
 		}
 		return rr, nil
 	}
 	return nil, errUnknownSite
 }
 
-// repoRootForImportDynamic finds a *repoRoot for a custom domain that's not
+// repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not
 // statically known by repoRootForImportPathStatic.
 //
 // This handles custom import paths like "name.tld/pkg/foo" or just "name.tld".
-func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*repoRoot, error) {
+func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
 	slash := strings.Index(importPath, "/")
 	if slash < 0 {
 		slash = len(importPath)
@@ -772,7 +778,7 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re
 		return nil, fmt.Errorf(msg, err)
 	}
 	defer body.Close()
-	imports, err := parseMetaGoImports(body)
+	imports, err := parseMetaGoImports(body, mod)
 	if err != nil {
 		return nil, fmt.Errorf("parsing %s: %v", importPath, err)
 	}
@@ -799,7 +805,7 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re
 		}
 		urlStr0 := urlStr
 		var imports []metaImport
-		urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, security)
+		urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security)
 		if err != nil {
 			return nil, err
 		}
@@ -812,15 +818,18 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re
 	if err := validateRepoRoot(mmi.RepoRoot); err != nil {
 		return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, mmi.RepoRoot, err)
 	}
-	rr := &repoRoot{
-		vcs:      vcsByCmd(mmi.VCS),
-		repo:     mmi.RepoRoot,
-		root:     mmi.Prefix,
-		isCustom: true,
-	}
-	if rr.vcs == nil {
+	vcs := vcsByCmd(mmi.VCS)
+	if vcs == nil && mmi.VCS != "mod" {
 		return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS)
 	}
+
+	rr := &RepoRoot{
+		Repo:     mmi.RepoRoot,
+		Root:     mmi.Prefix,
+		IsCustom: true,
+		VCS:      mmi.VCS,
+		vcs:      vcs,
+	}
 	return rr, nil
 }
 
@@ -851,7 +860,7 @@ var (
 // It is an error if no imports are found.
 // urlStr will still be valid if err != nil.
 // The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1"
-func metaImportsForPrefix(importPrefix string, security web.SecurityMode) (urlStr string, imports []metaImport, err error) {
+func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (urlStr string, imports []metaImport, err error) {
 	setCache := func(res fetchResult) (fetchResult, error) {
 		fetchCacheMu.Lock()
 		defer fetchCacheMu.Unlock()
@@ -871,7 +880,7 @@ func metaImportsForPrefix(importPrefix string, security web.SecurityMode) (urlSt
 		if err != nil {
 			return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)})
 		}
-		imports, err := parseMetaGoImports(body)
+		imports, err := parseMetaGoImports(body, mod)
 		if err != nil {
 			return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)})
 		}

src/cmd/go/internal/get/vcs_test.go

@@ -16,43 +16,43 @@ import (
 	"cmd/go/internal/web"
 )
 
-// Test that RepoRootForImportPath creates the correct RepoRoot for a given importPath.
+// Test that RepoRootForImportPath determines the correct RepoRoot for a given importPath.
 // TODO(cmang): Add tests for SVN and BZR.
 func TestRepoRootForImportPath(t *testing.T) {
 	testenv.MustHaveExternalNetwork(t)
 
 	tests := []struct {
 		path string
-		want *repoRoot
+		want *RepoRoot
 	}{
 		{
 			"github.com/golang/groupcache",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://github.com/golang/groupcache",
+				Repo: "https://github.com/golang/groupcache",
 			},
 		},
 		// Unicode letters in directories (issue 18660).
 		{
 			"github.com/user/unicode/испытание",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://github.com/user/unicode",
+				Repo: "https://github.com/user/unicode",
 			},
 		},
 		// IBM DevOps Services tests
 		{
 			"hub.jazz.net/git/user1/pkgname",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://hub.jazz.net/git/user1/pkgname",
+				Repo: "https://hub.jazz.net/git/user1/pkgname",
 			},
 		},
 		{
 			"hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://hub.jazz.net/git/user1/pkgname",
+				Repo: "https://hub.jazz.net/git/user1/pkgname",
 			},
 		},
 		{
@@ -91,9 +91,9 @@ func TestRepoRootForImportPath(t *testing.T) {
 		},
 		{
 			"hub.jazz.net/git/user/pkg.name",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://hub.jazz.net/git/user/pkg.name",
+				Repo: "https://hub.jazz.net/git/user/pkg.name",
 			},
 		},
 		// User names cannot have uppercase letters
@@ -104,9 +104,9 @@ func TestRepoRootForImportPath(t *testing.T) {
 		// OpenStack tests
 		{
 			"git.openstack.org/openstack/swift",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://git.openstack.org/openstack/swift",
+				Repo: "https://git.openstack.org/openstack/swift",
 			},
 		},
 		// Trailing .git is less preferred but included for
@@ -114,16 +114,16 @@ func TestRepoRootForImportPath(t *testing.T) {
 		// be compilable on both old and new go
 		{
 			"git.openstack.org/openstack/swift.git",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://git.openstack.org/openstack/swift.git",
+				Repo: "https://git.openstack.org/openstack/swift.git",
 			},
 		},
 		{
 			"git.openstack.org/openstack/swift/go/hummingbird",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://git.openstack.org/openstack/swift",
+				Repo: "https://git.openstack.org/openstack/swift",
 			},
 		},
 		{
@@ -150,23 +150,23 @@ func TestRepoRootForImportPath(t *testing.T) {
 		},
 		{
 			"git.apache.org/package-name.git",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://git.apache.org/package-name.git",
+				Repo: "https://git.apache.org/package-name.git",
 			},
 		},
 		{
 			"git.apache.org/package-name_2.x.git/path/to/lib",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://git.apache.org/package-name_2.x.git",
+				Repo: "https://git.apache.org/package-name_2.x.git",
 			},
 		},
 		{
 			"chiselapp.com/user/kyle/repository/fossilgg",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsFossil,
-				repo: "https://chiselapp.com/user/kyle/repository/fossilgg",
+				Repo: "https://chiselapp.com/user/kyle/repository/fossilgg",
 			},
 		},
 		{
@@ -181,21 +181,21 @@ func TestRepoRootForImportPath(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		got, err := repoRootForImportPath(test.path, web.Secure)
+		got, err := RepoRootForImportPath(test.path, IgnoreMod, web.Secure)
 		want := test.want
 
 		if want == nil {
 			if err == nil {
-				t.Errorf("repoRootForImportPath(%q): Error expected but not received", test.path)
+				t.Errorf("RepoRootForImportPath(%q): Error expected but not received", test.path)
 			}
 			continue
 		}
 		if err != nil {
-			t.Errorf("repoRootForImportPath(%q): %v", test.path, err)
+			t.Errorf("RepoRootForImportPath(%q): %v", test.path, err)
 			continue
 		}
-		if got.vcs.name != want.vcs.name || got.repo != want.repo {
-			t.Errorf("repoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.repo, want.vcs, want.repo)
+		if got.vcs.name != want.vcs.name || got.Repo != want.Repo {
+			t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.Repo, want.vcs, want.Repo)
 		}
 	}
 }
@@ -227,18 +227,18 @@ func TestFromDir(t *testing.T) {
 			f.Close()
 		}
 
-		want := repoRoot{
+		want := RepoRoot{
 			vcs:  vcs,
-			root: path.Join("example.com", vcs.name),
+			Root: path.Join("example.com", vcs.name),
 		}
-		var got repoRoot
-		got.vcs, got.root, err = vcsFromDir(dir, tempDir)
+		var got RepoRoot
+		got.vcs, got.Root, err = vcsFromDir(dir, tempDir)
 		if err != nil {
 			t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err)
 			continue
 		}
-		if got.vcs.name != want.vcs.name || got.root != want.root {
-			t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.root, want.vcs, want.root)
+		if got.vcs.name != want.vcs.name || got.Root != want.Root {
+			t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.Root, want.vcs, want.Root)
 		}
 	}
 }

src/cmd/go/internal/help/help.go

@@ -45,7 +45,15 @@ func Help(args []string) {
 		buf := new(bytes.Buffer)
 		PrintUsage(buf)
 		usage := &base.Command{Long: buf.String()}
-		tmpl(&commentWriter{W: os.Stdout}, documentationTemplate, append([]*base.Command{usage}, base.Commands...))
+		cmds := []*base.Command{usage}
+		for _, cmd := range base.Commands {
+			if cmd.UsageLine == "gopath-get" {
+				// Avoid duplication of the "get" documentation.
+				continue
+			}
+			cmds = append(cmds, cmd)
+		}
+		tmpl(&commentWriter{W: os.Stdout}, documentationTemplate, cmds)
 		fmt.Println("package main")
 		return
 	}

src/cmd/go/internal/help/helpdoc.go

@@ -30,7 +30,7 @@ the C or C++ compiler, respectively, to use.
 
 var HelpPackages = &base.Command{
 	UsageLine: "packages",
-	Short:     "package lists",
+	Short:     "package lists and patterns",
 	Long: `
 Many commands apply to a set of packages:
 
@@ -54,9 +54,11 @@ for packages to be built with the go tool:
 
 - "main" denotes the top-level package in a stand-alone executable.
 
-- "all" expands to all package directories found in all the GOPATH
+- "all" expands to all packages found in all the GOPATH
 trees. For example, 'go list all' lists all the packages on the local
-system.
+system. When using modules, "all" expands to all packages in
+the main module and their dependencies, including dependencies
+needed by tests of any of those.
 
 - "std" is like all but expands to just the packages in the standard
 Go library.

src/cmd/go/internal/load/flag.go

@@ -6,6 +6,7 @@ package load
 
 import (
 	"cmd/go/internal/base"
+	"cmd/go/internal/search"
 	"cmd/go/internal/str"
 	"fmt"
 	"strings"
@@ -92,7 +93,10 @@ func (f *PerPackageFlag) For(p *Package) []string {
 	return flags
 }
 
-var cmdlineMatchers []func(*Package) bool
+var (
+	cmdlineMatchers        []func(*Package) bool
+	cmdlineMatcherLiterals []func(*Package) bool
+)
 
 // SetCmdlinePatterns records the set of patterns given on the command line,
 // for use by the PerPackageFlags.
@@ -105,9 +109,15 @@ func setCmdlinePatterns(args []string, cwd string) {
 		args = []string{"."}
 	}
 	cmdlineMatchers = nil // allow reset for testing
+	cmdlineMatcherLiterals = nil
 	for _, arg := range args {
 		cmdlineMatchers = append(cmdlineMatchers, MatchPackage(arg, cwd))
 	}
+	for _, arg := range args {
+		if !strings.Contains(arg, "...") && !search.IsMetaPackage(arg) {
+			cmdlineMatcherLiterals = append(cmdlineMatcherLiterals, MatchPackage(arg, cwd))
+		}
+	}
 }
 
 // isCmdlinePkg reports whether p is a package listed on the command line.
@@ -119,3 +129,15 @@ func isCmdlinePkg(p *Package) bool {
 	}
 	return false
 }
+
+// isCmdlinePkgLiteral reports whether p is a package listed as
+// a literal package argument on the command line
+// (as opposed to being the result of expanding a wildcard).
+func isCmdlinePkgLiteral(p *Package) bool {
+	for _, m := range cmdlineMatcherLiterals {
+		if m(p) {
+			return true
+		}
+	}
+	return false
+}

src/cmd/go/internal/load/search.go

@@ -15,7 +15,7 @@ import (
 // MatchPackage(pattern, cwd)(p) reports whether package p matches pattern in the working directory cwd.
 func MatchPackage(pattern, cwd string) func(*Package) bool {
 	switch {
-	case strings.HasPrefix(pattern, "./") || strings.HasPrefix(pattern, "../") || pattern == "." || pattern == "..":
+	case search.IsRelativePath(pattern):
 		// Split pattern into leading pattern-free directory path
 		// (including all . and .. elements) and the final pattern.
 		var dir string

src/cmd/go/internal/vgo/vendor.go → src/cmd/go/internal/modcmd/vendor.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package vgo
+package modcmd
 
 import (
 	"bytes"
@@ -14,143 +14,165 @@ import (
 	"strings"
 
 	"cmd/go/internal/base"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/module"
 )
 
-var CmdVendor = &base.Command{
-	UsageLine: "vendor [-v]",
-	Short:     "vendor dependencies of current module",
-	Long: `
-Vendor resets the module's vendor directory to include all
-packages needed to build and test all packages in the module
-and their dependencies.
-
-The -v flag causes vendor to print to standard error the
-module paths of the modules processed and the import paths
-of the packages copied.
-	`,
-}
-
-var vendorV = CmdVendor.Flag.Bool("v", false, "")
+func runVendor() {
+	pkgs := modload.LoadVendor()
 
-func init() {
-	CmdVendor.Run = runVendor // break init cycle
-}
-
-func runVendor(cmd *base.Command, args []string) {
-	if Init(); !Enabled() {
-		base.Fatalf("vgo vendor: cannot use -m outside module")
-	}
-	if len(args) != 0 {
-		base.Fatalf("vgo vendor: vendor takes no arguments")
-	}
-	InitMod()
-	pkgs := ImportPaths([]string{"ALL"})
-
-	vdir := filepath.Join(ModRoot, "vendor")
+	vdir := filepath.Join(modload.ModRoot, "vendor")
 	if err := os.RemoveAll(vdir); err != nil {
-		base.Fatalf("vgo vendor: %v", err)
+		base.Fatalf("go vendor: %v", err)
 	}
 
 	modpkgs := make(map[module.Version][]string)
 	for _, pkg := range pkgs {
-		m := pkgmod[pkg]
-		if m == Target {
+		m := modload.PackageModule(pkg)
+		if m == modload.Target {
 			continue
 		}
 		modpkgs[m] = append(modpkgs[m], pkg)
 	}
 
 	var buf bytes.Buffer
-	for _, m := range buildList[1:] {
+	for _, m := range modload.BuildList()[1:] {
 		if pkgs := modpkgs[m]; len(pkgs) > 0 {
 			repl := ""
-			if r := replaced(m); r != nil {
-				repl = " => " + r.New.Path
-				if r.New.Version != "" {
-					repl += " " + r.New.Version
+			if r := modload.Replacement(m); r.Path != "" {
+				repl = " => " + r.Path
+				if r.Version != "" {
+					repl += " " + r.Version
 				}
 			}
 			fmt.Fprintf(&buf, "# %s %s%s\n", m.Path, m.Version, repl)
-			if *vendorV {
+			if *modV {
 				fmt.Fprintf(os.Stderr, "# %s %s%s\n", m.Path, m.Version, repl)
 			}
 			for _, pkg := range pkgs {
 				fmt.Fprintf(&buf, "%s\n", pkg)
-				if *vendorV {
+				if *modV {
 					fmt.Fprintf(os.Stderr, "%s\n", pkg)
 				}
 				vendorPkg(vdir, pkg)
 			}
 		}
 	}
-	if err := ioutil.WriteFile(filepath.Join(vdir, "vgo.list"), buf.Bytes(), 0666); err != nil {
-		base.Fatalf("vgo vendor: %v", err)
+	if buf.Len() == 0 {
+		fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
+		return
+	}
+	if err := ioutil.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil {
+		base.Fatalf("go vendor: %v", err)
 	}
 }
 
 func vendorPkg(vdir, pkg string) {
-	realPath := importmap[pkg]
-	if realPath != pkg && importmap[realPath] != "" {
+	realPath := modload.ImportMap(pkg)
+	if realPath != pkg && modload.ImportMap(realPath) != "" {
 		fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
 	}
 
 	dst := filepath.Join(vdir, pkg)
-	src := pkgdir[realPath]
+	src := modload.PackageDir(realPath)
 	if src == "" {
 		fmt.Fprintf(os.Stderr, "internal error: no pkg for %s -> %s\n", pkg, realPath)
 	}
-	copyDir(dst, src, false)
+	copyDir(dst, src, matchNonTest)
+	if m := modload.PackageModule(realPath); m.Path != "" {
+		copyMetadata(m.Path, realPath, dst, src)
+	}
 }
 
-func copyDir(dst, src string, recursive bool) {
+type metakey struct {
+	modPath string
+	dst     string
+}
+
+var copiedMetadata = make(map[metakey]bool)
+
+// copyMetadata copies metadata files from parents of src to parents of dst,
+// stopping after processing the src parent for modPath.
+func copyMetadata(modPath, pkg, dst, src string) {
+	for parent := 0; ; parent++ {
+		if copiedMetadata[metakey{modPath, dst}] {
+			break
+		}
+		copiedMetadata[metakey{modPath, dst}] = true
+		if parent > 0 {
+			copyDir(dst, src, matchMetadata)
+		}
+		if modPath == pkg {
+			break
+		}
+		pkg = filepath.Dir(pkg)
+		dst = filepath.Dir(dst)
+		src = filepath.Dir(src)
+	}
+}
+
+// metaPrefixes is the list of metadata file prefixes.
+// Vendoring copies metadata files from parents of copied directories.
+// Note that this list could be arbitrarily extended, and it is longer
+// in other tools (such as godep or dep). By using this limited set of
+// prefixes and also insisting on capitalized file names, we are trying
+// to nudge people toward more agreement on the naming
+// and also trying to avoid false positives.
+var metaPrefixes = []string{
+	"AUTHORS",
+	"CONTRIBUTORS",
+	"COPYLEFT",
+	"COPYING",
+	"COPYRIGHT",
+	"LEGAL",
+	"LICENSE",
+	"NOTICE",
+	"PATENTS",
+}
+
+// matchMetadata reports whether info is a metadata file.
+func matchMetadata(info os.FileInfo) bool {
+	name := info.Name()
+	for _, p := range metaPrefixes {
+		if strings.HasPrefix(name, p) {
+			return true
+		}
+	}
+	return false
+}
+
+// matchNonTest reports whether info is any non-test file (including non-Go files).
+func matchNonTest(info os.FileInfo) bool {
+	return !strings.HasSuffix(info.Name(), "_test.go")
+}
+
+// copyDir copies all regular files satisfying match(info) from src to dst.
+func copyDir(dst, src string, match func(os.FileInfo) bool) {
 	files, err := ioutil.ReadDir(src)
 	if err != nil {
-		base.Fatalf("vgo vendor: %v", err)
+		base.Fatalf("go vendor: %v", err)
 	}
 	if err := os.MkdirAll(dst, 0777); err != nil {
-		base.Fatalf("vgo vendor: %v", err)
+		base.Fatalf("go vendor: %v", err)
 	}
 	for _, file := range files {
-		if file.IsDir() {
-			if recursive || file.Name() == "testdata" {
-				copyDir(filepath.Join(dst, file.Name()), filepath.Join(src, file.Name()), true)
-			}
-			continue
-		}
-		if !file.Mode().IsRegular() {
+		if file.IsDir() || !file.Mode().IsRegular() || !match(file) {
 			continue
 		}
 		r, err := os.Open(filepath.Join(src, file.Name()))
 		if err != nil {
-			base.Fatalf("vgo vendor: %v", err)
+			base.Fatalf("go vendor: %v", err)
 		}
 		w, err := os.Create(filepath.Join(dst, file.Name()))
 		if err != nil {
-			base.Fatalf("vgo vendor: %v", err)
+			base.Fatalf("go vendor: %v", err)
 		}
 		if _, err := io.Copy(w, r); err != nil {
-			base.Fatalf("vgo vendor: %v", err)
+			base.Fatalf("go vendor: %v", err)
 		}
 		r.Close()
 		if err := w.Close(); err != nil {
-			base.Fatalf("vgo vendor: %v", err)
-		}
-	}
-}
-
-// hasPathPrefix reports whether the path s begins with the
-// elements in prefix.
-func hasPathPrefix(s, prefix string) bool {
-	switch {
-	default:
-		return false
-	case len(s) == len(prefix):
-		return s == prefix
-	case len(s) > len(prefix):
-		if prefix != "" && prefix[len(prefix)-1] == '/' {
-			return strings.HasPrefix(s, prefix)
+			base.Fatalf("go vendor: %v", err)
 		}
-		return s[len(prefix)] == '/' && s[:len(prefix)] == prefix
 	}
 }

src/cmd/go/internal/vgo/verify.go → src/cmd/go/internal/modcmd/verify.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package vgo
+package modcmd
 
 import (
 	"bytes"
@@ -13,43 +13,14 @@ import (
 
 	"cmd/go/internal/base"
 	"cmd/go/internal/dirhash"
+	"cmd/go/internal/modfetch"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/module"
 )
 
-var CmdVerify = &base.Command{
-	UsageLine: "verify",
-	Run:       runVerify,
-	Short:     "verify downloaded modules against expected hashes",
-	Long: `
-Verify checks that the dependencies of the current module,
-which are stored in a local downloaded source cache,
-have not been modified since being downloaded.
-
-If all the modules are unmodified, verify prints
-
-	all modules verified
-
-and exits successfully (status 0). Otherwise, verify reports
-which modules have been changed and exits with a non-zero status.
-	`,
-}
-
-func runVerify(cmd *base.Command, args []string) {
-	if Init(); !Enabled() {
-		base.Fatalf("vgo verify: cannot use outside module")
-	}
-	if len(args) != 0 {
-		// TODO: take arguments
-		base.Fatalf("vgo verify: verify takes no arguments")
-	}
-
-	// Make go.mod consistent but don't load any packages.
-	InitMod()
-	iterate(func(*loader) {})
-	writeGoMod()
-
+func runVerify() {
 	ok := true
-	for _, mod := range buildList[1:] {
+	for _, mod := range modload.LoadBuildList()[1:] {
 		ok = verifyMod(mod) && ok
 	}
 	if ok {
@@ -59,9 +30,9 @@ func runVerify(cmd *base.Command, args []string) {
 
 func verifyMod(mod module.Version) bool {
 	ok := true
-	zip := filepath.Join(srcV, "cache", mod.Path, "/@v/", mod.Version+".zip")
+	zip := filepath.Join(modfetch.SrcMod, "cache/download", mod.Path, "/@v/", mod.Version+".zip")
 	_, zipErr := os.Stat(zip)
-	dir := filepath.Join(srcV, mod.Path+"@"+mod.Version)
+	dir := filepath.Join(modfetch.SrcMod, mod.Path+"@"+mod.Version)
 	_, dirErr := os.Stat(dir)
 	data, err := ioutil.ReadFile(zip + "hash")
 	if err != nil {

src/cmd/go/internal/modfetch/convert.go → src/cmd/go/internal/modconv/convert.go

@@ -2,16 +2,20 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package modfetch
+package modconv
 
 import (
 	"fmt"
 	"os"
 	"sort"
 	"strings"
+	"sync"
 
-	"cmd/go/internal/modconv"
+	"cmd/go/internal/base"
+	"cmd/go/internal/modfetch"
 	"cmd/go/internal/modfile"
+	"cmd/go/internal/module"
+	"cmd/go/internal/par"
 	"cmd/go/internal/semver"
 )
 
@@ -23,47 +27,45 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
 	if i >= 0 {
 		j = strings.LastIndex(file[:i], "/")
 	}
-	convert := modconv.Converters[file[i+1:]]
+	convert := Converters[file[i+1:]]
 	if convert == nil && j != -2 {
-		convert = modconv.Converters[file[j+1:]]
+		convert = Converters[file[j+1:]]
 	}
 	if convert == nil {
 		return fmt.Errorf("unknown legacy config file %s", file)
 	}
-	require, err := convert(file, data)
+	mf, err := convert(file, data)
 	if err != nil {
 		return fmt.Errorf("parsing %s: %v", file, err)
 	}
 
 	// Convert requirements block, which may use raw SHA1 hashes as versions,
 	// to valid semver requirement list, respecting major versions.
-	need := make(map[string]string)
-	for _, r := range require {
-		if r.Path == "" {
+	var work par.Work
+	for _, r := range mf.Require {
+		m := r.Mod
+		if m.Path == "" {
 			continue
 		}
+		work.Add(r.Mod)
+	}
 
-		// TODO: Something better here.
-		if strings.HasPrefix(r.Path, "github.com/") || strings.HasPrefix(r.Path, "golang.org/x/") {
-			f := strings.Split(r.Path, "/")
-			if len(f) > 3 {
-				r.Path = strings.Join(f[:3], "/")
-			}
-		}
-
-		repo, err := Lookup(r.Path)
+	var (
+		mu   sync.Mutex
+		need = make(map[string]string)
+	)
+	work.Do(10, func(item interface{}) {
+		r := item.(module.Version)
+		repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "vgo: lookup %s: %v\n", r.Path, err)
-			continue
-		}
-		info, err := repo.Stat(r.Version)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "vgo: stat %s@%s: %v\n", r.Path, r.Version, err)
-			continue
+			fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err)
+			return
 		}
+		mu.Lock()
 		path := repo.ModulePath()
 		need[path] = semver.Max(need[path], info.Version)
-	}
+		mu.Unlock()
+	})
 
 	var paths []string
 	for path := range need {
@@ -71,8 +73,15 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
 	}
 	sort.Strings(paths)
 	for _, path := range paths {
-		f.AddRequire(path, need[path])
+		f.AddNewRequire(path, need[path], false)
 	}
 
+	for _, r := range mf.Replace {
+		err := f.AddReplace(r.Old.Path, r.Old.Version, r.New.Path, r.New.Version)
+		if err != nil {
+			return fmt.Errorf("add replace: %v", err)
+		}
+	}
+	f.Cleanup()
 	return nil
 }

src/cmd/go/internal/modfetch/convert_test.go → src/cmd/go/internal/modconv/convert_test.go

@@ -2,19 +2,49 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package modfetch
+package modconv
 
 import (
 	"bytes"
+	"fmt"
 	"internal/testenv"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"cmd/go/internal/cfg"
-	"cmd/go/internal/modconv"
+	"cmd/go/internal/modfetch"
+	"cmd/go/internal/modfetch/codehost"
 	"cmd/go/internal/modfile"
+	"cmd/go/internal/module"
 )
 
+func TestMain(m *testing.M) {
+	os.Exit(testMain(m))
+}
+
+func testMain(m *testing.M) int {
+	if _, err := exec.LookPath("git"); err != nil {
+		fmt.Fprintln(os.Stderr, "skipping because git binary not found")
+		fmt.Println("PASS")
+		return 0
+	}
+
+	dir, err := ioutil.TempDir("", "modconv-test-")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	modfetch.SrcMod = filepath.Join(dir, "src/mod")
+	codehost.WorkRoot = filepath.Join(dir, "codework")
+
+	return m.Run()
+}
+
 func TestConvertLegacyConfig(t *testing.T) {
 	testenv.MustHaveExternalNetwork(t)
 
@@ -65,7 +95,7 @@ func TestConvertLegacyConfig(t *testing.T) {
 			require (
 				github.com/AdRoll/goamz v0.0.0-20150130162828-d3664b76d905
 				github.com/MSOpenTech/azure-sdk-for-go v0.0.0-20150323223030-d90753bcad2e
-				github.com/Sirupsen/logrus v0.0.0-20150409230825-55eb11d21d2a
+				github.com/Sirupsen/logrus v0.7.3
 				github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd
 				github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b
 				github.com/bugsnag/panicwrap v0.0.0-20141110184334-e5f9854865b9
@@ -117,23 +147,32 @@ func TestConvertLegacyConfig(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			repo, err := Lookup(tt.path)
-			if err != nil {
-				t.Fatal(err)
-			}
 
-			out, err := repo.GoMod(tt.vers)
+			dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers})
 			if err != nil {
 				t.Fatal(err)
 			}
-			prefix := modconv.Prefix + "\n"
-			if !bytes.HasPrefix(out, []byte(prefix)) {
-				t.Fatalf("go.mod missing prefix %q:\n%s", prefix, out)
-			}
-			out = out[len(prefix):]
-			if !bytes.Equal(out, want) {
-				t.Fatalf("final go.mod:\n%s\n\nwant:\n%s", out, want)
+
+			for name := range Converters {
+				file := filepath.Join(dir, name)
+				data, err := ioutil.ReadFile(file)
+				if err == nil {
+					f := new(modfile.File)
+					f.AddModuleStmt(tt.path)
+					if err := ConvertLegacyConfig(f, filepath.ToSlash(file), data); err != nil {
+						t.Fatal(err)
+					}
+					out, err := f.Format()
+					if err != nil {
+						t.Fatalf("format after conversion: %v", err)
+					}
+					if !bytes.Equal(out, want) {
+						t.Fatalf("final go.mod:\n%s\n\nwant:\n%s", out, want)
+					}
+					return
+				}
 			}
+			t.Fatalf("no converter found for %s@%s", tt.path, tt.vers)
 		})
 	}
 }

src/cmd/go/internal/modconv/dep.go

@@ -9,11 +9,13 @@ import (
 	"strconv"
 	"strings"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 	"cmd/go/internal/semver"
 )
 
-func ParseGopkgLock(file string, data []byte) ([]module.Version, error) {
+func ParseGopkgLock(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	var list []module.Version
 	var r *module.Version
 	for lineno, line := range strings.Split(string(data), "\n") {
@@ -66,6 +68,7 @@ func ParseGopkgLock(file string, data []byte) ([]module.Version, error) {
 		if r.Path == "" || r.Version == "" {
 			return nil, fmt.Errorf("%s: empty [[projects]] stanza (%s)", file, r.Path)
 		}
+		mf.Require = append(mf.Require, &modfile.Require{Mod: r})
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/glide.go

@@ -5,12 +5,14 @@
 package modconv
 
 import (
-	"cmd/go/internal/module"
 	"strings"
+
+	"cmd/go/internal/modfile"
+	"cmd/go/internal/module"
 )
 
-func ParseGlideLock(file string, data []byte) ([]module.Version, error) {
-	var list []module.Version
+func ParseGlideLock(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	imports := false
 	name := ""
 	for lineno, line := range strings.Split(string(data), "\n") {
@@ -32,9 +34,9 @@ func ParseGlideLock(file string, data []byte) ([]module.Version, error) {
 		if strings.HasPrefix(line, "  version:") {
 			version := strings.TrimSpace(line[len("  version:"):])
 			if name != "" && version != "" {
-				list = append(list, module.Version{Path: name, Version: version})
+				mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: name, Version: version}})
 			}
 		}
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/glock.go

@@ -7,17 +7,18 @@ package modconv
 import (
 	"strings"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseGLOCKFILE(file string, data []byte) ([]module.Version, error) {
-	var list []module.Version
+func ParseGLOCKFILE(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	for lineno, line := range strings.Split(string(data), "\n") {
 		lineno++
 		f := strings.Fields(line)
 		if len(f) >= 2 && f[0] != "cmd" {
-			list = append(list, module.Version{Path: f[0], Version: f[1]})
+			mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[1]}})
 		}
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/godeps.go

@@ -7,10 +7,11 @@ package modconv
 import (
 	"encoding/json"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseGodepsJSON(file string, data []byte) ([]module.Version, error) {
+func ParseGodepsJSON(file string, data []byte) (*modfile.File, error) {
 	var cfg struct {
 		ImportPath string
 		Deps       []struct {
@@ -21,9 +22,9 @@ func ParseGodepsJSON(file string, data []byte) ([]module.Version, error) {
 	if err := json.Unmarshal(data, &cfg); err != nil {
 		return nil, err
 	}
-	var list []module.Version
+	mf := new(modfile.File)
 	for _, d := range cfg.Deps {
-		list = append(list, module.Version{Path: d.ImportPath, Version: d.Rev})
+		mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: d.ImportPath, Version: d.Rev}})
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/modconv.go

@@ -4,9 +4,9 @@
 
 package modconv
 
-import "cmd/go/internal/module"
+import "cmd/go/internal/modfile"
 
-var Converters = map[string]func(string, []byte) ([]module.Version, error){
+var Converters = map[string]func(string, []byte) (*modfile.File, error){
 	"GLOCKFILE":          ParseGLOCKFILE,
 	"Godeps/Godeps.json": ParseGodepsJSON,
 	"Gopkg.lock":         ParseGopkgLock,
@@ -17,9 +17,3 @@ var Converters = map[string]func(string, []byte) ([]module.Version, error){
 	"vendor/manifest":    ParseVendorManifest,
 	"vendor/vendor.json": ParseVendorJSON,
 }
-
-// Prefix is a line we write at the top of auto-converted go.mod files
-// for dependencies before caching them.
-// In case of bugs in the converter, if we bump this version number,
-// then all the cached copies will be ignored.
-const Prefix = "//vgo 0.0.4\n"

src/cmd/go/internal/modconv/modconv_test.go

@@ -7,7 +7,6 @@ package modconv
 import (
 	"bytes"
 	"fmt"
-	"internal/testenv"
 	"io/ioutil"
 	"path/filepath"
 	"testing"
@@ -26,8 +25,6 @@ var extMap = map[string]string{
 }
 
 func Test(t *testing.T) {
-	testenv.MustHaveExternalNetwork(t)
-
 	tests, _ := filepath.Glob("testdata/*")
 	if len(tests) == 0 {
 		t.Fatalf("no tests found")
@@ -58,8 +55,8 @@ func Test(t *testing.T) {
 				t.Error(err)
 			}
 			var buf bytes.Buffer
-			for _, r := range out {
-				fmt.Fprintf(&buf, "%s %s\n", r.Path, r.Version)
+			for _, r := range out.Require {
+				fmt.Fprintf(&buf, "%s %s\n", r.Mod.Path, r.Mod.Version)
 			}
 			if !bytes.Equal(buf.Bytes(), want) {
 				t.Errorf("have:\n%s\nwant:\n%s", buf.Bytes(), want)

src/cmd/go/internal/modconv/tsv.go

@@ -7,17 +7,18 @@ package modconv
 import (
 	"strings"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseDependenciesTSV(file string, data []byte) ([]module.Version, error) {
-	var list []module.Version
+func ParseDependenciesTSV(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	for lineno, line := range strings.Split(string(data), "\n") {
 		lineno++
 		f := strings.Split(line, "\t")
 		if len(f) >= 3 {
-			list = append(list, module.Version{Path: f[0], Version: f[2]})
+			mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[2]}})
 		}
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/vconf.go

@@ -7,11 +7,12 @@ package modconv
 import (
 	"strings"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseVendorConf(file string, data []byte) ([]module.Version, error) {
-	var list []module.Version
+func ParseVendorConf(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	for lineno, line := range strings.Split(string(data), "\n") {
 		lineno++
 		if i := strings.Index(line, "#"); i >= 0 {
@@ -19,8 +20,8 @@ func ParseVendorConf(file string, data []byte) ([]module.Version, error) {
 		}
 		f := strings.Fields(line)
 		if len(f) >= 2 {
-			list = append(list, module.Version{Path: f[0], Version: f[1]})
+			mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[1]}})
 		}
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/vjson.go

@@ -7,10 +7,11 @@ package modconv
 import (
 	"encoding/json"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseVendorJSON(file string, data []byte) ([]module.Version, error) {
+func ParseVendorJSON(file string, data []byte) (*modfile.File, error) {
 	var cfg struct {
 		Package []struct {
 			Path     string
@@ -20,9 +21,9 @@ func ParseVendorJSON(file string, data []byte) ([]module.Version, error) {
 	if err := json.Unmarshal(data, &cfg); err != nil {
 		return nil, err
 	}
-	var list []module.Version
+	mf := new(modfile.File)
 	for _, d := range cfg.Package {
-		list = append(list, module.Version{Path: d.Path, Version: d.Revision})
+		mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: d.Path, Version: d.Revision}})
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/vmanifest.go

@@ -7,10 +7,11 @@ package modconv
 import (
 	"encoding/json"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseVendorManifest(file string, data []byte) ([]module.Version, error) {
+func ParseVendorManifest(file string, data []byte) (*modfile.File, error) {
 	var cfg struct {
 		Dependencies []struct {
 			ImportPath string
@@ -20,9 +21,9 @@ func ParseVendorManifest(file string, data []byte) ([]module.Version, error) {
 	if err := json.Unmarshal(data, &cfg); err != nil {
 		return nil, err
 	}
-	var list []module.Version
+	mf := new(modfile.File)
 	for _, d := range cfg.Dependencies {
-		list = append(list, module.Version{Path: d.ImportPath, Version: d.Revision})
+		mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: d.ImportPath, Version: d.Revision}})
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/vyml.go

@@ -5,12 +5,14 @@
 package modconv
 
 import (
-	"cmd/go/internal/module"
 	"strings"
+
+	"cmd/go/internal/modfile"
+	"cmd/go/internal/module"
 )
 
-func ParseVendorYML(file string, data []byte) ([]module.Version, error) {
-	var list []module.Version
+func ParseVendorYML(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	vendors := false
 	path := ""
 	for lineno, line := range strings.Split(string(data), "\n") {
@@ -32,9 +34,9 @@ func ParseVendorYML(file string, data []byte) ([]module.Version, error) {
 		if strings.HasPrefix(line, "  rev:") {
 			rev := strings.TrimSpace(line[len("  rev:"):])
 			if path != "" && rev != "" {
-				list = append(list, module.Version{Path: path, Version: rev})
+				mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: path, Version: rev}})
 			}
 		}
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modfetch/bitbucket/fetch.go

-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bitbucket
-
-import (
-	"fmt"
-	"strings"
-
-	"cmd/go/internal/modfetch/codehost"
-	"cmd/go/internal/modfetch/gitrepo"
-)
-
-func Lookup(path string) (codehost.Repo, error) {
-	f := strings.Split(path, "/")
-	if len(f) < 3 || f[0] != "bitbucket.org" {
-		return nil, fmt.Errorf("bitbucket repo must be bitbucket.org/org/project")
-	}
-	path = f[0] + "/" + f[1] + "/" + f[2]
-	return gitrepo.Repo("https://"+path, path)
-}

src/cmd/go/internal/modfetch/gopkgin.go → src/cmd/go/internal/modfetch/cache_test.go

@@ -2,21 +2,24 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// TODO: Figure out what gopkg.in should do.
-
 package modfetch
 
 import (
-	"cmd/go/internal/modfetch/codehost"
-	"cmd/go/internal/modfetch/gitrepo"
-	"cmd/go/internal/modfile"
-	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
 )
 
-func gopkginLookup(path string) (codehost.Repo, error) {
-	root, _, _, _, ok := modfile.ParseGopkgIn(path)
-	if !ok {
-		return nil, fmt.Errorf("invalid gopkg.in/ path: %q", path)
+func TestWriteDiskCache(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "go-writeCache-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	err = writeDiskCache(filepath.Join(tmpdir, "file"), []byte("data"))
+	if err != nil {
+		t.Fatal(err)
 	}
-	return gitrepo.Repo("https://"+root, root)
 }

src/cmd/go/internal/modfetch/codehost/codehost.go

@@ -16,6 +16,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"cmd/go/internal/cfg"
@@ -32,10 +33,8 @@ const (
 // A Repo represents a code hosting source.
 // Typical implementations include local version control repositories,
 // remote version control servers, and code hosting sites.
+// A Repo must be safe for simultaneous use by multiple goroutines.
 type Repo interface {
-	// Root returns the import path of the root directory of the repository.
-	Root() string
-
 	// List lists all tags with the given prefix.
 	Tags(prefix string) (tags []string, err error)
 
@@ -50,6 +49,9 @@ type Repo interface {
 
 	// ReadFile reads the given file in the file tree corresponding to revision rev.
 	// It should refuse to read more than maxSize bytes.
+	//
+	// If the requested file does not exist it should return an error for which
+	// os.IsNotExist(err) returns true.
 	ReadFile(rev, file string, maxSize int64) (data []byte, err error)
 
 	// ReadZip downloads a zip file for the subdir subdirectory
@@ -66,8 +68,9 @@ type Repo interface {
 type RevInfo struct {
 	Name    string    // complete ID in underlying repository
 	Short   string    // shortened ID, for use in pseudo-version
-	Version string    // TODO what is this?
+	Version string    // version used in lookup
 	Time    time.Time // commit time
+	Tags    []string  // known tags for commit
 }
 
 // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
@@ -92,7 +95,7 @@ func ShortenSHA1(rev string) string {
 }
 
 // WorkRoot is the root of the cached work directory.
-// It is set by cmd/go/internal/vgo.InitMod.
+// It is set by cmd/go/internal/modload.InitMod.
 var WorkRoot string
 
 // WorkDir returns the name of the cached work directory to use for the
@@ -113,21 +116,20 @@ func WorkDir(typ, name string) (string, error) {
 	key := typ + ":" + name
 	dir := filepath.Join(WorkRoot, fmt.Sprintf("%x", sha256.Sum256([]byte(key))))
 	data, err := ioutil.ReadFile(dir + ".info")
-	if err == nil {
+	info, err2 := os.Stat(dir)
+	if err == nil && err2 == nil && info.IsDir() {
+		// Info file and directory both already exist: reuse.
 		have := strings.TrimSuffix(string(data), "\n")
 		if have != key {
 			return "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key)
 		}
-		_, err := os.Stat(dir)
-		if err != nil {
-			return "", fmt.Errorf("%s exists but %s does not", dir+".info", dir)
-		}
 		if cfg.BuildX {
 			fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name)
 		}
 		return dir, nil
 	}
 
+	// Info file or directory missing. Start from scratch.
 	if cfg.BuildX {
 		fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name)
 	}
@@ -157,19 +159,36 @@ func (e *RunError) Error() string {
 	return text
 }
 
+var dirLock sync.Map
+
 // Run runs the command line in the given directory
 // (an empty dir means the current directory).
 // It returns the standard output and, for a non-zero exit,
 // a *RunError indicating the command, exit status, and standard error.
 // Standard error is unavailable for commands that exit successfully.
 func Run(dir string, cmdline ...interface{}) ([]byte, error) {
+	if dir != "" {
+		muIface, ok := dirLock.Load(dir)
+		if !ok {
+			muIface, _ = dirLock.LoadOrStore(dir, new(sync.Mutex))
+		}
+		mu := muIface.(*sync.Mutex)
+		mu.Lock()
+		defer mu.Unlock()
+	}
+
 	cmd := str.StringList(cmdline...)
 	if cfg.BuildX {
-		var cd string
+		var text string
 		if dir != "" {
-			cd = "cd " + dir + "; "
+			text = "cd " + dir + "; "
 		}
-		fmt.Fprintf(os.Stderr, "%s%s\n", cd, strings.Join(cmd, " "))
+		text += strings.Join(cmd, " ")
+		fmt.Fprintf(os.Stderr, "%s\n", text)
+		start := time.Now()
+		defer func() {
+			fmt.Fprintf(os.Stderr, "%.3fs # %s\n", time.Since(start).Seconds(), text)
+		}()
 	}
 	// TODO: Impose limits on command output size.
 	// TODO: Set environment to get English error messages.

src/cmd/go/internal/modfetch/codehost/shell.go

+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Interactive debugging shell for codehost.Repo implementations.
+
+package main
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"time"
+
+	"cmd/go/internal/modfetch/codehost"
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "usage: go run shell.go vcs remote\n")
+	os.Exit(2)
+}
+
+func main() {
+	codehost.WorkRoot = "/tmp/vcswork"
+	log.SetFlags(0)
+	log.SetPrefix("shell: ")
+	flag.Usage = usage
+	flag.Parse()
+	if flag.NArg() != 2 {
+		usage()
+	}
+
+	repo, err := codehost.NewRepo(flag.Arg(0), flag.Arg(1))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	b := bufio.NewReader(os.Stdin)
+	for {
+		fmt.Fprintf(os.Stderr, ">>> ")
+		line, err := b.ReadString('\n')
+		if err != nil {
+			log.Fatal(err)
+		}
+		f := strings.Fields(line)
+		if len(f) == 0 {
+			continue
+		}
+		switch f[0] {
+		default:
+			fmt.Fprintf(os.Stderr, "?unknown command\n")
+			continue
+		case "tags":
+			prefix := ""
+			if len(f) == 2 {
+				prefix = f[1]
+			}
+			if len(f) > 2 {
+				fmt.Fprintf(os.Stderr, "?usage: tags [prefix]\n")
+				continue
+			}
+			tags, err := repo.Tags(prefix)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+			for _, tag := range tags {
+				fmt.Printf("%s\n", tag)
+			}
+
+		case "stat":
+			if len(f) != 2 {
+				fmt.Fprintf(os.Stderr, "?usage: stat rev\n")
+				continue
+			}
+			info, err := repo.Stat(f[1])
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+			fmt.Printf("name=%s short=%s version=%s time=%s\n", info.Name, info.Short, info.Version, info.Time.UTC().Format(time.RFC3339))
+
+		case "read":
+			if len(f) != 3 {
+				fmt.Fprintf(os.Stderr, "?usage: read rev file\n")
+				continue
+			}
+			data, err := repo.ReadFile(f[1], f[2], 10<<20)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+			os.Stdout.Write(data)
+
+		case "zip":
+			if len(f) != 4 {
+				fmt.Fprintf(os.Stderr, "?usage: zip rev subdir output\n")
+				continue
+			}
+			subdir := f[2]
+			if subdir == "-" {
+				subdir = ""
+			}
+			rc, _, err := repo.ReadZip(f[1], subdir, 10<<20)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+			data, err := ioutil.ReadAll(rc)
+			rc.Close()
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+
+			if f[3] != "-" {
+				if err := ioutil.WriteFile(f[3], data, 0666); err != nil {
+					fmt.Fprintf(os.Stderr, "?%s\n", err)
+					continue
+				}
+			}
+			z, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+			for _, f := range z.File {
+				fmt.Printf("%s %d\n", f.Name, f.UncompressedSize64)
+			}
+		}
+	}
+}

src/cmd/go/internal/modfetch/domain.go

-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Support for custom domains.
-
-package modfetch
-
-import (
-	"encoding/xml"
-	"fmt"
-	"io"
-	"net/url"
-	"os"
-	"strings"
-
-	"cmd/go/internal/modfetch/codehost"
-	"cmd/go/internal/modfetch/gitrepo"
-)
-
-// metaImport represents the parsed <meta name="go-import"
-// content="prefix vcs reporoot" /> tags from HTML files.
-type metaImport struct {
-	Prefix, VCS, RepoRoot string
-}
-
-func lookupCustomDomain(path string) (Repo, error) {
-	dom := path
-	if i := strings.Index(dom, "/"); i >= 0 {
-		dom = dom[:i]
-	}
-	if !strings.Contains(dom, ".") {
-		return nil, fmt.Errorf("unknown module %s: not a domain name", path)
-	}
-	var body io.ReadCloser
-	err := webGetGoGet("https://"+path+"?go-get=1", &body)
-	if body != nil {
-		defer body.Close()
-	}
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "FindRepo: %v\n", err)
-		return nil, err
-	}
-	// Note: accepting a non-200 OK here, so people can serve a
-	// meta import in their http 404 page.
-	imports, err := parseMetaGoImports(body)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "findRepo: %v\n", err)
-		return nil, err
-	}
-	if len(imports) == 0 {
-		return nil, fmt.Errorf("unknown module %s: no go-import tags", path)
-	}
-
-	// First look for new module definition.
-	for _, imp := range imports {
-		if path == imp.Prefix || strings.HasPrefix(path, imp.Prefix+"/") {
-			if imp.VCS == "mod" {
-				u, err := url.Parse(imp.RepoRoot)
-				if err != nil {
-					return nil, fmt.Errorf("invalid module URL %q", imp.RepoRoot)
-				} else if u.Scheme != "https" {
-					// TODO: Allow -insecure flag as a build flag?
-					return nil, fmt.Errorf("invalid module URL %q: must be HTTPS", imp.RepoRoot)
-				}
-				return newProxyRepo(imp.RepoRoot, imp.Prefix), nil
-			}
-		}
-	}
-
-	// Fall back to redirections to known version control systems.
-	for _, imp := range imports {
-		if path == imp.Prefix {
-			if !strings.HasPrefix(imp.RepoRoot, "https://") {
-				// TODO: Allow -insecure flag as a build flag?
-				return nil, fmt.Errorf("invalid server URL %q: must be HTTPS", imp.RepoRoot)
-			}
-			if imp.VCS == "git" {
-				code, err := gitrepo.Repo(imp.RepoRoot, imp.Prefix)
-				if err != nil {
-					return nil, err
-				}
-				return newCodeRepo(code, path)
-			}
-			return nil, fmt.Errorf("unknown VCS, Repo: %s, %s", imp.VCS, imp.RepoRoot)
-		}
-	}
-
-	// Check for redirect to repo root.
-	for _, imp := range imports {
-		if strings.HasPrefix(path, imp.Prefix+"/") {
-			return nil, &ModuleSubdirError{imp.Prefix}
-		}
-	}
-
-	return nil, fmt.Errorf("unknown module %s: no matching go-import tags", path)
-}
-
-type ModuleSubdirError struct {
-	ModulePath string
-}
-
-func (e *ModuleSubdirError) Error() string {
-	return fmt.Sprintf("module root is %q", e.ModulePath)
-}
-
-type customPrefix struct {
-	codehost.Repo
-	root string
-}
-
-func (c *customPrefix) Root() string {
-	return c.root
-}
-
-// parseMetaGoImports returns meta imports from the HTML in r.
-// Parsing ends at the end of the <head> section or the beginning of the <body>.
-func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
-	d := xml.NewDecoder(r)
-	d.CharsetReader = charsetReader
-	d.Strict = false
-	var t xml.Token
-	for {
-		t, err = d.RawToken()
-		if err != nil {
-			if err == io.EOF || len(imports) > 0 {
-				err = nil
-			}
-			return
-		}
-		if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
-			return
-		}
-		if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
-			return
-		}
-		e, ok := t.(xml.StartElement)
-		if !ok || !strings.EqualFold(e.Name.Local, "meta") {
-			continue
-		}
-		if attrValue(e.Attr, "name") != "go-import" {
-			continue
-		}
-		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
-			imports = append(imports, metaImport{
-				Prefix:   f[0],
-				VCS:      f[1],
-				RepoRoot: f[2],
-			})
-		}
-	}
-}
-
-// attrValue returns the attribute value for the case-insensitive key
-// `name', or the empty string if nothing is found.
-func attrValue(attrs []xml.Attr, name string) string {
-	for _, a := range attrs {
-		if strings.EqualFold(a.Name.Local, name) {
-			return a.Value
-		}
-	}
-	return ""
-}
-
-// charsetReader returns a reader for the given charset. Currently
-// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful
-// error which is printed by go get, so the user can find why the package
-// wasn't downloaded if the encoding is not supported. Note that, in
-// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters
-// greater than 0x7f are not rejected).
-func charsetReader(charset string, input io.Reader) (io.Reader, error) {
-	switch strings.ToLower(charset) {
-	case "ascii":
-		return input, nil
-	default:
-		return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
-	}
-}

src/cmd/go/internal/modfetch/github/fetch.go

-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package github
-
-import (
-	"fmt"
-	"strings"
-
-	"cmd/go/internal/modfetch/codehost"
-	"cmd/go/internal/modfetch/gitrepo"
-)
-
-// Lookup returns the code repository enclosing the given module path,
-// which must begin with github.com/.
-func Lookup(path string) (codehost.Repo, error) {
-	f := strings.Split(path, "/")
-	if len(f) < 3 || f[0] != "github.com" {
-		return nil, fmt.Errorf("github repo must be github.com/org/project")
-	}
-	path = f[0] + "/" + f[1] + "/" + f[2]
-	return gitrepo.Repo("https://"+path, path)
-}

src/cmd/go/internal/modfetch/googlesource/fetch.go

-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package googlesource
-
-import (
-	"fmt"
-	"strings"
-
-	"cmd/go/internal/modfetch/codehost"
-	"cmd/go/internal/modfetch/gitrepo"
-)
-
-func Lookup(path string) (codehost.Repo, error) {
-	i := strings.Index(path, "/")
-	if i+1 == len(path) || !strings.HasSuffix(path[:i+1], ".googlesource.com/") {
-		return nil, fmt.Errorf("not *.googlesource.com/*")
-	}
-	j := strings.Index(path[i+1:], "/")
-	if j >= 0 {
-		path = path[:i+1+j]
-	}
-	return gitrepo.Repo("https://"+path, path)
-}

src/cmd/go/internal/modfetch/proxy.go

@@ -55,6 +55,7 @@ func (p *proxyRepo) Versions(prefix string) ([]string, error) {
 			list = append(list, f[0])
 		}
 	}
+	SortVersions(list)
 	return list, nil
 }
 
@@ -134,7 +135,7 @@ func (p *proxyRepo) Zip(version string, tmpdir string) (tmpfile string, err erro
 	defer body.Close()
 
 	// Spool to local file.
-	f, err := ioutil.TempFile(tmpdir, "vgo-proxy-download-")
+	f, err := ioutil.TempFile(tmpdir, "go-proxy-download-")
 	if err != nil {
 		return "", err
 	}

src/cmd/go/internal/modfile/testdata/replace2.golden

@@ -5,4 +5,6 @@ replace (
 	xyz v1.3.4 => my/xyz v1.3.4-me
 	xyz v1.4.5 => "/tmp/my dir"
 	xyz v1.5.6 => my/xyz v1.5.6
+
+	xyz => my/other/xyz v1.5.4
 )

src/cmd/go/internal/modfile/testdata/replace2.in

@@ -5,4 +5,6 @@ replace (
 	"xyz" v1.3.4 => "my/xyz" "v1.3.4-me"
 	xyz "v1.4.5" => "/tmp/my dir"
 	xyz v1.5.6 => my/xyz v1.5.6
+
+	xyz => my/other/xyz v1.5.4
 )

src/cmd/go/internal/web2/web.go

@@ -280,7 +280,7 @@ func Get(url string, options ...Option) error {
 	return err
 }
 
-var githubMessage = `vgo: 403 response from api.github.com
+var githubMessage = `go: 403 response from api.github.com
 
 GitHub applies fairly small rate limits to unauthenticated users, and
 you appear to be hitting them. To authenticate, please visit

src/cmd/go/internal/work/action.go

@@ -327,8 +327,8 @@ func (b *Builder) AutoAction(mode, depMode BuildMode, p *load.Package) *Action {
 // depMode is the action (build or install) to use when building dependencies.
 // To turn package main into an executable, call b.Link instead.
 func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Action {
-	if mode != ModeBuild && p.Internal.Local && p.Target == "" {
-		// Imported via local path. No permanent target.
+	if mode != ModeBuild && (p.Internal.Local || p.Module != nil) && p.Target == "" {
+		// Imported via local path or using modules. No permanent target.
 		mode = ModeBuild
 	}
 	if mode != ModeBuild && p.Name == "main" {

src/cmd/go/testdata/mod/example.com_v1.0.0.txt

+Written by hand.
+Test case for module at root of domain.
+
+-- .mod --
+module example.com
+-- .info --
+{"Version": "v1.0.0"}
+-- x.go --
+package x