cmd/go: merge module support from x/vgo repo

This CL corresponds to CL 123361, the final manual CL in that repo,
making this the final manual sync.

All future commits will happen in this repo (the main Go repo),
and we'll update x/vgo automatically with a fixed patch+script.

Change-Id: I572243309c1809727604fd704705a23c30e85d1a
Reviewed-on: https://go-review.googlesource.com/123576
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

src/cmd/go/go_test.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"debug/elf"
 	"debug/macho"
+	"flag"
 	"fmt"
 	"go/format"
 	"internal/race"
@@ -112,6 +113,12 @@ func TestMain(m *testing.M) {
 	}
 	os.Unsetenv("GOROOT_FINAL")
 
+	flag.Parse()
+	if *proxyAddr != "" {
+		StartProxy()
+		select {}
+	}
+
 	if canRun {
 		args := []string{"build", "-tags", "testgo", "-o", "testgo" + exeSuffix}
 		if race.Enabled {
@@ -417,7 +424,8 @@ func (tg *testgoData) doRun(args []string) error {
 func (tg *testgoData) run(args ...string) {
 	tg.t.Helper()
 	if status := tg.doRun(args); status != nil {
-		tg.t.Logf("go %v failed unexpectedly: %v", args, status)
+		wd, _ := os.Getwd()
+		tg.t.Logf("go %v failed unexpectedly in %s: %v", args, wd, status)
 		tg.t.FailNow()
 	}
 }
@@ -760,24 +768,51 @@ func (tg *testgoData) wantNotStale(pkg, reason, msg string) {
 	}
 }
 
+// If -testwork is specified, the test prints the name of the temp directory
+// and does not remove it when done, so that a programmer can
+// poke at the test file tree afterward.
+var testWork = flag.Bool("testwork", false, "")
+
 // cleanup cleans up a test that runs testgo.
 func (tg *testgoData) cleanup() {
 	tg.t.Helper()
 	if tg.wd != "" {
+		wd, _ := os.Getwd()
+		tg.t.Logf("ended in %s", wd)
+
 		if err := os.Chdir(tg.wd); err != nil {
 			// We are unlikely to be able to continue.
 			fmt.Fprintln(os.Stderr, "could not restore working directory, crashing:", err)
 			os.Exit(2)
 		}
 	}
+	if *testWork {
+		tg.t.Logf("TESTWORK=%s\n", tg.path("."))
+		return
+	}
 	for _, path := range tg.temps {
-		tg.check(os.RemoveAll(path))
+		tg.check(removeAll(path))
 	}
 	if tg.tempdir != "" {
-		tg.check(os.RemoveAll(tg.tempdir))
+		tg.check(removeAll(tg.tempdir))
 	}
 }
 
+func removeAll(dir string) error {
+	// module cache has 0444 directories;
+	// make them writable in order to remove content.
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil // ignore errors walking in file system
+		}
+		if info.IsDir() {
+			os.Chmod(path, 0777)
+		}
+		return nil
+	})
+	return os.RemoveAll(dir)
+}
+
 // failSSH puts an ssh executable in the PATH that always fails.
 // This is to stub out uses of ssh by go get.
 func (tg *testgoData) failSSH() {
@@ -1745,8 +1780,8 @@ func TestGoGetTestOnlyPkg(t *testing.T) {
 	defer tg.cleanup()
 	tg.tempDir("gopath")
 	tg.setenv("GOPATH", tg.path("gopath"))
-	tg.run("get", "golang.org/x/tour/content")
-	tg.run("get", "-t", "golang.org/x/tour/content")
+	tg.run("get", "golang.org/x/tour/content...")
+	tg.run("get", "-t", "golang.org/x/tour/content...")
 }
 
 func TestInstalls(t *testing.T) {
@@ -3165,11 +3200,25 @@ func TestBuildDashIInstallsDependencies(t *testing.T) {
 	checkbar("cmd")
 }
 
-func TestGoBuildInTestOnlyDirectoryFailsWithAGoodError(t *testing.T) {
+func TestGoBuildTestOnly(t *testing.T) {
 	tg := testgo(t)
 	defer tg.cleanup()
-	tg.runFail("build", "./testdata/testonly")
-	tg.grepStderr("no non-test Go files in", "go build ./testdata/testonly produced unexpected error")
+	tg.makeTempdir()
+	tg.setenv("GOPATH", tg.path("."))
+	tg.tempFile("src/testonly/t_test.go", `package testonly`)
+	tg.tempFile("src/testonly2/t.go", `package testonly2`)
+	tg.cd(tg.path("src"))
+
+	// Named explicitly, test-only packages should be reported as unbuildable/uninstallable,
+	// even if there is a wildcard also matching.
+	tg.runFail("build", "testonly", "testonly...")
+	tg.grepStderr("no non-test Go files in", "go build ./xtestonly produced unexpected error")
+	tg.runFail("install", "./testonly")
+	tg.grepStderr("no non-test Go files in", "go install ./testonly produced unexpected error")
+
+	// Named through a wildcards, the test-only packages should be silently ignored.
+	tg.run("build", "testonly...")
+	tg.run("install", "./testonly...")
 }
 
 func TestGoTestDetectsTestOnlyImportCycles(t *testing.T) {
@@ -6115,12 +6164,12 @@ func TestBadCommandLines(t *testing.T) {
 	tg.tempFile("src/@x/x.go", "package x\n")
 	tg.setenv("GOPATH", tg.path("."))
 	tg.runFail("build", "@x")
-	tg.grepStderr("invalid input directory name \"@x\"", "did not reject @x directory")
+	tg.grepStderr("invalid input directory name \"@x\"|cannot use path@version syntax", "did not reject @x directory")
 
 	tg.tempFile("src/@x/y/y.go", "package y\n")
 	tg.setenv("GOPATH", tg.path("."))
 	tg.runFail("build", "@x/y")
-	tg.grepStderr("invalid import path \"@x/y\"", "did not reject @x/y import path")
+	tg.grepStderr("invalid import path \"@x/y\"|cannot use path@version syntax", "did not reject @x/y import path")
 
 	tg.tempFile("src/-x/x.go", "package x\n")
 	tg.setenv("GOPATH", tg.path("."))

src/cmd/go/internal/cfg/cfg.go

@@ -21,6 +21,7 @@ var (
 	BuildA                 bool   // -a flag
 	BuildBuildmode         string // -buildmode flag
 	BuildContext           = build.Default
+	BuildGetmode           string             // -getmode flag
 	BuildI                 bool               // -i flag
 	BuildLinkshared        bool               // -linkshared flag
 	BuildMSan              bool               // -msan flag
@@ -67,6 +68,11 @@ var (
 	Goos      = BuildContext.GOOS
 	ExeSuffix string
 	Gopath    = filepath.SplitList(BuildContext.GOPATH)
+
+	// ModulesEnabled specifies whether the go command is running
+	// in module-aware mode (as opposed to GOPATH mode).
+	// It is equal to modload.Enabled, but not all packages can import modload.
+	ModulesEnabled bool
 )
 
 func init() {

src/cmd/go/internal/envcmd/env.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strings"
 
@@ -16,7 +17,7 @@ import (
 	"cmd/go/internal/cache"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
-	"cmd/go/internal/vgo"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/work"
 )
 
@@ -112,6 +113,18 @@ func findEnv(env []cfg.EnvVar, name string) string {
 
 // ExtraEnvVars returns environment variables that should not leak into child processes.
 func ExtraEnvVars() []cfg.EnvVar {
+	gomod := ""
+	if modload.Init(); modload.ModRoot != "" {
+		gomod = filepath.Join(modload.ModRoot, "go.mod")
+	}
+	return []cfg.EnvVar{
+		{Name: "GOMOD", Value: gomod},
+	}
+}
+
+// ExtraEnvVarsCostly returns environment variables that should not leak into child processes
+// but are costly to evaluate.
+func ExtraEnvVarsCostly() []cfg.EnvVar {
 	var b work.Builder
 	b.Init()
 	cppflags, cflags, cxxflags, fflags, ldflags, err := b.CFlags(&load.Package{})
@@ -121,6 +134,7 @@ func ExtraEnvVars() []cfg.EnvVar {
 		return nil
 	}
 	cmd := b.GccCmd(".", "")
+
 	return []cfg.EnvVar{
 		// Note: Update the switch in runEnv below when adding to this list.
 		{Name: "CGO_CFLAGS", Value: strings.Join(cflags, " ")},
@@ -130,19 +144,19 @@ func ExtraEnvVars() []cfg.EnvVar {
 		{Name: "CGO_LDFLAGS", Value: strings.Join(ldflags, " ")},
 		{Name: "PKG_CONFIG", Value: b.PkgconfigCmd()},
 		{Name: "GOGCCFLAGS", Value: strings.Join(cmd[3:], " ")},
-		{Name: "VGOMODROOT", Value: vgo.ModRoot},
 	}
 }
 
 func runEnv(cmd *base.Command, args []string) {
 	env := cfg.CmdEnv
+	env = append(env, ExtraEnvVars()...)
 
-	// Do we need to call ExtraEnvVars, which is a bit expensive?
+	// Do we need to call ExtraEnvVarsCostly, which is a bit expensive?
 	// Only if we're listing all environment variables ("go env")
 	// or the variables being requested are in the extra list.
-	needExtra := true
+	needCostly := true
 	if len(args) > 0 {
-		needExtra = false
+		needCostly = false
 		for _, arg := range args {
 			switch arg {
 			case "CGO_CFLAGS",
@@ -152,12 +166,12 @@ func runEnv(cmd *base.Command, args []string) {
 				"CGO_LDFLAGS",
 				"PKG_CONFIG",
 				"GOGCCFLAGS":
-				needExtra = true
+				needCostly = true
 			}
 		}
 	}
-	if needExtra {
-		env = append(env, ExtraEnvVars()...)
+	if needCostly {
+		env = append(env, ExtraEnvVarsCostly()...)
 	}
 
 	if len(args) > 0 {

src/cmd/go/internal/fix/fix.go

@@ -9,8 +9,8 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/str"
-	"cmd/go/internal/vgo"
 	"fmt"
 	"os"
 )
@@ -34,9 +34,9 @@ See also: go fmt, go vet.
 func runFix(cmd *base.Command, args []string) {
 	printed := false
 	for _, pkg := range load.Packages(args) {
-		if vgo.Enabled() && !pkg.Module.Top {
+		if modload.Enabled() && !pkg.Module.Main {
 			if !printed {
-				fmt.Fprintf(os.Stderr, "vgo: not fixing packages in dependency modules\n")
+				fmt.Fprintf(os.Stderr, "go: not fixing packages in dependency modules\n")
 				printed = true
 			}
 			continue

src/cmd/go/internal/fmtcmd/fmt.go

@@ -16,8 +16,8 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/str"
-	"cmd/go/internal/vgo"
 )
 
 func init() {
@@ -60,9 +60,9 @@ func runFmt(cmd *base.Command, args []string) {
 		}()
 	}
 	for _, pkg := range load.PackagesAndErrors(args) {
-		if vgo.Enabled() && !pkg.Module.Top {
+		if modload.Enabled() && !pkg.Module.Main {
 			if !printed {
-				fmt.Fprintf(os.Stderr, "vgo: not formatting packages in dependency modules\n")
+				fmt.Fprintf(os.Stderr, "go: not formatting packages in dependency modules\n")
 				printed = true
 			}
 			continue

src/cmd/go/internal/generate/generate.go

@@ -21,7 +21,7 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
-	"cmd/go/internal/vgo"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/work"
 )
 
@@ -161,9 +161,9 @@ func runGenerate(cmd *base.Command, args []string) {
 	// Even if the arguments are .go files, this loop suffices.
 	printed := false
 	for _, pkg := range load.Packages(args) {
-		if vgo.Enabled() && !pkg.Module.Top {
+		if modload.Enabled() && !pkg.Module.Main {
 			if !printed {
-				fmt.Fprintf(os.Stderr, "vgo: not generating in packages in dependency modules\n")
+				fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
 				printed = true
 			}
 			continue

src/cmd/go/internal/get/discovery.go

@@ -28,7 +28,7 @@ func charsetReader(charset string, input io.Reader) (io.Reader, error) {
 
 // parseMetaGoImports returns meta imports from the HTML in r.
 // Parsing ends at the end of the <head> section or the beginning of the <body>.
-func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
+func parseMetaGoImports(r io.Reader, mod ModuleMode) (imports []metaImport, err error) {
 	d := xml.NewDecoder(r)
 	d.CharsetReader = charsetReader
 	d.Strict = false
@@ -39,13 +39,13 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
 			if err == io.EOF || len(imports) > 0 {
 				err = nil
 			}
-			return
+			break
 		}
 		if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
-			return
+			break
 		}
 		if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
-			return
+			break
 		}
 		e, ok := t.(xml.StartElement)
 		if !ok || !strings.EqualFold(e.Name.Local, "meta") {
@@ -55,13 +55,6 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
 			continue
 		}
 		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
-			// Ignore VCS type "mod", which is new Go modules.
-			// This code is for old go get and must ignore the new mod lines.
-			// Otherwise matchGoImport will complain about two
-			// different metaImport lines for the same Prefix.
-			if f[1] == "mod" {
-				continue
-			}
 			imports = append(imports, metaImport{
 				Prefix:   f[0],
 				VCS:      f[1],
@@ -69,6 +62,27 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
 			})
 		}
 	}
+
+	// Extract mod entries if we are paying attention to them.
+	var list []metaImport
+	var have map[string]bool
+	if mod == PreferMod {
+		have = make(map[string]bool)
+		for _, m := range imports {
+			if m.VCS == "mod" {
+				have[m.Prefix] = true
+				list = append(list, m)
+			}
+		}
+	}
+
+	// Append non-mod entries, ignoring those superseded by a mod entry.
+	for _, m := range imports {
+		if m.VCS != "mod" && !have[m.Prefix] {
+			list = append(list, m)
+		}
+	}
+	return list, nil
 }
 
 // attrValue returns the attribute value for the case-insensitive key

src/cmd/go/internal/get/get.go

@@ -18,13 +18,12 @@ import (
 	"cmd/go/internal/load"
 	"cmd/go/internal/search"
 	"cmd/go/internal/str"
-	"cmd/go/internal/vgo"
 	"cmd/go/internal/web"
 	"cmd/go/internal/work"
 )
 
 var CmdGet = &base.Command{
-	UsageLine: "get [-d] [-f] [-fix] [-insecure] [-t] [-u] [-v] [build flags] [packages]",
+	UsageLine: "get [-d] [-f] [-t] [-u] [-v] [-fix] [-insecure] [build flags] [packages]",
 	Short:     "download and install packages and dependencies",
 	Long: `
 Get downloads the packages named by the import paths, along with their
@@ -75,25 +74,49 @@ For more about specifying packages, see 'go help packages'.
 For more about how 'go get' finds source code to
 download, see 'go help importpath'.
 
+This text describes the behavior of get when using GOPATH
+to manage source code and dependencies.
+If instead the go command is running in module-aware mode,
+the details of get's flags and effects change, as does 'go help get'.
+See 'go help modules' and 'go help module-get'.
+
 See also: go build, go install, go clean.
 	`,
 }
 
-var getD = CmdGet.Flag.Bool("d", false, "")
-var getF = CmdGet.Flag.Bool("f", false, "")
-var getT = CmdGet.Flag.Bool("t", false, "")
-var getU = CmdGet.Flag.Bool("u", false, "")
-var getFix = CmdGet.Flag.Bool("fix", false, "")
-var getInsecure = CmdGet.Flag.Bool("insecure", false, "")
+var HelpGopathGet = &base.Command{
+	UsageLine: "gopath-get",
+	Short:     "legacy GOPATH go get",
+	Long: `
+The 'go get' command changes behavior depending on whether the
+go command is running in module-aware mode or legacy GOPATH mode.
+This help text, accessible as 'go help gopath-get' even in module-aware mode,
+describes 'go get' as it operates in legacy GOPATH mode.
+
+Usage: ` + CmdGet.UsageLine + `
+` + CmdGet.Long,
+}
+
+var (
+	getD   = CmdGet.Flag.Bool("d", false, "")
+	getF   = CmdGet.Flag.Bool("f", false, "")
+	getT   = CmdGet.Flag.Bool("t", false, "")
+	getU   = CmdGet.Flag.Bool("u", false, "")
+	getFix = CmdGet.Flag.Bool("fix", false, "")
+
+	Insecure bool
+)
 
 func init() {
 	work.AddBuildFlags(CmdGet)
 	CmdGet.Run = runGet // break init loop
+	CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "")
 }
 
 func runGet(cmd *base.Command, args []string) {
-	if vgo.Enabled() {
-		base.Fatalf("go get: vgo not implemented")
+	if cfg.ModulesEnabled {
+		// Should not happen: main.go should install the separate module-enabled get code.
+		base.Fatalf("go get: modules not implemented")
 	}
 
 	work.BuildInit()
@@ -167,7 +190,7 @@ func runGet(cmd *base.Command, args []string) {
 		return
 	}
 
-	work.InstallPackages(args, true)
+	work.InstallPackages(args)
 }
 
 // downloadPaths prepares the list of paths to pass to download.
@@ -176,6 +199,12 @@ func runGet(cmd *base.Command, args []string) {
 // in the hope that we can figure out the repository from the
 // initial ...-free prefix.
 func downloadPaths(args []string) []string {
+	for _, arg := range args {
+		if strings.Contains(arg, "@") {
+			base.Fatalf("go: cannot use path@version syntax in GOPATH mode")
+		}
+	}
+
 	args = load.ImportPathsForGoGet(args)
 	var out []string
 	for _, a := range args {
@@ -379,7 +408,7 @@ func downloadPackage(p *load.Package) error {
 	)
 
 	security := web.Secure
-	if *getInsecure {
+	if Insecure {
 		security = web.Insecure
 	}
 
@@ -402,16 +431,16 @@ func downloadPackage(p *load.Package) error {
 			}
 			repo = remote
 			if !*getF && err == nil {
-				if rr, err := repoRootForImportPath(p.ImportPath, security); err == nil {
-					repo := rr.repo
+				if rr, err := RepoRootForImportPath(p.ImportPath, IgnoreMod, security); err == nil {
+					repo := rr.Repo
 					if rr.vcs.resolveRepo != nil {
 						resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo)
 						if err == nil {
 							repo = resolved
 						}
 					}
-					if remote != repo && rr.isCustom {
-						return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.root, repo, dir, remote)
+					if remote != repo && rr.IsCustom {
+						return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.Root, repo, dir, remote)
 					}
 				}
 			}
@@ -419,13 +448,13 @@ func downloadPackage(p *load.Package) error {
 	} else {
 		// Analyze the import path to determine the version control system,
 		// repository, and the import path for the root of the repository.
-		rr, err := repoRootForImportPath(p.ImportPath, security)
+		rr, err := RepoRootForImportPath(p.ImportPath, IgnoreMod, security)
 		if err != nil {
 			return err
 		}
-		vcs, repo, rootPath = rr.vcs, rr.repo, rr.root
+		vcs, repo, rootPath = rr.vcs, rr.Repo, rr.Root
 	}
-	if !blindRepo && !vcs.isSecure(repo) && !*getInsecure {
+	if !blindRepo && !vcs.isSecure(repo) && !Insecure {
 		return fmt.Errorf("cannot download, %v uses insecure protocol", repo)
 	}
 

src/cmd/go/internal/get/pkg_test.go

@@ -33,15 +33,18 @@ func TestFoldDup(t *testing.T) {
 
 var parseMetaGoImportsTests = []struct {
 	in  string
+	mod ModuleMode
 	out []metaImport
 }{
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
+		IgnoreMod,
 		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
 	},
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		<meta name="go-import" content="baz/quux git http://github.com/rsc/baz/quux">`,
+		IgnoreMod,
 		[]metaImport{
 			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
 			{"baz/quux", "git", "http://github.com/rsc/baz/quux"},
@@ -50,6 +53,7 @@ var parseMetaGoImportsTests = []struct {
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">`,
+		IgnoreMod,
 		[]metaImport{
 			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
 		},
@@ -57,35 +61,48 @@ var parseMetaGoImportsTests = []struct {
 	{
 		`<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">
 		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
+		IgnoreMod,
 		[]metaImport{
 			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
 		},
 	},
+	{
+		`<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">
+		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
+		PreferMod,
+		[]metaImport{
+			{"foo/bar", "mod", "http://github.com/rsc/baz/quux"},
+		},
+	},
 	{
 		`<head>
 		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		</head>`,
+		IgnoreMod,
 		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
 	},
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		<body>`,
+		IgnoreMod,
 		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
 	},
 	{
 		`<!doctype html><meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
+		IgnoreMod,
 		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
 	},
 	{
 		// XML doesn't like <div style=position:relative>.
 		`<!doctype html><title>Page Not Found</title><meta name=go-import content="chitin.io/chitin git https://github.com/chitin-io/chitin"><div style=position:relative>DRAFT</div>`,
+		IgnoreMod,
 		[]metaImport{{"chitin.io/chitin", "git", "https://github.com/chitin-io/chitin"}},
 	},
 }
 
 func TestParseMetaGoImports(t *testing.T) {
 	for i, tt := range parseMetaGoImportsTests {
-		out, err := parseMetaGoImports(strings.NewReader(tt.in))
+		out, err := parseMetaGoImports(strings.NewReader(tt.in), tt.mod)
 		if err != nil {
 			t.Errorf("test#%d: %v", i, err)
 			continue

src/cmd/go/internal/get/vcs.go

@@ -624,27 +624,29 @@ func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error {
 	return nil
 }
 
-// repoRoot represents a version control system, a repo, and a root of
-// where to put it on disk.
-type repoRoot struct {
-	vcs *vcsCmd
-
-	// repo is the repository URL, including scheme
-	repo string
-
-	// root is the import path corresponding to the root of the
-	// repository
-	root string
-
-	// isCustom is true for custom import paths (those defined by HTML meta tags)
-	isCustom bool
+// RepoRoot describes the repository root for a tree of source code.
+type RepoRoot struct {
+	Repo     string // repository URL, including scheme
+	Root     string // import path corresponding to root of repo
+	IsCustom bool   // defined by served <meta> tags (as opposed to hard-coded pattern)
+	VCS      string // vcs type ("mod", "git", ...)
+
+	vcs *vcsCmd // internal: vcs command access
 }
 
 var httpPrefixRE = regexp.MustCompile(`^https?:`)
 
-// repoRootForImportPath analyzes importPath to determine the
+// ModuleMode specifies whether to prefer modules when looking up code sources.
+type ModuleMode int
+
+const (
+	IgnoreMod ModuleMode = iota
+	PreferMod
+)
+
+// RepoRootForImportPath analyzes importPath to determine the
 // version control system, and code repository to use.
-func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoRoot, error) {
+func RepoRootForImportPath(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
 	rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths)
 	if err == errUnknownSite {
 		// If there are wildcards, look up the thing before the wildcard,
@@ -654,7 +656,7 @@ func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoR
 		if i := strings.Index(lookup, "/.../"); i >= 0 {
 			lookup = lookup[:i]
 		}
-		rr, err = repoRootForImportDynamic(lookup, security)
+		rr, err = repoRootForImportDynamic(lookup, mod, security)
 		if err != nil {
 			err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err)
 		}
@@ -667,7 +669,7 @@ func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoR
 		}
 	}
 
-	if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.root, "...") {
+	if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") {
 		// Do not allow wildcards in the repo root.
 		rr = nil
 		err = fmt.Errorf("cannot expand ... in %q", importPath)
@@ -680,7 +682,7 @@ var errUnknownSite = errors.New("dynamic lookup required to find mapping")
 // repoRootFromVCSPaths attempts to map importPath to a repoRoot
 // using the mappings defined in vcsPaths.
 // If scheme is non-empty, that scheme is forced.
-func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, vcsPaths []*vcsPath) (*repoRoot, error) {
+func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) {
 	// A common error is to use https://packagepath because that's what
 	// hg and git require. Diagnose this helpfully.
 	if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil {
@@ -733,28 +735,32 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode,
 					if security == web.Secure && !vcs.isSecureScheme(scheme) {
 						continue
 					}
-					if vcs.ping(scheme, match["repo"]) == nil {
+					if vcs.pingCmd != "" && vcs.ping(scheme, match["repo"]) == nil {
 						match["repo"] = scheme + "://" + match["repo"]
-						break
+						goto Found
 					}
 				}
+				// No scheme found. Fall back to the first one.
+				match["repo"] = vcs.scheme[0] + "://" + match["repo"]
+			Found:
 			}
 		}
-		rr := &repoRoot{
+		rr := &RepoRoot{
+			Repo: match["repo"],
+			Root: match["root"],
+			VCS:  vcs.cmd,
 			vcs:  vcs,
-			repo: match["repo"],
-			root: match["root"],
 		}
 		return rr, nil
 	}
 	return nil, errUnknownSite
 }
 
-// repoRootForImportDynamic finds a *repoRoot for a custom domain that's not
+// repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not
 // statically known by repoRootForImportPathStatic.
 //
 // This handles custom import paths like "name.tld/pkg/foo" or just "name.tld".
-func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*repoRoot, error) {
+func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
 	slash := strings.Index(importPath, "/")
 	if slash < 0 {
 		slash = len(importPath)
@@ -772,7 +778,7 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re
 		return nil, fmt.Errorf(msg, err)
 	}
 	defer body.Close()
-	imports, err := parseMetaGoImports(body)
+	imports, err := parseMetaGoImports(body, mod)
 	if err != nil {
 		return nil, fmt.Errorf("parsing %s: %v", importPath, err)
 	}
@@ -799,7 +805,7 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re
 		}
 		urlStr0 := urlStr
 		var imports []metaImport
-		urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, security)
+		urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security)
 		if err != nil {
 			return nil, err
 		}
@@ -812,15 +818,18 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re
 	if err := validateRepoRoot(mmi.RepoRoot); err != nil {
 		return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, mmi.RepoRoot, err)
 	}
-	rr := &repoRoot{
-		vcs:      vcsByCmd(mmi.VCS),
-		repo:     mmi.RepoRoot,
-		root:     mmi.Prefix,
-		isCustom: true,
-	}
-	if rr.vcs == nil {
+	vcs := vcsByCmd(mmi.VCS)
+	if vcs == nil && mmi.VCS != "mod" {
 		return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS)
 	}
+
+	rr := &RepoRoot{
+		Repo:     mmi.RepoRoot,
+		Root:     mmi.Prefix,
+		IsCustom: true,
+		VCS:      mmi.VCS,
+		vcs:      vcs,
+	}
 	return rr, nil
 }
 
@@ -851,7 +860,7 @@ var (
 // It is an error if no imports are found.
 // urlStr will still be valid if err != nil.
 // The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1"
-func metaImportsForPrefix(importPrefix string, security web.SecurityMode) (urlStr string, imports []metaImport, err error) {
+func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (urlStr string, imports []metaImport, err error) {
 	setCache := func(res fetchResult) (fetchResult, error) {
 		fetchCacheMu.Lock()
 		defer fetchCacheMu.Unlock()
@@ -871,7 +880,7 @@ func metaImportsForPrefix(importPrefix string, security web.SecurityMode) (urlSt
 		if err != nil {
 			return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)})
 		}
-		imports, err := parseMetaGoImports(body)
+		imports, err := parseMetaGoImports(body, mod)
 		if err != nil {
 			return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)})
 		}

src/cmd/go/internal/get/vcs_test.go

@@ -16,43 +16,43 @@ import (
 	"cmd/go/internal/web"
 )
 
-// Test that RepoRootForImportPath creates the correct RepoRoot for a given importPath.
+// Test that RepoRootForImportPath determines the correct RepoRoot for a given importPath.
 // TODO(cmang): Add tests for SVN and BZR.
 func TestRepoRootForImportPath(t *testing.T) {
 	testenv.MustHaveExternalNetwork(t)
 
 	tests := []struct {
 		path string
-		want *repoRoot
+		want *RepoRoot
 	}{
 		{
 			"github.com/golang/groupcache",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://github.com/golang/groupcache",
+				Repo: "https://github.com/golang/groupcache",
 			},
 		},
 		// Unicode letters in directories (issue 18660).
 		{
 			"github.com/user/unicode/испытание",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://github.com/user/unicode",
+				Repo: "https://github.com/user/unicode",
 			},
 		},
 		// IBM DevOps Services tests
 		{
 			"hub.jazz.net/git/user1/pkgname",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://hub.jazz.net/git/user1/pkgname",
+				Repo: "https://hub.jazz.net/git/user1/pkgname",
 			},
 		},
 		{
 			"hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://hub.jazz.net/git/user1/pkgname",
+				Repo: "https://hub.jazz.net/git/user1/pkgname",
 			},
 		},
 		{
@@ -91,9 +91,9 @@ func TestRepoRootForImportPath(t *testing.T) {
 		},
 		{
 			"hub.jazz.net/git/user/pkg.name",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://hub.jazz.net/git/user/pkg.name",
+				Repo: "https://hub.jazz.net/git/user/pkg.name",
 			},
 		},
 		// User names cannot have uppercase letters
@@ -104,9 +104,9 @@ func TestRepoRootForImportPath(t *testing.T) {
 		// OpenStack tests
 		{
 			"git.openstack.org/openstack/swift",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://git.openstack.org/openstack/swift",
+				Repo: "https://git.openstack.org/openstack/swift",
 			},
 		},
 		// Trailing .git is less preferred but included for
@@ -114,16 +114,16 @@ func TestRepoRootForImportPath(t *testing.T) {
 		// be compilable on both old and new go
 		{
 			"git.openstack.org/openstack/swift.git",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://git.openstack.org/openstack/swift.git",
+				Repo: "https://git.openstack.org/openstack/swift.git",
 			},
 		},
 		{
 			"git.openstack.org/openstack/swift/go/hummingbird",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://git.openstack.org/openstack/swift",
+				Repo: "https://git.openstack.org/openstack/swift",
 			},
 		},
 		{
@@ -150,23 +150,23 @@ func TestRepoRootForImportPath(t *testing.T) {
 		},
 		{
 			"git.apache.org/package-name.git",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://git.apache.org/package-name.git",
+				Repo: "https://git.apache.org/package-name.git",
 			},
 		},
 		{
 			"git.apache.org/package-name_2.x.git/path/to/lib",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsGit,
-				repo: "https://git.apache.org/package-name_2.x.git",
+				Repo: "https://git.apache.org/package-name_2.x.git",
 			},
 		},
 		{
 			"chiselapp.com/user/kyle/repository/fossilgg",
-			&repoRoot{
+			&RepoRoot{
 				vcs:  vcsFossil,
-				repo: "https://chiselapp.com/user/kyle/repository/fossilgg",
+				Repo: "https://chiselapp.com/user/kyle/repository/fossilgg",
 			},
 		},
 		{
@@ -181,21 +181,21 @@ func TestRepoRootForImportPath(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		got, err := repoRootForImportPath(test.path, web.Secure)
+		got, err := RepoRootForImportPath(test.path, IgnoreMod, web.Secure)
 		want := test.want
 
 		if want == nil {
 			if err == nil {
-				t.Errorf("repoRootForImportPath(%q): Error expected but not received", test.path)
+				t.Errorf("RepoRootForImportPath(%q): Error expected but not received", test.path)
 			}
 			continue
 		}
 		if err != nil {
-			t.Errorf("repoRootForImportPath(%q): %v", test.path, err)
+			t.Errorf("RepoRootForImportPath(%q): %v", test.path, err)
 			continue
 		}
-		if got.vcs.name != want.vcs.name || got.repo != want.repo {
-			t.Errorf("repoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.repo, want.vcs, want.repo)
+		if got.vcs.name != want.vcs.name || got.Repo != want.Repo {
+			t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.Repo, want.vcs, want.Repo)
 		}
 	}
 }
@@ -227,18 +227,18 @@ func TestFromDir(t *testing.T) {
 			f.Close()
 		}
 
-		want := repoRoot{
+		want := RepoRoot{
 			vcs:  vcs,
-			root: path.Join("example.com", vcs.name),
+			Root: path.Join("example.com", vcs.name),
 		}
-		var got repoRoot
-		got.vcs, got.root, err = vcsFromDir(dir, tempDir)
+		var got RepoRoot
+		got.vcs, got.Root, err = vcsFromDir(dir, tempDir)
 		if err != nil {
 			t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err)
 			continue
 		}
-		if got.vcs.name != want.vcs.name || got.root != want.root {
-			t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.root, want.vcs, want.root)
+		if got.vcs.name != want.vcs.name || got.Root != want.Root {
+			t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.Root, want.vcs, want.Root)
 		}
 	}
 }

src/cmd/go/internal/help/help.go

@@ -45,7 +45,15 @@ func Help(args []string) {
 		buf := new(bytes.Buffer)
 		PrintUsage(buf)
 		usage := &base.Command{Long: buf.String()}
-		tmpl(&commentWriter{W: os.Stdout}, documentationTemplate, append([]*base.Command{usage}, base.Commands...))
+		cmds := []*base.Command{usage}
+		for _, cmd := range base.Commands {
+			if cmd.UsageLine == "gopath-get" {
+				// Avoid duplication of the "get" documentation.
+				continue
+			}
+			cmds = append(cmds, cmd)
+		}
+		tmpl(&commentWriter{W: os.Stdout}, documentationTemplate, cmds)
 		fmt.Println("package main")
 		return
 	}

src/cmd/go/internal/help/helpdoc.go

@@ -30,7 +30,7 @@ the C or C++ compiler, respectively, to use.
 
 var HelpPackages = &base.Command{
 	UsageLine: "packages",
-	Short:     "package lists",
+	Short:     "package lists and patterns",
 	Long: `
 Many commands apply to a set of packages:
 
@@ -54,9 +54,11 @@ for packages to be built with the go tool:
 
 - "main" denotes the top-level package in a stand-alone executable.
 
-- "all" expands to all package directories found in all the GOPATH
+- "all" expands to all packages found in all the GOPATH
 trees. For example, 'go list all' lists all the packages on the local
-system.
+system. When using modules, "all" expands to all packages in
+the main module and their dependencies, including dependencies
+needed by tests of any of those.
 
 - "std" is like all but expands to just the packages in the standard
 Go library.

src/cmd/go/internal/list/list.go

@@ -19,14 +19,20 @@ import (
 	"cmd/go/internal/cache"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/work"
 )
 
 var CmdList = &base.Command{
-	UsageLine: "list [-cgo] [-deps] [-e] [-export] [-f format] [-json] [-test] [build flags] [packages]",
-	Short:     "list packages",
+	// Note: -f -json -m are listed explicitly because they are the most common list flags.
+	// Do not send CLs removing them because they're covered by [list flags].
+	UsageLine: "list [-f format] [-json] [-m] [list flags] [build flags] [packages]",
+	Short:     "list packages or modules",
 	Long: `
-List lists the packages named by the import paths, one per line.
+List lists the named packages, one per line.
+The most commonly-used flags are -f and -json, which control the form
+of the output printed for each package. Other list flags, documented below,
+control more specific details.
 
 The default output shows the package import path:
 
@@ -36,8 +42,8 @@ The default output shows the package import path:
     golang.org/x/net/html
 
 The -f flag specifies an alternate format for the list, using the
-syntax of package template. The default output is equivalent to -f
-'{{.ImportPath}}'. The struct being passed to the template is:
+syntax of package template. The default output is equivalent
+to -f '{{.ImportPath}}'. The struct being passed to the template is:
 
     type Package struct {
         Dir           string  // directory containing package sources
@@ -57,6 +63,7 @@ syntax of package template. The default output is equivalent to -f
         ForTest       string  // package is only for use in named test
         DepOnly       bool    // package is only a dependency, not explicitly listed
         Export        string  // file containing export data (when using -export)
+        Module        *Module // info about package's containing module, if any (can be nil)
 
         // Source files
         GoFiles        []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
@@ -109,6 +116,9 @@ The error information, if any, is
         Err           string   // the error itself
     }
 
+The module information is a Module struct, defined in the discussion
+of list -m below.
+
 The template function "join" calls strings.Join.
 
 The template function "context" returns the build context, defined as:
@@ -178,9 +188,90 @@ The extra entries added by the -cgo and -test flags are absolute paths
 referring to cached copies of generated Go source files.
 Although they are Go source files, the paths may not end in ".go".
 
+The -m flag causes list to list modules instead of packages.
+
+When listing modules, the -f flag still specifies a format template
+applied to a Go struct, but now a Module struct:
+
+    type Module struct {
+        Path     string       // module path
+        Version  string       // module version
+        Versions []string     // available module versions (with -versions)
+        Replace  *Module      // replaced by this module
+        Time     *time.Time   // time version was created
+        Update   *Module      // available update, if any (with -u)
+        Main     bool         // is this the main module?
+        Indirect bool         // is this module only an indirect dependency of main module?
+        Dir      string       // directory holding files for this module, if any
+        Error    *ModuleError // error loading module
+    }
+
+    type ModuleError struct {
+        Err string // the error itself
+    }
+
+The default output is to print the module path and then
+information about the version and replacement if any.
+For example, 'go list -m all' might print:
+
+    my/main/module
+    golang.org/x/text v0.3.0 => /tmp/text
+    rsc.io/pdf v0.1.1
+
+The Module struct has a String method that formats this
+line of output, so that the default format is equivalent
+to -f '{{.String}}'.
+
+Note that when a module has been replaced, its Replace field
+describes the replacement module, and its Dir field is set to
+the replacement's source code, if present. (That is, if Replace
+is non-nil, then Dir is set to Replace.Dir, with no access to
+the replaced source code.)
+
+The -u flag adds information about available upgrades.
+When the latest version of a given module is newer than
+the current one, list -u sets the Module's Update field
+to information about the newer module.
+The Module's String method indicates an available upgrade by
+formatting the newer version in brackets after the current version.
+For example, 'go list -m -u all' might print:
+
+    my/main/module
+    golang.org/x/text v0.3.0 [v0.4.0] => /tmp/text
+    rsc.io/pdf v0.1.1 [v0.1.2]
+
+(For tools, 'go list -m -u -json all' may be more convenient to parse.)
+
+The -versions flag causes list to set the Module's Versions field
+to a list of all known versions of that module, ordered according
+to semantic versioning, earliest to latest. The flag also changes
+the default output format to display the module path followed by the
+space-separated version list.
+
+The arguments to list -m are interpreted as a list of modules, not packages.
+The main module is the module containing the current directory.
+The active modules are the main module and its dependencies.
+With no arguments, list -m shows the main module.
+With arguments, list -m shows the modules specified by the arguments.
+Any of the active modules can be specified by its module path.
+The special pattern "all" specifies all the active modules, first the main
+module and then dependencies sorted by module path.
+A pattern containing "..." specifies the active modules whose
+module paths match the pattern.
+A query of the form path@version specifies the result of that query,
+which is not limited to active modules.
+See 'go help module' for more about module queries.
+
+The template function "module" takes a single string argument
+that must be a module path or query and returns the specified
+module as a Module struct. If an error occurs, the result will
+be a Module struct with a non-nil Error field.
+
 For more about build flags, see 'go help build'.
 
 For more about specifying packages, see 'go help packages'.
+
+For more about modules, see 'go help modules'.
 	`,
 }
 
@@ -189,13 +280,19 @@ func init() {
 	work.AddBuildFlags(CmdList)
 }
 
-var listCgo = CmdList.Flag.Bool("cgo", false, "")
-var listDeps = CmdList.Flag.Bool("deps", false, "")
-var listE = CmdList.Flag.Bool("e", false, "")
-var listExport = CmdList.Flag.Bool("export", false, "")
-var listFmt = CmdList.Flag.String("f", "{{.ImportPath}}", "")
-var listJson = CmdList.Flag.Bool("json", false, "")
-var listTest = CmdList.Flag.Bool("test", false, "")
+var (
+	listCgo      = CmdList.Flag.Bool("cgo", false, "")
+	listDeps     = CmdList.Flag.Bool("deps", false, "")
+	listE        = CmdList.Flag.Bool("e", false, "")
+	listExport   = CmdList.Flag.Bool("export", false, "")
+	listFmt      = CmdList.Flag.String("f", "", "")
+	listJson     = CmdList.Flag.Bool("json", false, "")
+	listM        = CmdList.Flag.Bool("m", false, "")
+	listU        = CmdList.Flag.Bool("u", false, "")
+	listTest     = CmdList.Flag.Bool("test", false, "")
+	listVersions = CmdList.Flag.Bool("versions", false, "")
+)
+
 var nl = []byte{'\n'}
 
 func runList(cmd *base.Command, args []string) {
@@ -203,10 +300,21 @@ func runList(cmd *base.Command, args []string) {
 	out := newTrackingWriter(os.Stdout)
 	defer out.w.Flush()
 
-	var do func(*load.PackagePublic)
+	if *listFmt == "" {
+		if *listM {
+			*listFmt = "{{.String}}"
+			if *listVersions {
+				*listFmt = `{{.Path}}{{range .Versions}} {{.}}{{end}}`
+			}
+		} else {
+			*listFmt = "{{.ImportPath}}"
+		}
+	}
+
+	var do func(interface{})
 	if *listJson {
-		do = func(p *load.PackagePublic) {
-			b, err := json.MarshalIndent(p, "", "\t")
+		do = func(x interface{}) {
+			b, err := json.MarshalIndent(x, "", "\t")
 			if err != nil {
 				out.Flush()
 				base.Fatalf("%s", err)
@@ -225,13 +333,14 @@ func runList(cmd *base.Command, args []string) {
 		fm := template.FuncMap{
 			"join":    strings.Join,
 			"context": context,
+			"module":  modload.ModuleInfo,
 		}
 		tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt)
 		if err != nil {
 			base.Fatalf("%s", err)
 		}
-		do = func(p *load.PackagePublic) {
-			if err := tmpl.Execute(out, p); err != nil {
+		do = func(x interface{}) {
+			if err := tmpl.Execute(out, x); err != nil {
 				out.Flush()
 				base.Fatalf("%s", err)
 			}
@@ -241,6 +350,50 @@ func runList(cmd *base.Command, args []string) {
 		}
 	}
 
+	if *listM {
+		// Module mode.
+		if *listCgo {
+			base.Fatalf("go list -cgo cannot be used with -m")
+		}
+		if *listDeps {
+			// TODO(rsc): Could make this mean something with -m.
+			base.Fatalf("go list -deps cannot be used with -m")
+		}
+		if *listExport {
+			base.Fatalf("go list -export cannot be used with -m")
+		}
+		if *listTest {
+			base.Fatalf("go list -test cannot be used with -m")
+		}
+
+		if modload.Init(); !modload.Enabled() {
+			base.Fatalf("go list -m: not using modules")
+		}
+		modload.LoadBuildList()
+
+		mods := modload.ListModules(args, *listU, *listVersions)
+		if !*listE {
+			for _, m := range mods {
+				if m.Error != nil {
+					base.Errorf("go list -m %s: %v", m.Path, m.Error.Err)
+				}
+			}
+			base.ExitIfErrors()
+		}
+		for _, m := range mods {
+			do(m)
+		}
+		return
+	}
+
+	// Package mode (not -m).
+	if *listU {
+		base.Fatalf("go list -u can only be used with -m")
+	}
+	if *listVersions {
+		base.Fatalf("go list -versions can only be used with -m")
+	}
+
 	var pkgs []*load.Package
 	if *listE {
 		pkgs = load.PackagesAndErrors(args)

src/cmd/go/internal/load/flag.go

@@ -6,6 +6,7 @@ package load
 
 import (
 	"cmd/go/internal/base"
+	"cmd/go/internal/search"
 	"cmd/go/internal/str"
 	"fmt"
 	"strings"
@@ -92,7 +93,10 @@ func (f *PerPackageFlag) For(p *Package) []string {
 	return flags
 }
 
-var cmdlineMatchers []func(*Package) bool
+var (
+	cmdlineMatchers        []func(*Package) bool
+	cmdlineMatcherLiterals []func(*Package) bool
+)
 
 // SetCmdlinePatterns records the set of patterns given on the command line,
 // for use by the PerPackageFlags.
@@ -105,9 +109,15 @@ func setCmdlinePatterns(args []string, cwd string) {
 		args = []string{"."}
 	}
 	cmdlineMatchers = nil // allow reset for testing
+	cmdlineMatcherLiterals = nil
 	for _, arg := range args {
 		cmdlineMatchers = append(cmdlineMatchers, MatchPackage(arg, cwd))
 	}
+	for _, arg := range args {
+		if !strings.Contains(arg, "...") && !search.IsMetaPackage(arg) {
+			cmdlineMatcherLiterals = append(cmdlineMatcherLiterals, MatchPackage(arg, cwd))
+		}
+	}
 }
 
 // isCmdlinePkg reports whether p is a package listed on the command line.
@@ -119,3 +129,15 @@ func isCmdlinePkg(p *Package) bool {
 	}
 	return false
 }
+
+// isCmdlinePkgLiteral reports whether p is a package listed as
+// a literal package argument on the command line
+// (as opposed to being the result of expanding a wildcard).
+func isCmdlinePkgLiteral(p *Package) bool {
+	for _, m := range cmdlineMatcherLiterals {
+		if m(p) {
+			return true
+		}
+	}
+	return false
+}

src/cmd/go/internal/load/pkg.go

@@ -25,7 +25,17 @@ import (
 	"cmd/go/internal/modinfo"
 	"cmd/go/internal/search"
 	"cmd/go/internal/str"
-	"cmd/go/internal/vgo"
+)
+
+var (
+	// module hooks; nil if module use is disabled
+	ModBinDir            func() string                                                   // return effective bin directory
+	ModLookup            func(parentPath, path string) (dir, realPath string, err error) // lookup effective meaning of import
+	ModPackageModuleInfo func(path string) *modinfo.ModulePublic                         // return module info for Package struct
+	ModImportPaths       func(args []string) []string                                    // expand import paths
+	ModPackageBuildInfo  func(main string, deps []string) string                         // return module info to embed in binary
+	ModInfoProg          func(info string) []byte                                        // wrap module info in .go code for binary
+	ModImportFromFiles   func([]string)                                                  // update go.mod to add modules for imports in these files
 )
 
 var IgnoreImports bool // control whether we ignore imports in packages
@@ -55,6 +65,7 @@ type PackagePublic struct {
 	ForTest       string                `json:",omitempty"` // package is only for use in named test
 	DepOnly       bool                  `json:",omitempty"` // package is only as a dependency, not explicitly listed
 	Export        string                `json:",omitempty"` // file containing export data (set by go list -export)
+	Module        *modinfo.ModulePublic `json:",omitempty"` // info about package's module, if any
 
 	// Stale and StaleReason remain here *only* for the list command.
 	// They are only initialized in preparation for list execution.
@@ -103,8 +114,6 @@ type PackagePublic struct {
 	TestImports  []string `json:",omitempty"` // imports from TestGoFiles
 	XTestGoFiles []string `json:",omitempty"` // _test.go files outside package
 	XTestImports []string `json:",omitempty"` // imports from XTestGoFiles
-
-	Module *modinfo.ModulePublic `json:",omitempty"` // info about package module
 }
 
 // AllFiles returns the names of all the files considered for the package.
@@ -147,6 +156,7 @@ type PackageInternal struct {
 	ForceLibrary      bool                 // this package is a library (even if named "main")
 	CmdlineFiles      bool                 // package built from files listed on command line
 	CmdlinePkg        bool                 // package listed on command line
+	CmdlinePkgLiteral bool                 // package listed as literal on command line (not via wildcard)
 	Local             bool                 // imported via local path (./ or ../)
 	LocalPrefix       string               // interpret ./ and ../ imports relative to this prefix
 	ExeName           string               // desired name for temporary executable
@@ -416,6 +426,41 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 	stk.Push(path)
 	defer stk.Pop()
 
+	if strings.HasPrefix(path, "mod/") {
+		// Paths beginning with "mod/" might accidentally
+		// look in the module cache directory tree in $GOPATH/src/mod/.
+		// This prefix is owned by the Go core for possible use in the
+		// standard library (since it does not begin with a domain name),
+		// so it's OK to disallow entirely.
+		return &Package{
+			PackagePublic: PackagePublic{
+				ImportPath: path,
+				Error: &PackageError{
+					ImportStack: stk.Copy(),
+					Err:         fmt.Sprintf("disallowed import path %q", path),
+				},
+			},
+		}
+	}
+
+	if strings.Contains(path, "@") {
+		var text string
+		if cfg.ModulesEnabled {
+			text = "can only use path@version syntax with 'go get'"
+		} else {
+			text = "cannot use path@version syntax in GOPATH mode"
+		}
+		return &Package{
+			PackagePublic: PackagePublic{
+				ImportPath: path,
+				Error: &PackageError{
+					ImportStack: stk.Copy(),
+					Err:         text,
+				},
+			},
+		}
+	}
+
 	// Determine canonical identifier for this package.
 	// For a local import the identifier is the pseudo-import path
 	// we create from the full directory to the package.
@@ -424,18 +469,18 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 	importPath := path
 	origPath := path
 	isLocal := build.IsLocalImport(path)
-	var vgoDir string
-	var vgoErr error
+	var modDir string
+	var modErr error
 	if isLocal {
 		importPath = dirToImportPath(filepath.Join(srcDir, path))
-	} else if vgo.Enabled() {
+	} else if cfg.ModulesEnabled {
 		parentPath := ""
 		if parent != nil {
 			parentPath = parent.ImportPath
 		}
 		var p string
-		vgoDir, p, vgoErr = vgo.Lookup(parentPath, path)
-		if vgoErr == nil {
+		modDir, p, modErr = ModLookup(parentPath, path)
+		if modErr == nil {
 			importPath = p
 		}
 	} else if mode&ResolveImport != 0 {
@@ -464,11 +509,14 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 		// in order to return partial information.
 		var bp *build.Package
 		var err error
-		if vgoDir != "" {
-			bp, err = cfg.BuildContext.ImportDir(vgoDir, 0)
-		} else if vgoErr != nil {
+		if modDir != "" {
+			bp, err = cfg.BuildContext.ImportDir(modDir, 0)
+		} else if modErr != nil {
+			bp = new(build.Package)
+			err = fmt.Errorf("unknown import path %q: %v", importPath, modErr)
+		} else if cfg.ModulesEnabled && path != "unsafe" {
 			bp = new(build.Package)
-			err = fmt.Errorf("unknown import path %q: %v", importPath, vgoErr)
+			err = fmt.Errorf("unknown import path %q: internal error: module loader did not resolve import", importPath)
 		} else {
 			buildMode := build.ImportComment
 			if mode&ResolveImport == 0 || path != origPath {
@@ -480,10 +528,10 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 		bp.ImportPath = importPath
 		if cfg.GOBIN != "" {
 			bp.BinDir = cfg.GOBIN
-		} else if vgo.Enabled() {
-			bp.BinDir = vgo.BinDir()
+		} else if cfg.ModulesEnabled {
+			bp.BinDir = ModBinDir()
 		}
-		if vgoDir == "" && err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path &&
+		if modDir == "" && err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path &&
 			!strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/") {
 			err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment)
 		}
@@ -492,7 +540,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 			p = setErrorPos(p, importPos)
 		}
 
-		if vgoDir == "" && origPath != cleanImport(origPath) {
+		if modDir == "" && origPath != cleanImport(origPath) {
 			p.Error = &PackageError{
 				ImportStack: stk.Copy(),
 				Err:         fmt.Sprintf("non-canonical import path: %q should be %q", origPath, pathpkg.Clean(origPath)),
@@ -568,14 +616,14 @@ func isDir(path string) bool {
 // There are two different resolutions applied.
 // First, there is Go 1.5 vendoring (golang.org/s/go15vendor).
 // If vendor expansion doesn't trigger, then the path is also subject to
-// Go 1.11 vgo legacy conversion (golang.org/issue/25069).
+// Go 1.11 module legacy conversion (golang.org/issue/25069).
 func ResolveImportPath(parent *Package, path string) (found string) {
-	if vgo.Enabled() {
+	if cfg.ModulesEnabled {
 		parentPath := ""
 		if parent != nil {
 			parentPath = parent.ImportPath
 		}
-		if _, p, e := vgo.Lookup(parentPath, path); e == nil {
+		if _, p, e := ModLookup(parentPath, path); e == nil {
 			return p
 		}
 		return path
@@ -1108,8 +1156,9 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
 	// of fmt before it attempts to load as a command-line argument.
 	// Because loads are cached, the later load will be a no-op,
 	// so it is important that the first load can fill in CmdlinePkg correctly.
-	// Hence the call to an explicit matching check here.
+	// Hence the call to a separate matching check here.
 	p.Internal.CmdlinePkg = isCmdlinePkg(p)
+	p.Internal.CmdlinePkgLiteral = isCmdlinePkgLiteral(p)
 
 	p.Internal.Asmflags = BuildAsmflags.For(p)
 	p.Internal.Gcflags = BuildGcflags.For(p)
@@ -1157,8 +1206,8 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
 			// Install cross-compiled binaries to subdirectories of bin.
 			elem = full
 		}
-		if p.Internal.Build.BinDir == "" && vgo.Enabled() {
-			p.Internal.Build.BinDir = vgo.BinDir()
+		if p.Internal.Build.BinDir == "" && cfg.ModulesEnabled {
+			p.Internal.Build.BinDir = ModBinDir()
 		}
 		if p.Internal.Build.BinDir != "" {
 			// Install to GOBIN or bin of GOPATH entry.
@@ -1411,10 +1460,10 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
 		return
 	}
 
-	if vgo.Enabled() {
-		p.Module = vgo.PackageModuleInfo(p.ImportPath)
+	if cfg.ModulesEnabled {
+		p.Module = ModPackageModuleInfo(p.ImportPath)
 		if p.Name == "main" {
-			p.Internal.BuildInfo = vgo.PackageBuildInfo(p.ImportPath, p.Deps)
+			p.Internal.BuildInfo = ModPackageBuildInfo(p.ImportPath, p.Deps)
 		}
 	}
 }
@@ -1695,7 +1744,10 @@ func ImportPaths(args []string) []string {
 	if cmdlineMatchers == nil {
 		SetCmdlinePatterns(search.CleanImportPaths(args))
 	}
-	return vgo.ImportPaths(args)
+	if cfg.ModulesEnabled {
+		return ModImportPaths(args)
+	}
+	return search.ImportPaths(args)
 }
 
 func ImportPathsForGoGet(args []string) []string {
@@ -1790,7 +1842,9 @@ func GoFilesPackage(gofiles []string) *Package {
 	}
 	ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil }
 
-	vgo.AddImports(gofiles)
+	if cfg.ModulesEnabled {
+		ModImportFromFiles(gofiles)
+	}
 
 	var err error
 	if dir == "" {
@@ -1820,8 +1874,8 @@ func GoFilesPackage(gofiles []string) *Package {
 		}
 		if cfg.GOBIN != "" {
 			pkg.Target = filepath.Join(cfg.GOBIN, exe)
-		} else if vgo.Enabled() {
-			pkg.Target = filepath.Join(vgo.BinDir(), exe)
+		} else if cfg.ModulesEnabled {
+			pkg.Target = filepath.Join(ModBinDir(), exe)
 		}
 	}
 

src/cmd/go/internal/load/search.go

@@ -15,7 +15,7 @@ import (
 // MatchPackage(pattern, cwd)(p) reports whether package p matches pattern in the working directory cwd.
 func MatchPackage(pattern, cwd string) func(*Package) bool {
 	switch {
-	case strings.HasPrefix(pattern, "./") || strings.HasPrefix(pattern, "../") || pattern == "." || pattern == "..":
+	case search.IsRelativePath(pattern):
 		// Split pattern into leading pattern-free directory path
 		// (including all . and .. elements) and the final pattern.
 		var dir string

src/cmd/go/internal/vgo/vendor.go → src/cmd/go/internal/modcmd/vendor.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package vgo
+package modcmd
 
 import (
 	"bytes"
@@ -14,143 +14,165 @@ import (
 	"strings"
 
 	"cmd/go/internal/base"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/module"
 )
 
-var CmdVendor = &base.Command{
-	UsageLine: "vendor [-v]",
-	Short:     "vendor dependencies of current module",
-	Long: `
-Vendor resets the module's vendor directory to include all
-packages needed to build and test all packages in the module
-and their dependencies.
-
-The -v flag causes vendor to print to standard error the
-module paths of the modules processed and the import paths
-of the packages copied.
-	`,
-}
-
-var vendorV = CmdVendor.Flag.Bool("v", false, "")
+func runVendor() {
+	pkgs := modload.LoadVendor()
 
-func init() {
-	CmdVendor.Run = runVendor // break init cycle
-}
-
-func runVendor(cmd *base.Command, args []string) {
-	if Init(); !Enabled() {
-		base.Fatalf("vgo vendor: cannot use -m outside module")
-	}
-	if len(args) != 0 {
-		base.Fatalf("vgo vendor: vendor takes no arguments")
-	}
-	InitMod()
-	pkgs := ImportPaths([]string{"ALL"})
-
-	vdir := filepath.Join(ModRoot, "vendor")
+	vdir := filepath.Join(modload.ModRoot, "vendor")
 	if err := os.RemoveAll(vdir); err != nil {
-		base.Fatalf("vgo vendor: %v", err)
+		base.Fatalf("go vendor: %v", err)
 	}
 
 	modpkgs := make(map[module.Version][]string)
 	for _, pkg := range pkgs {
-		m := pkgmod[pkg]
-		if m == Target {
+		m := modload.PackageModule(pkg)
+		if m == modload.Target {
 			continue
 		}
 		modpkgs[m] = append(modpkgs[m], pkg)
 	}
 
 	var buf bytes.Buffer
-	for _, m := range buildList[1:] {
+	for _, m := range modload.BuildList()[1:] {
 		if pkgs := modpkgs[m]; len(pkgs) > 0 {
 			repl := ""
-			if r := replaced(m); r != nil {
-				repl = " => " + r.New.Path
-				if r.New.Version != "" {
-					repl += " " + r.New.Version
+			if r := modload.Replacement(m); r.Path != "" {
+				repl = " => " + r.Path
+				if r.Version != "" {
+					repl += " " + r.Version
 				}
 			}
 			fmt.Fprintf(&buf, "# %s %s%s\n", m.Path, m.Version, repl)
-			if *vendorV {
+			if *modV {
 				fmt.Fprintf(os.Stderr, "# %s %s%s\n", m.Path, m.Version, repl)
 			}
 			for _, pkg := range pkgs {
 				fmt.Fprintf(&buf, "%s\n", pkg)
-				if *vendorV {
+				if *modV {
 					fmt.Fprintf(os.Stderr, "%s\n", pkg)
 				}
 				vendorPkg(vdir, pkg)
 			}
 		}
 	}
-	if err := ioutil.WriteFile(filepath.Join(vdir, "vgo.list"), buf.Bytes(), 0666); err != nil {
-		base.Fatalf("vgo vendor: %v", err)
+	if buf.Len() == 0 {
+		fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
+		return
+	}
+	if err := ioutil.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil {
+		base.Fatalf("go vendor: %v", err)
 	}
 }
 
 func vendorPkg(vdir, pkg string) {
-	realPath := importmap[pkg]
-	if realPath != pkg && importmap[realPath] != "" {
+	realPath := modload.ImportMap(pkg)
+	if realPath != pkg && modload.ImportMap(realPath) != "" {
 		fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
 	}
 
 	dst := filepath.Join(vdir, pkg)
-	src := pkgdir[realPath]
+	src := modload.PackageDir(realPath)
 	if src == "" {
 		fmt.Fprintf(os.Stderr, "internal error: no pkg for %s -> %s\n", pkg, realPath)
 	}
-	copyDir(dst, src, false)
+	copyDir(dst, src, matchNonTest)
+	if m := modload.PackageModule(realPath); m.Path != "" {
+		copyMetadata(m.Path, realPath, dst, src)
+	}
+}
+
+type metakey struct {
+	modPath string
+	dst     string
+}
+
+var copiedMetadata = make(map[metakey]bool)
+
+// copyMetadata copies metadata files from parents of src to parents of dst,
+// stopping after processing the src parent for modPath.
+func copyMetadata(modPath, pkg, dst, src string) {
+	for parent := 0; ; parent++ {
+		if copiedMetadata[metakey{modPath, dst}] {
+			break
+		}
+		copiedMetadata[metakey{modPath, dst}] = true
+		if parent > 0 {
+			copyDir(dst, src, matchMetadata)
+		}
+		if modPath == pkg {
+			break
+		}
+		pkg = filepath.Dir(pkg)
+		dst = filepath.Dir(dst)
+		src = filepath.Dir(src)
+	}
 }
 
-func copyDir(dst, src string, recursive bool) {
+// metaPrefixes is the list of metadata file prefixes.
+// Vendoring copies metadata files from parents of copied directories.
+// Note that this list could be arbitrarily extended, and it is longer
+// in other tools (such as godep or dep). By using this limited set of
+// prefixes and also insisting on capitalized file names, we are trying
+// to nudge people toward more agreement on the naming
+// and also trying to avoid false positives.
+var metaPrefixes = []string{
+	"AUTHORS",
+	"CONTRIBUTORS",
+	"COPYLEFT",
+	"COPYING",
+	"COPYRIGHT",
+	"LEGAL",
+	"LICENSE",
+	"NOTICE",
+	"PATENTS",
+}
+
+// matchMetadata reports whether info is a metadata file.
+func matchMetadata(info os.FileInfo) bool {
+	name := info.Name()
+	for _, p := range metaPrefixes {
+		if strings.HasPrefix(name, p) {
+			return true
+		}
+	}
+	return false
+}
+
+// matchNonTest reports whether info is any non-test file (including non-Go files).
+func matchNonTest(info os.FileInfo) bool {
+	return !strings.HasSuffix(info.Name(), "_test.go")
+}
+
+// copyDir copies all regular files satisfying match(info) from src to dst.
+func copyDir(dst, src string, match func(os.FileInfo) bool) {
 	files, err := ioutil.ReadDir(src)
 	if err != nil {
-		base.Fatalf("vgo vendor: %v", err)
+		base.Fatalf("go vendor: %v", err)
 	}
 	if err := os.MkdirAll(dst, 0777); err != nil {
-		base.Fatalf("vgo vendor: %v", err)
+		base.Fatalf("go vendor: %v", err)
 	}
 	for _, file := range files {
-		if file.IsDir() {
-			if recursive || file.Name() == "testdata" {
-				copyDir(filepath.Join(dst, file.Name()), filepath.Join(src, file.Name()), true)
-			}
-			continue
-		}
-		if !file.Mode().IsRegular() {
+		if file.IsDir() || !file.Mode().IsRegular() || !match(file) {
 			continue
 		}
 		r, err := os.Open(filepath.Join(src, file.Name()))
 		if err != nil {
-			base.Fatalf("vgo vendor: %v", err)
+			base.Fatalf("go vendor: %v", err)
 		}
 		w, err := os.Create(filepath.Join(dst, file.Name()))
 		if err != nil {
-			base.Fatalf("vgo vendor: %v", err)
+			base.Fatalf("go vendor: %v", err)
 		}
 		if _, err := io.Copy(w, r); err != nil {
-			base.Fatalf("vgo vendor: %v", err)
+			base.Fatalf("go vendor: %v", err)
 		}
 		r.Close()
 		if err := w.Close(); err != nil {
-			base.Fatalf("vgo vendor: %v", err)
-		}
-	}
-}
-
-// hasPathPrefix reports whether the path s begins with the
-// elements in prefix.
-func hasPathPrefix(s, prefix string) bool {
-	switch {
-	default:
-		return false
-	case len(s) == len(prefix):
-		return s == prefix
-	case len(s) > len(prefix):
-		if prefix != "" && prefix[len(prefix)-1] == '/' {
-			return strings.HasPrefix(s, prefix)
+			base.Fatalf("go vendor: %v", err)
 		}
-		return s[len(prefix)] == '/' && s[:len(prefix)] == prefix
 	}
 }

src/cmd/go/internal/vgo/verify.go → src/cmd/go/internal/modcmd/verify.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package vgo
+package modcmd
 
 import (
 	"bytes"
@@ -13,43 +13,14 @@ import (
 
 	"cmd/go/internal/base"
 	"cmd/go/internal/dirhash"
+	"cmd/go/internal/modfetch"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/module"
 )
 
-var CmdVerify = &base.Command{
-	UsageLine: "verify",
-	Run:       runVerify,
-	Short:     "verify downloaded modules against expected hashes",
-	Long: `
-Verify checks that the dependencies of the current module,
-which are stored in a local downloaded source cache,
-have not been modified since being downloaded.
-
-If all the modules are unmodified, verify prints
-
-	all modules verified
-
-and exits successfully (status 0). Otherwise, verify reports
-which modules have been changed and exits with a non-zero status.
-	`,
-}
-
-func runVerify(cmd *base.Command, args []string) {
-	if Init(); !Enabled() {
-		base.Fatalf("vgo verify: cannot use outside module")
-	}
-	if len(args) != 0 {
-		// TODO: take arguments
-		base.Fatalf("vgo verify: verify takes no arguments")
-	}
-
-	// Make go.mod consistent but don't load any packages.
-	InitMod()
-	iterate(func(*loader) {})
-	writeGoMod()
-
+func runVerify() {
 	ok := true
-	for _, mod := range buildList[1:] {
+	for _, mod := range modload.LoadBuildList()[1:] {
 		ok = verifyMod(mod) && ok
 	}
 	if ok {
@@ -59,9 +30,9 @@ func runVerify(cmd *base.Command, args []string) {
 
 func verifyMod(mod module.Version) bool {
 	ok := true
-	zip := filepath.Join(srcV, "cache", mod.Path, "/@v/", mod.Version+".zip")
+	zip := filepath.Join(modfetch.SrcMod, "cache/download", mod.Path, "/@v/", mod.Version+".zip")
 	_, zipErr := os.Stat(zip)
-	dir := filepath.Join(srcV, mod.Path+"@"+mod.Version)
+	dir := filepath.Join(modfetch.SrcMod, mod.Path+"@"+mod.Version)
 	_, dirErr := os.Stat(dir)
 	data, err := ioutil.ReadFile(zip + "hash")
 	if err != nil {

src/cmd/go/internal/modfetch/convert.go → src/cmd/go/internal/modconv/convert.go

@@ -2,16 +2,20 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package modfetch
+package modconv
 
 import (
 	"fmt"
 	"os"
 	"sort"
 	"strings"
+	"sync"
 
-	"cmd/go/internal/modconv"
+	"cmd/go/internal/base"
+	"cmd/go/internal/modfetch"
 	"cmd/go/internal/modfile"
+	"cmd/go/internal/module"
+	"cmd/go/internal/par"
 	"cmd/go/internal/semver"
 )
 
@@ -23,47 +27,45 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
 	if i >= 0 {
 		j = strings.LastIndex(file[:i], "/")
 	}
-	convert := modconv.Converters[file[i+1:]]
+	convert := Converters[file[i+1:]]
 	if convert == nil && j != -2 {
-		convert = modconv.Converters[file[j+1:]]
+		convert = Converters[file[j+1:]]
 	}
 	if convert == nil {
 		return fmt.Errorf("unknown legacy config file %s", file)
 	}
-	require, err := convert(file, data)
+	mf, err := convert(file, data)
 	if err != nil {
 		return fmt.Errorf("parsing %s: %v", file, err)
 	}
 
 	// Convert requirements block, which may use raw SHA1 hashes as versions,
 	// to valid semver requirement list, respecting major versions.
-	need := make(map[string]string)
-	for _, r := range require {
-		if r.Path == "" {
+	var work par.Work
+	for _, r := range mf.Require {
+		m := r.Mod
+		if m.Path == "" {
 			continue
 		}
-
-		// TODO: Something better here.
-		if strings.HasPrefix(r.Path, "github.com/") || strings.HasPrefix(r.Path, "golang.org/x/") {
-			f := strings.Split(r.Path, "/")
-			if len(f) > 3 {
-				r.Path = strings.Join(f[:3], "/")
-			}
+		work.Add(r.Mod)
 	}
 
-		repo, err := Lookup(r.Path)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "vgo: lookup %s: %v\n", r.Path, err)
-			continue
-		}
-		info, err := repo.Stat(r.Version)
+	var (
+		mu   sync.Mutex
+		need = make(map[string]string)
+	)
+	work.Do(10, func(item interface{}) {
+		r := item.(module.Version)
+		repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "vgo: stat %s@%s: %v\n", r.Path, r.Version, err)
-			continue
+			fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err)
+			return
 		}
+		mu.Lock()
 		path := repo.ModulePath()
 		need[path] = semver.Max(need[path], info.Version)
-	}
+		mu.Unlock()
+	})
 
 	var paths []string
 	for path := range need {
@@ -71,8 +73,15 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
 	}
 	sort.Strings(paths)
 	for _, path := range paths {
-		f.AddRequire(path, need[path])
+		f.AddNewRequire(path, need[path], false)
 	}
 
+	for _, r := range mf.Replace {
+		err := f.AddReplace(r.Old.Path, r.Old.Version, r.New.Path, r.New.Version)
+		if err != nil {
+			return fmt.Errorf("add replace: %v", err)
+		}
+	}
+	f.Cleanup()
 	return nil
 }

src/cmd/go/internal/modfetch/convert_test.go → src/cmd/go/internal/modconv/convert_test.go

@@ -2,19 +2,49 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package modfetch
+package modconv
 
 import (
 	"bytes"
+	"fmt"
 	"internal/testenv"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"cmd/go/internal/cfg"
-	"cmd/go/internal/modconv"
+	"cmd/go/internal/modfetch"
+	"cmd/go/internal/modfetch/codehost"
 	"cmd/go/internal/modfile"
+	"cmd/go/internal/module"
 )
 
+func TestMain(m *testing.M) {
+	os.Exit(testMain(m))
+}
+
+func testMain(m *testing.M) int {
+	if _, err := exec.LookPath("git"); err != nil {
+		fmt.Fprintln(os.Stderr, "skipping because git binary not found")
+		fmt.Println("PASS")
+		return 0
+	}
+
+	dir, err := ioutil.TempDir("", "modconv-test-")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	modfetch.SrcMod = filepath.Join(dir, "src/mod")
+	codehost.WorkRoot = filepath.Join(dir, "codework")
+
+	return m.Run()
+}
+
 func TestConvertLegacyConfig(t *testing.T) {
 	testenv.MustHaveExternalNetwork(t)
 
@@ -65,7 +95,7 @@ func TestConvertLegacyConfig(t *testing.T) {
 			require (
 				github.com/AdRoll/goamz v0.0.0-20150130162828-d3664b76d905
 				github.com/MSOpenTech/azure-sdk-for-go v0.0.0-20150323223030-d90753bcad2e
-				github.com/Sirupsen/logrus v0.0.0-20150409230825-55eb11d21d2a
+				github.com/Sirupsen/logrus v0.7.3
 				github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd
 				github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b
 				github.com/bugsnag/panicwrap v0.0.0-20141110184334-e5f9854865b9
@@ -117,23 +147,32 @@ func TestConvertLegacyConfig(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			repo, err := Lookup(tt.path)
+
+			dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers})
 			if err != nil {
 				t.Fatal(err)
 			}
 
-			out, err := repo.GoMod(tt.vers)
-			if err != nil {
+			for name := range Converters {
+				file := filepath.Join(dir, name)
+				data, err := ioutil.ReadFile(file)
+				if err == nil {
+					f := new(modfile.File)
+					f.AddModuleStmt(tt.path)
+					if err := ConvertLegacyConfig(f, filepath.ToSlash(file), data); err != nil {
 						t.Fatal(err)
 					}
-			prefix := modconv.Prefix + "\n"
-			if !bytes.HasPrefix(out, []byte(prefix)) {
-				t.Fatalf("go.mod missing prefix %q:\n%s", prefix, out)
+					out, err := f.Format()
+					if err != nil {
+						t.Fatalf("format after conversion: %v", err)
 					}
-			out = out[len(prefix):]
 					if !bytes.Equal(out, want) {
 						t.Fatalf("final go.mod:\n%s\n\nwant:\n%s", out, want)
 					}
+					return
+				}
+			}
+			t.Fatalf("no converter found for %s@%s", tt.path, tt.vers)
 		})
 	}
 }

src/cmd/go/internal/modconv/dep.go

@@ -9,11 +9,13 @@ import (
 	"strconv"
 	"strings"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 	"cmd/go/internal/semver"
 )
 
-func ParseGopkgLock(file string, data []byte) ([]module.Version, error) {
+func ParseGopkgLock(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	var list []module.Version
 	var r *module.Version
 	for lineno, line := range strings.Split(string(data), "\n") {
@@ -66,6 +68,7 @@ func ParseGopkgLock(file string, data []byte) ([]module.Version, error) {
 		if r.Path == "" || r.Version == "" {
 			return nil, fmt.Errorf("%s: empty [[projects]] stanza (%s)", file, r.Path)
 		}
+		mf.Require = append(mf.Require, &modfile.Require{Mod: r})
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/glide.go

@@ -5,12 +5,14 @@
 package modconv
 
 import (
-	"cmd/go/internal/module"
 	"strings"
+
+	"cmd/go/internal/modfile"
+	"cmd/go/internal/module"
 )
 
-func ParseGlideLock(file string, data []byte) ([]module.Version, error) {
-	var list []module.Version
+func ParseGlideLock(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	imports := false
 	name := ""
 	for lineno, line := range strings.Split(string(data), "\n") {
@@ -32,9 +34,9 @@ func ParseGlideLock(file string, data []byte) ([]module.Version, error) {
 		if strings.HasPrefix(line, "  version:") {
 			version := strings.TrimSpace(line[len("  version:"):])
 			if name != "" && version != "" {
-				list = append(list, module.Version{Path: name, Version: version})
+				mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: name, Version: version}})
 			}
 		}
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/glock.go

@@ -7,17 +7,18 @@ package modconv
 import (
 	"strings"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseGLOCKFILE(file string, data []byte) ([]module.Version, error) {
-	var list []module.Version
+func ParseGLOCKFILE(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	for lineno, line := range strings.Split(string(data), "\n") {
 		lineno++
 		f := strings.Fields(line)
 		if len(f) >= 2 && f[0] != "cmd" {
-			list = append(list, module.Version{Path: f[0], Version: f[1]})
+			mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[1]}})
 		}
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/godeps.go

@@ -7,10 +7,11 @@ package modconv
 import (
 	"encoding/json"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseGodepsJSON(file string, data []byte) ([]module.Version, error) {
+func ParseGodepsJSON(file string, data []byte) (*modfile.File, error) {
 	var cfg struct {
 		ImportPath string
 		Deps       []struct {
@@ -21,9 +22,9 @@ func ParseGodepsJSON(file string, data []byte) ([]module.Version, error) {
 	if err := json.Unmarshal(data, &cfg); err != nil {
 		return nil, err
 	}
-	var list []module.Version
+	mf := new(modfile.File)
 	for _, d := range cfg.Deps {
-		list = append(list, module.Version{Path: d.ImportPath, Version: d.Rev})
+		mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: d.ImportPath, Version: d.Rev}})
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/modconv.go

@@ -4,9 +4,9 @@
 
 package modconv
 
-import "cmd/go/internal/module"
+import "cmd/go/internal/modfile"
 
-var Converters = map[string]func(string, []byte) ([]module.Version, error){
+var Converters = map[string]func(string, []byte) (*modfile.File, error){
 	"GLOCKFILE":          ParseGLOCKFILE,
 	"Godeps/Godeps.json": ParseGodepsJSON,
 	"Gopkg.lock":         ParseGopkgLock,
@@ -17,9 +17,3 @@ var Converters = map[string]func(string, []byte) ([]module.Version, error){
 	"vendor/manifest":    ParseVendorManifest,
 	"vendor/vendor.json": ParseVendorJSON,
 }
-
-// Prefix is a line we write at the top of auto-converted go.mod files
-// for dependencies before caching them.
-// In case of bugs in the converter, if we bump this version number,
-// then all the cached copies will be ignored.
-const Prefix = "//vgo 0.0.4\n"

src/cmd/go/internal/modconv/modconv_test.go

@@ -7,7 +7,6 @@ package modconv
 import (
 	"bytes"
 	"fmt"
-	"internal/testenv"
 	"io/ioutil"
 	"path/filepath"
 	"testing"
@@ -26,8 +25,6 @@ var extMap = map[string]string{
 }
 
 func Test(t *testing.T) {
-	testenv.MustHaveExternalNetwork(t)
-
 	tests, _ := filepath.Glob("testdata/*")
 	if len(tests) == 0 {
 		t.Fatalf("no tests found")
@@ -58,8 +55,8 @@ func Test(t *testing.T) {
 				t.Error(err)
 			}
 			var buf bytes.Buffer
-			for _, r := range out {
-				fmt.Fprintf(&buf, "%s %s\n", r.Path, r.Version)
+			for _, r := range out.Require {
+				fmt.Fprintf(&buf, "%s %s\n", r.Mod.Path, r.Mod.Version)
 			}
 			if !bytes.Equal(buf.Bytes(), want) {
 				t.Errorf("have:\n%s\nwant:\n%s", buf.Bytes(), want)

src/cmd/go/internal/modconv/tsv.go

@@ -7,17 +7,18 @@ package modconv
 import (
 	"strings"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseDependenciesTSV(file string, data []byte) ([]module.Version, error) {
-	var list []module.Version
+func ParseDependenciesTSV(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	for lineno, line := range strings.Split(string(data), "\n") {
 		lineno++
 		f := strings.Split(line, "\t")
 		if len(f) >= 3 {
-			list = append(list, module.Version{Path: f[0], Version: f[2]})
+			mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[2]}})
 		}
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/vconf.go

@@ -7,11 +7,12 @@ package modconv
 import (
 	"strings"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseVendorConf(file string, data []byte) ([]module.Version, error) {
-	var list []module.Version
+func ParseVendorConf(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	for lineno, line := range strings.Split(string(data), "\n") {
 		lineno++
 		if i := strings.Index(line, "#"); i >= 0 {
@@ -19,8 +20,8 @@ func ParseVendorConf(file string, data []byte) ([]module.Version, error) {
 		}
 		f := strings.Fields(line)
 		if len(f) >= 2 {
-			list = append(list, module.Version{Path: f[0], Version: f[1]})
+			mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[1]}})
 		}
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/vjson.go

@@ -7,10 +7,11 @@ package modconv
 import (
 	"encoding/json"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseVendorJSON(file string, data []byte) ([]module.Version, error) {
+func ParseVendorJSON(file string, data []byte) (*modfile.File, error) {
 	var cfg struct {
 		Package []struct {
 			Path     string
@@ -20,9 +21,9 @@ func ParseVendorJSON(file string, data []byte) ([]module.Version, error) {
 	if err := json.Unmarshal(data, &cfg); err != nil {
 		return nil, err
 	}
-	var list []module.Version
+	mf := new(modfile.File)
 	for _, d := range cfg.Package {
-		list = append(list, module.Version{Path: d.Path, Version: d.Revision})
+		mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: d.Path, Version: d.Revision}})
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/vmanifest.go

@@ -7,10 +7,11 @@ package modconv
 import (
 	"encoding/json"
 
+	"cmd/go/internal/modfile"
 	"cmd/go/internal/module"
 )
 
-func ParseVendorManifest(file string, data []byte) ([]module.Version, error) {
+func ParseVendorManifest(file string, data []byte) (*modfile.File, error) {
 	var cfg struct {
 		Dependencies []struct {
 			ImportPath string
@@ -20,9 +21,9 @@ func ParseVendorManifest(file string, data []byte) ([]module.Version, error) {
 	if err := json.Unmarshal(data, &cfg); err != nil {
 		return nil, err
 	}
-	var list []module.Version
+	mf := new(modfile.File)
 	for _, d := range cfg.Dependencies {
-		list = append(list, module.Version{Path: d.ImportPath, Version: d.Revision})
+		mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: d.ImportPath, Version: d.Revision}})
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modconv/vyml.go

@@ -5,12 +5,14 @@
 package modconv
 
 import (
-	"cmd/go/internal/module"
 	"strings"
+
+	"cmd/go/internal/modfile"
+	"cmd/go/internal/module"
 )
 
-func ParseVendorYML(file string, data []byte) ([]module.Version, error) {
-	var list []module.Version
+func ParseVendorYML(file string, data []byte) (*modfile.File, error) {
+	mf := new(modfile.File)
 	vendors := false
 	path := ""
 	for lineno, line := range strings.Split(string(data), "\n") {
@@ -32,9 +34,9 @@ func ParseVendorYML(file string, data []byte) ([]module.Version, error) {
 		if strings.HasPrefix(line, "  rev:") {
 			rev := strings.TrimSpace(line[len("  rev:"):])
 			if path != "" && rev != "" {
-				list = append(list, module.Version{Path: path, Version: rev})
+				mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: path, Version: rev}})
 			}
 		}
 	}
-	return list, nil
+	return mf, nil
 }

src/cmd/go/internal/modfetch/bitbucket/fetch.go

-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bitbucket
-
-import (
-	"fmt"
-	"strings"
-
-	"cmd/go/internal/modfetch/codehost"
-	"cmd/go/internal/modfetch/gitrepo"
-)
-
-func Lookup(path string) (codehost.Repo, error) {
-	f := strings.Split(path, "/")
-	if len(f) < 3 || f[0] != "bitbucket.org" {
-		return nil, fmt.Errorf("bitbucket repo must be bitbucket.org/org/project")
-	}
-	path = f[0] + "/" + f[1] + "/" + f[2]
-	return gitrepo.Repo("https://"+path, path)
-}

src/cmd/go/internal/modfetch/gopkgin.go → src/cmd/go/internal/modfetch/cache_test.go

@@ -2,21 +2,24 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// TODO: Figure out what gopkg.in should do.
-
 package modfetch
 
 import (
-	"cmd/go/internal/modfetch/codehost"
-	"cmd/go/internal/modfetch/gitrepo"
-	"cmd/go/internal/modfile"
-	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
 )
 
-func gopkginLookup(path string) (codehost.Repo, error) {
-	root, _, _, _, ok := modfile.ParseGopkgIn(path)
-	if !ok {
-		return nil, fmt.Errorf("invalid gopkg.in/ path: %q", path)
+func TestWriteDiskCache(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "go-writeCache-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	err = writeDiskCache(filepath.Join(tmpdir, "file"), []byte("data"))
+	if err != nil {
+		t.Fatal(err)
 	}
-	return gitrepo.Repo("https://"+root, root)
 }

src/cmd/go/internal/modfetch/codehost/codehost.go

@@ -16,6 +16,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"cmd/go/internal/cfg"
@@ -32,10 +33,8 @@ const (
 // A Repo represents a code hosting source.
 // Typical implementations include local version control repositories,
 // remote version control servers, and code hosting sites.
+// A Repo must be safe for simultaneous use by multiple goroutines.
 type Repo interface {
-	// Root returns the import path of the root directory of the repository.
-	Root() string
-
 	// List lists all tags with the given prefix.
 	Tags(prefix string) (tags []string, err error)
 
@@ -50,6 +49,9 @@ type Repo interface {
 
 	// ReadFile reads the given file in the file tree corresponding to revision rev.
 	// It should refuse to read more than maxSize bytes.
+	//
+	// If the requested file does not exist it should return an error for which
+	// os.IsNotExist(err) returns true.
 	ReadFile(rev, file string, maxSize int64) (data []byte, err error)
 
 	// ReadZip downloads a zip file for the subdir subdirectory
@@ -66,8 +68,9 @@ type Repo interface {
 type RevInfo struct {
 	Name    string    // complete ID in underlying repository
 	Short   string    // shortened ID, for use in pseudo-version
-	Version string    // TODO what is this?
+	Version string    // version used in lookup
 	Time    time.Time // commit time
+	Tags    []string  // known tags for commit
 }
 
 // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
@@ -92,7 +95,7 @@ func ShortenSHA1(rev string) string {
 }
 
 // WorkRoot is the root of the cached work directory.
-// It is set by cmd/go/internal/vgo.InitMod.
+// It is set by cmd/go/internal/modload.InitMod.
 var WorkRoot string
 
 // WorkDir returns the name of the cached work directory to use for the
@@ -113,21 +116,20 @@ func WorkDir(typ, name string) (string, error) {
 	key := typ + ":" + name
 	dir := filepath.Join(WorkRoot, fmt.Sprintf("%x", sha256.Sum256([]byte(key))))
 	data, err := ioutil.ReadFile(dir + ".info")
-	if err == nil {
+	info, err2 := os.Stat(dir)
+	if err == nil && err2 == nil && info.IsDir() {
+		// Info file and directory both already exist: reuse.
 		have := strings.TrimSuffix(string(data), "\n")
 		if have != key {
 			return "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key)
 		}
-		_, err := os.Stat(dir)
-		if err != nil {
-			return "", fmt.Errorf("%s exists but %s does not", dir+".info", dir)
-		}
 		if cfg.BuildX {
 			fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name)
 		}
 		return dir, nil
 	}
 
+	// Info file or directory missing. Start from scratch.
 	if cfg.BuildX {
 		fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name)
 	}
@@ -157,19 +159,36 @@ func (e *RunError) Error() string {
 	return text
 }
 
+var dirLock sync.Map
+
 // Run runs the command line in the given directory
 // (an empty dir means the current directory).
 // It returns the standard output and, for a non-zero exit,
 // a *RunError indicating the command, exit status, and standard error.
 // Standard error is unavailable for commands that exit successfully.
 func Run(dir string, cmdline ...interface{}) ([]byte, error) {
+	if dir != "" {
+		muIface, ok := dirLock.Load(dir)
+		if !ok {
+			muIface, _ = dirLock.LoadOrStore(dir, new(sync.Mutex))
+		}
+		mu := muIface.(*sync.Mutex)
+		mu.Lock()
+		defer mu.Unlock()
+	}
+
 	cmd := str.StringList(cmdline...)
 	if cfg.BuildX {
-		var cd string
+		var text string
 		if dir != "" {
-			cd = "cd " + dir + "; "
-		}
-		fmt.Fprintf(os.Stderr, "%s%s\n", cd, strings.Join(cmd, " "))
+			text = "cd " + dir + "; "
+		}
+		text += strings.Join(cmd, " ")
+		fmt.Fprintf(os.Stderr, "%s\n", text)
+		start := time.Now()
+		defer func() {
+			fmt.Fprintf(os.Stderr, "%.3fs # %s\n", time.Since(start).Seconds(), text)
+		}()
 	}
 	// TODO: Impose limits on command output size.
 	// TODO: Set environment to get English error messages.

src/cmd/go/internal/modfetch/codehost/shell.go

+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Interactive debugging shell for codehost.Repo implementations.
+
+package main
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"time"
+
+	"cmd/go/internal/modfetch/codehost"
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "usage: go run shell.go vcs remote\n")
+	os.Exit(2)
+}
+
+func main() {
+	codehost.WorkRoot = "/tmp/vcswork"
+	log.SetFlags(0)
+	log.SetPrefix("shell: ")
+	flag.Usage = usage
+	flag.Parse()
+	if flag.NArg() != 2 {
+		usage()
+	}
+
+	repo, err := codehost.NewRepo(flag.Arg(0), flag.Arg(1))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	b := bufio.NewReader(os.Stdin)
+	for {
+		fmt.Fprintf(os.Stderr, ">>> ")
+		line, err := b.ReadString('\n')
+		if err != nil {
+			log.Fatal(err)
+		}
+		f := strings.Fields(line)
+		if len(f) == 0 {
+			continue
+		}
+		switch f[0] {
+		default:
+			fmt.Fprintf(os.Stderr, "?unknown command\n")
+			continue
+		case "tags":
+			prefix := ""
+			if len(f) == 2 {
+				prefix = f[1]
+			}
+			if len(f) > 2 {
+				fmt.Fprintf(os.Stderr, "?usage: tags [prefix]\n")
+				continue
+			}
+			tags, err := repo.Tags(prefix)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+			for _, tag := range tags {
+				fmt.Printf("%s\n", tag)
+			}
+
+		case "stat":
+			if len(f) != 2 {
+				fmt.Fprintf(os.Stderr, "?usage: stat rev\n")
+				continue
+			}
+			info, err := repo.Stat(f[1])
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+			fmt.Printf("name=%s short=%s version=%s time=%s\n", info.Name, info.Short, info.Version, info.Time.UTC().Format(time.RFC3339))
+
+		case "read":
+			if len(f) != 3 {
+				fmt.Fprintf(os.Stderr, "?usage: read rev file\n")
+				continue
+			}
+			data, err := repo.ReadFile(f[1], f[2], 10<<20)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+			os.Stdout.Write(data)
+
+		case "zip":
+			if len(f) != 4 {
+				fmt.Fprintf(os.Stderr, "?usage: zip rev subdir output\n")
+				continue
+			}
+			subdir := f[2]
+			if subdir == "-" {
+				subdir = ""
+			}
+			rc, _, err := repo.ReadZip(f[1], subdir, 10<<20)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+			data, err := ioutil.ReadAll(rc)
+			rc.Close()
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+
+			if f[3] != "-" {
+				if err := ioutil.WriteFile(f[3], data, 0666); err != nil {
+					fmt.Fprintf(os.Stderr, "?%s\n", err)
+					continue
+				}
+			}
+			z, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "?%s\n", err)
+				continue
+			}
+			for _, f := range z.File {
+				fmt.Printf("%s %d\n", f.Name, f.UncompressedSize64)
+			}
+		}
+	}
+}

src/cmd/go/internal/modfetch/domain.go

-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Support for custom domains.
-
-package modfetch
-
-import (
-	"encoding/xml"
-	"fmt"
-	"io"
-	"net/url"
-	"os"
-	"strings"
-
-	"cmd/go/internal/modfetch/codehost"
-	"cmd/go/internal/modfetch/gitrepo"
-)
-
-// metaImport represents the parsed <meta name="go-import"
-// content="prefix vcs reporoot" /> tags from HTML files.
-type metaImport struct {
-	Prefix, VCS, RepoRoot string
-}
-
-func lookupCustomDomain(path string) (Repo, error) {
-	dom := path
-	if i := strings.Index(dom, "/"); i >= 0 {
-		dom = dom[:i]
-	}
-	if !strings.Contains(dom, ".") {
-		return nil, fmt.Errorf("unknown module %s: not a domain name", path)
-	}
-	var body io.ReadCloser
-	err := webGetGoGet("https://"+path+"?go-get=1", &body)
-	if body != nil {
-		defer body.Close()
-	}
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "FindRepo: %v\n", err)
-		return nil, err
-	}
-	// Note: accepting a non-200 OK here, so people can serve a
-	// meta import in their http 404 page.
-	imports, err := parseMetaGoImports(body)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "findRepo: %v\n", err)
-		return nil, err
-	}
-	if len(imports) == 0 {
-		return nil, fmt.Errorf("unknown module %s: no go-import tags", path)
-	}
-
-	// First look for new module definition.
-	for _, imp := range imports {
-		if path == imp.Prefix || strings.HasPrefix(path, imp.Prefix+"/") {
-			if imp.VCS == "mod" {
-				u, err := url.Parse(imp.RepoRoot)
-				if err != nil {
-					return nil, fmt.Errorf("invalid module URL %q", imp.RepoRoot)
-				} else if u.Scheme != "https" {
-					// TODO: Allow -insecure flag as a build flag?
-					return nil, fmt.Errorf("invalid module URL %q: must be HTTPS", imp.RepoRoot)
-				}
-				return newProxyRepo(imp.RepoRoot, imp.Prefix), nil
-			}
-		}
-	}
-
-	// Fall back to redirections to known version control systems.
-	for _, imp := range imports {
-		if path == imp.Prefix {
-			if !strings.HasPrefix(imp.RepoRoot, "https://") {
-				// TODO: Allow -insecure flag as a build flag?
-				return nil, fmt.Errorf("invalid server URL %q: must be HTTPS", imp.RepoRoot)
-			}
-			if imp.VCS == "git" {
-				code, err := gitrepo.Repo(imp.RepoRoot, imp.Prefix)
-				if err != nil {
-					return nil, err
-				}
-				return newCodeRepo(code, path)
-			}
-			return nil, fmt.Errorf("unknown VCS, Repo: %s, %s", imp.VCS, imp.RepoRoot)
-		}
-	}
-
-	// Check for redirect to repo root.
-	for _, imp := range imports {
-		if strings.HasPrefix(path, imp.Prefix+"/") {
-			return nil, &ModuleSubdirError{imp.Prefix}
-		}
-	}
-
-	return nil, fmt.Errorf("unknown module %s: no matching go-import tags", path)
-}
-
-type ModuleSubdirError struct {
-	ModulePath string
-}
-
-func (e *ModuleSubdirError) Error() string {
-	return fmt.Sprintf("module root is %q", e.ModulePath)
-}
-
-type customPrefix struct {
-	codehost.Repo
-	root string
-}
-
-func (c *customPrefix) Root() string {
-	return c.root
-}
-
-// parseMetaGoImports returns meta imports from the HTML in r.
-// Parsing ends at the end of the <head> section or the beginning of the <body>.
-func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
-	d := xml.NewDecoder(r)
-	d.CharsetReader = charsetReader
-	d.Strict = false
-	var t xml.Token
-	for {
-		t, err = d.RawToken()
-		if err != nil {
-			if err == io.EOF || len(imports) > 0 {
-				err = nil
-			}
-			return
-		}
-		if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
-			return
-		}
-		if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
-			return
-		}
-		e, ok := t.(xml.StartElement)
-		if !ok || !strings.EqualFold(e.Name.Local, "meta") {
-			continue
-		}
-		if attrValue(e.Attr, "name") != "go-import" {
-			continue
-		}
-		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
-			imports = append(imports, metaImport{
-				Prefix:   f[0],
-				VCS:      f[1],
-				RepoRoot: f[2],
-			})
-		}
-	}
-}
-
-// attrValue returns the attribute value for the case-insensitive key
-// `name', or the empty string if nothing is found.
-func attrValue(attrs []xml.Attr, name string) string {
-	for _, a := range attrs {
-		if strings.EqualFold(a.Name.Local, name) {
-			return a.Value
-		}
-	}
-	return ""
-}
-
-// charsetReader returns a reader for the given charset. Currently
-// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful
-// error which is printed by go get, so the user can find why the package
-// wasn't downloaded if the encoding is not supported. Note that, in
-// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters
-// greater than 0x7f are not rejected).
-func charsetReader(charset string, input io.Reader) (io.Reader, error) {
-	switch strings.ToLower(charset) {
-	case "ascii":
-		return input, nil
-	default:
-		return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
-	}
-}

src/cmd/go/internal/modfetch/github/fetch.go

-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package github
-
-import (
-	"fmt"
-	"strings"
-
-	"cmd/go/internal/modfetch/codehost"
-	"cmd/go/internal/modfetch/gitrepo"
-)
-
-// Lookup returns the code repository enclosing the given module path,
-// which must begin with github.com/.
-func Lookup(path string) (codehost.Repo, error) {
-	f := strings.Split(path, "/")
-	if len(f) < 3 || f[0] != "github.com" {
-		return nil, fmt.Errorf("github repo must be github.com/org/project")
-	}
-	path = f[0] + "/" + f[1] + "/" + f[2]
-	return gitrepo.Repo("https://"+path, path)
-}

src/cmd/go/internal/modfetch/googlesource/fetch.go

-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package googlesource
-
-import (
-	"fmt"
-	"strings"
-
-	"cmd/go/internal/modfetch/codehost"
-	"cmd/go/internal/modfetch/gitrepo"
-)
-
-func Lookup(path string) (codehost.Repo, error) {
-	i := strings.Index(path, "/")
-	if i+1 == len(path) || !strings.HasSuffix(path[:i+1], ".googlesource.com/") {
-		return nil, fmt.Errorf("not *.googlesource.com/*")
-	}
-	j := strings.Index(path[i+1:], "/")
-	if j >= 0 {
-		path = path[:i+1+j]
-	}
-	return gitrepo.Repo("https://"+path, path)
-}

src/cmd/go/internal/modfetch/proxy.go

@@ -55,6 +55,7 @@ func (p *proxyRepo) Versions(prefix string) ([]string, error) {
 			list = append(list, f[0])
 		}
 	}
+	SortVersions(list)
 	return list, nil
 }
 
@@ -134,7 +135,7 @@ func (p *proxyRepo) Zip(version string, tmpdir string) (tmpfile string, err erro
 	defer body.Close()
 
 	// Spool to local file.
-	f, err := ioutil.TempFile(tmpdir, "vgo-proxy-download-")
+	f, err := ioutil.TempFile(tmpdir, "go-proxy-download-")
 	if err != nil {
 		return "", err
 	}

src/cmd/go/internal/modfetch/query.go

-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package modfetch
-
-import (
-	"cmd/go/internal/module"
-	"cmd/go/internal/semver"
-	"fmt"
-	"strings"
-)
-
-// Query looks up a revision of a given module given a version query string.
-// The module must be a complete module path.
-// The version must take one of the following forms:
-//
-//	- the literal string "latest", denoting the latest available tagged version
-//	- v1.2.3, a semantic version string
-//	- v1 or v1.2, an abbreviated semantic version string completed by adding zeroes (v1.0.0 or v1.2.0);
-//	- >v1.2.3, denoting the earliest available version after v1.2.3
-//	- <v1.2.3, denoting the latest available version before v1.2.3
-//	- an RFC 3339 time stamp, denoting the latest available version at that time
-//	- a Unix time expressed as seconds since 1970, denoting the latest available version at that time
-//	- a repository commit identifier, denoting that version
-//
-// The time stamps can be followed by an optional @branch suffix to limit the
-// result to revisions on a particular branch name.
-//
-func Query(path, vers string, allowed func(module.Version) bool) (*RevInfo, error) {
-	repo, err := Lookup(path)
-	if err != nil {
-		return nil, err
-	}
-
-	if strings.HasPrefix(vers, "v") && semver.IsValid(vers) {
-		// TODO: This turns query for "v2" into Stat "v2.0.0",
-		// but probably it should allow checking for a branch named "v2".
-		vers = semver.Canonical(vers)
-		if allowed != nil && !allowed(module.Version{Path: path, Version: vers}) {
-			return nil, fmt.Errorf("%s@%s excluded", path, vers)
-		}
-		return repo.Stat(vers)
-	}
-	if strings.HasPrefix(vers, ">") || strings.HasPrefix(vers, "<") || vers == "latest" {
-		var op string
-		if vers != "latest" {
-			if !semver.IsValid(vers[1:]) {
-				return nil, fmt.Errorf("invalid semantic version in range %s", vers)
-			}
-			op, vers = vers[:1], vers[1:]
-		}
-		versions, err := repo.Versions("")
-		if err != nil {
-			return nil, err
-		}
-		if len(versions) == 0 && vers == "latest" {
-			return repo.Latest()
-		}
-		if vers == "latest" {
-			for i := len(versions) - 1; i >= 0; i-- {
-				if allowed == nil || allowed(module.Version{Path: path, Version: versions[i]}) {
-					return repo.Stat(versions[i])
-				}
-			}
-		} else if op == "<" {
-			for i := len(versions) - 1; i >= 0; i-- {
-				if semver.Compare(versions[i], vers) < 0 && (allowed == nil || allowed(module.Version{Path: path, Version: versions[i]})) {
-					return repo.Stat(versions[i])
-				}
-			}
-		} else {
-			for i := 0; i < len(versions); i++ {
-				if semver.Compare(versions[i], vers) > 0 && (allowed == nil || allowed(module.Version{Path: path, Version: versions[i]})) {
-					return repo.Stat(versions[i])
-				}
-			}
-		}
-		return nil, fmt.Errorf("no matching versions for %s%s", op, vers)
-	}
-	// TODO: Time queries, maybe.
-
-	return repo.Stat(vers)
-}

src/cmd/go/internal/modfetch/unzip.go

@@ -11,9 +11,11 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"cmd/go/internal/modfetch/codehost"
+	"cmd/go/internal/str"
 )
 
 func Unzip(dir, zipfile, prefix string, maxSize int64) error {
@@ -49,12 +51,15 @@ func Unzip(dir, zipfile, prefix string, maxSize int64) error {
 	// Check total size.
 	var size int64
 	for _, zf := range z.File {
-		if !strings.HasPrefix(zf.Name, prefix) {
+		if !str.HasPathPrefix(zf.Name, prefix) {
 			return fmt.Errorf("unzip %v: unexpected file name %s", zipfile, zf.Name)
 		}
-		if strings.HasSuffix(zf.Name, "/") {
+		if zf.Name == prefix || strings.HasSuffix(zf.Name, "/") {
 			continue
 		}
+		if filepath.Clean(zf.Name) != zf.Name || strings.HasPrefix(zf.Name[len(prefix)+1:], "/") {
+			return fmt.Errorf("unzip %v: invalid file name %s", zipfile, zf.Name)
+		}
 		s := int64(zf.UncompressedSize64)
 		if s < 0 || maxSize-size < s {
 			return fmt.Errorf("unzip %v: content too large", zipfile)
@@ -63,11 +68,18 @@ func Unzip(dir, zipfile, prefix string, maxSize int64) error {
 	}
 
 	// Unzip, enforcing sizes checked earlier.
+	dirs := map[string]bool{dir: true}
 	for _, zf := range z.File {
-		if strings.HasSuffix(zf.Name, "/") {
+		if zf.Name == prefix || strings.HasSuffix(zf.Name, "/") {
 			continue
 		}
-		dst := filepath.Join(dir, zf.Name[len(prefix):])
+		name := zf.Name[len(prefix):]
+		dst := filepath.Join(dir, name)
+		parent := filepath.Dir(dst)
+		for parent != dir {
+			dirs[parent] = true
+			parent = filepath.Dir(parent)
+		}
 		if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil {
 			return err
 		}
@@ -77,7 +89,6 @@ func Unzip(dir, zipfile, prefix string, maxSize int64) error {
 		}
 		r, err := zf.Open()
 		if err != nil {
-			r.Close()
 			w.Close()
 			return fmt.Errorf("unzip %v: %v", zipfile, err)
 		}
@@ -96,5 +107,17 @@ func Unzip(dir, zipfile, prefix string, maxSize int64) error {
 		}
 	}
 
+	// Mark directories unwritable, best effort.
+	var dirlist []string
+	for dir := range dirs {
+		dirlist = append(dirlist, dir)
+	}
+	sort.Strings(dirlist)
+
+	// Run over list backward to chmod children before parents.
+	for i := len(dirlist) - 1; i >= 0; i-- {
+		os.Chmod(dir, 0555)
+	}
+
 	return nil
 }

src/cmd/go/internal/modfile/testdata/replace2.golden

@@ -5,4 +5,6 @@ replace (
 	xyz v1.3.4 => my/xyz v1.3.4-me
 	xyz v1.4.5 => "/tmp/my dir"
 	xyz v1.5.6 => my/xyz v1.5.6
+
+	xyz => my/other/xyz v1.5.4
 )

src/cmd/go/internal/modfile/testdata/replace2.in

@@ -5,4 +5,6 @@ replace (
 	"xyz" v1.3.4 => "my/xyz" "v1.3.4-me"
 	xyz "v1.4.5" => "/tmp/my dir"
 	xyz v1.5.6 => my/xyz v1.5.6
+
+	xyz => my/other/xyz v1.5.4
 )