cmd/go: make get -u upgrade only modules providing packages

Currently, 'go get -u' upgrades modules matching command line
arguments and any modules they transitively require. 'go get -u' with
no positional arguments upgrades all modules transitively required by
the main module. This usually adds a large number of indirect
requirements, which is surprising to users.

With this change, 'go get' will load packages specified by
its arguments using a similar process to other commands
('go build', etc). Only modules providing packages will be upgraded.

'go get -u' now upgrades modules providing packages transitively
imported by the command-line arguments. 'go get -u' without arguments
will only upgrade modules needed by the package in the current
directory.

'go get -m' will load all packages within a module. 'go get -m -u'
without arguments will upgrade modules needed by the main module. It
is equivalent to 'go get -u all'. Neither command will upgrade modules
that are required but not used.

Note that 'go get -m' and 'go get -d' both download modules in order
to load packages.

Fixes #26902

Change-Id: I2bad686b3ca8c9de985a81fb42b16a36bb4cc3ea
Reviewed-on: https://go-review.googlesource.com/c/go/+/174099
Run-TryBot: Jay Conrod <jayconrod@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>

src/cmd/go/alldocs.go

@@ -619,6 +619,13 @@
 // each specified package path must be a module path as well,
 // not the import path of a package below the module root.
 //
+// When the -m and -u flags are used together, 'go get' will upgrade
+// modules that provide packages depended on by the modules named on
+// the command line. For example, 'go get -u -m A' will upgrade A and
+// any module providing packages imported by packages in A.
+// 'go get -u -m' will upgrade modules that provided packages needed
+// by the main module.
+//
 // The -insecure flag permits fetching from repositories and resolving
 // custom domains using insecure schemes such as HTTP. Use with caution.
 //
@@ -640,12 +647,11 @@
 // the named packages, including downloading necessary dependencies,
 // but not to build and install them.
 //
-// With no package arguments, 'go get' applies to the main module,
-// and to the Go package in the current directory, if any. In particular,
-// 'go get -u' and 'go get -u=patch' update all the dependencies of the
-// main module. With no package arguments and also without -u,
-// 'go get' is not much more than 'go install', and 'go get -d' not much
-// more than 'go list'.
+// With no package arguments, 'go get' applies to Go package in the
+// current directory, if any. In particular, 'go get -u' and
+// 'go get -u=patch' update all the dependencies of that package.
+// With no package arguments and also without -u, 'go get' is not much more
+// than 'go install', and 'go get -d' not much more than 'go list'.
 //
 // For more about modules, see 'go help modules'.
 //

src/cmd/go/internal/modfetch/codehost/vcs.go

@@ -29,8 +29,9 @@ import (
 // The caller should report this error instead of continuing to probe
 // other possible module paths.
 //
-// TODO(bcmills): See if we can invert this. (Return a distinguished error for
-// “repo not found” and treat everything else as terminal.)
+// TODO(golang.org/issue/31730): See if we can invert this. (Return a
+// distinguished error for “repo not found” and treat everything else
+// as terminal.)
 type VCSError struct {
 	Err error
 }

src/cmd/go/internal/modload/load.go

@@ -33,7 +33,8 @@ import (
 
 // buildList is the list of modules to use for building packages.
 // It is initialized by calling ImportPaths, ImportFromFiles,
-// LoadALL, or LoadBuildList, each of which uses loaded.load.
+// ModulePackages, LoadALL, or LoadBuildList, each of which uses
+// loaded.load.
 //
 // Ideally, exactly ONE of those functions would be called,
 // and exactly once. Most of the time, that's true.
@@ -53,27 +54,22 @@ var loaded *loader
 // ImportPaths returns the set of packages matching the args (patterns),
 // adding modules to the build list as needed to satisfy new imports.
 func ImportPaths(patterns []string) []*search.Match {
-	InitMod()
-
-	var matches []*search.Match
-	for _, pattern := range search.CleanPatterns(patterns) {
-		m := &search.Match{
-			Pattern: pattern,
-			Literal: !strings.Contains(pattern, "...") && !search.IsMetaPackage(pattern),
-		}
-		if m.Literal {
-			m.Pkgs = []string{pattern}
-		}
-		matches = append(matches, m)
-	}
+	matches := ImportPathsQuiet(patterns)
+	search.WarnUnmatched(matches)
+	return matches
+}
 
-	fsDirs := make([][]string, len(matches))
-	loaded = newLoader()
-	updateMatches := func(iterating bool) {
+// ImportPathsQuiet is like ImportPaths but does not warn about patterns with no matches.
+func ImportPathsQuiet(patterns []string) []*search.Match {
+	var fsDirs [][]string
+	updateMatches := func(matches []*search.Match, iterating bool) {
 		for i, m := range matches {
 			switch {
 			case build.IsLocalImport(m.Pattern) || filepath.IsAbs(m.Pattern):
 				// Evaluate list of file system directories on first iteration.
+				if fsDirs == nil {
+					fsDirs = make([][]string, len(matches))
+				}
 				if fsDirs[i] == nil {
 					var dirs []string
 					if m.Literal {
@@ -167,23 +163,113 @@ func ImportPaths(patterns []string) []*search.Match {
 				if len(m.Pkgs) == 0 {
 					m.Pkgs = search.MatchPackages(m.Pattern).Pkgs
 				}
+
+			default:
+				m.Pkgs = []string{m.Pattern}
+			}
+		}
+	}
+
+	return loadPatterns(patterns, true, updateMatches)
+}
+
+// ModulePackages returns packages provided by each module in patterns.
+// patterns may contain module paths, patterns matching module paths,
+// "all" (interpreted as package pattern "all"), and "." (interpreted
+// as the main module). Additional modules (including modules providing
+// dependencies) may be added to the build list or upgraded.
+func ModulePackages(patterns []string) []*search.Match {
+	updateMatches := func(matches []*search.Match, iterating bool) {
+		for _, m := range matches {
+			switch {
+			case search.IsRelativePath(m.Pattern) || filepath.IsAbs(m.Pattern):
+				if m.Pattern != "." {
+					base.Errorf("go: path %s is not a module", m.Pattern)
+					continue
 				}
+				m.Pkgs = matchPackages("...", loaded.tags, false, []module.Version{Target})
+
+			case strings.Contains(m.Pattern, "..."):
+				match := search.MatchPattern(m.Pattern)
+				var matched []module.Version
+				for _, mod := range buildList {
+					if match(mod.Path) || str.HasPathPrefix(m.Pattern, mod.Path) {
+						matched = append(matched, mod)
 					}
 				}
+				m.Pkgs = matchPackages(m.Pattern, loaded.tags, false, matched)
 
+			case m.Pattern == "all":
+				loaded.testAll = true
+				if iterating {
+					// Enumerate the packages in the main module.
+					// We'll load the dependencies as we find them.
+					m.Pkgs = matchPackages("...", loaded.tags, false, []module.Version{Target})
+				} else {
+					// Starting with the packages in the main module,
+					// enumerate the full list of "all".
+					m.Pkgs = loaded.computePatternAll(m.Pkgs)
+				}
+
+			default:
+				found := false
+				for _, mod := range buildList {
+					if mod.Path == m.Pattern {
+						found = true
+						m.Pkgs = matchPackages("...", loaded.tags, false, []module.Version{mod})
+						break
+					}
+				}
+				if !found {
+					base.Errorf("go %s: module not in build list", m.Pattern)
+				}
+			}
+		}
+	}
+	return loadPatterns(patterns, false, updateMatches)
+}
+
+// loadPatterns returns a set of packages matching the args (patterns),
+// adding modules to the build list as needed to satisfy new imports.
+//
+// useTags indicates whether to use the default build constraints to
+// filter source files. If useTags is false, only "ignore" and malformed
+// build tag requirements are considered false.
+//
+// The interpretation of patterns is determined by updateMatches, which will be
+// called repeatedly until the build list is finalized. updateMatches should
+// should store a list of matching packages in each search.Match when it is
+// called. The iterating parameter is true if the build list has not been
+// finalized yet.
+//
+// If errors are encountered, loadPatterns will print them and exit.
+// On success, loadPatterns will update the build list and write go.mod.
+func loadPatterns(patterns []string, useTags bool, updateMatches func(matches []*search.Match, iterating bool)) []*search.Match {
+	InitMod()
+
+	var matches []*search.Match
+	for _, pattern := range search.CleanPatterns(patterns) {
+		matches = append(matches, &search.Match{
+			Pattern: pattern,
+			Literal: !strings.Contains(pattern, "...") && !search.IsMetaPackage(pattern),
+		})
+	}
+
+	loaded = newLoader()
+	if !useTags {
+		loaded.tags = anyTags
+	}
 	loaded.load(func() []string {
 		var roots []string
-		updateMatches(true)
+		updateMatches(matches, true)
 		for _, m := range matches {
-			for _, pkg := range m.Pkgs {
-				roots = append(roots, pkg)
-			}
+			roots = append(roots, m.Pkgs...)
 		}
 		return roots
 	})
 
 	// One last pass to finalize wildcards.
-	updateMatches(false)
+	updateMatches(matches, false)
 
 	// A given module path may be used as itself or as a replacement for another
 	// module, but not both at the same time. Otherwise, the aliasing behavior is
@@ -204,7 +290,6 @@ func ImportPaths(patterns []string) []*search.Match {
 	base.ExitIfErrors()
 	WriteGoMod()
 
-	search.WarnUnmatched(matches)
 	return matches
 }
 
@@ -410,6 +495,20 @@ func PackageModule(path string) module.Version {
 	return pkg.mod
 }
 
+// PackageImports returns the imports for the package named by the import path.
+// It does not include test imports. It returns nil for unknown packages.
+func PackageImports(path string) []string {
+	pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
+	if !ok {
+		return nil
+	}
+	imports := make([]string, len(pkg.imports))
+	for i, p := range pkg.imports {
+		imports[i] = p.path
+	}
+	return imports
+}
+
 // ModuleUsedDirectly reports whether the main module directly imports
 // some package in the module with the given path.
 func ModuleUsedDirectly(path string) bool {

src/cmd/go/internal/modload/query.go

@@ -442,3 +442,13 @@ func (e *packageNotInModuleError) Error() string {
 	}
 	return fmt.Sprintf("module %s@%s%s found, but does not contain package %s", e.mod.Path, e.query, found, e.pattern)
 }
+
+// ModuleHasRootPackage returns whether module m contains a package m.Path.
+func ModuleHasRootPackage(m module.Version) (bool, error) {
+	root, isLocal, err := fetch(m)
+	if err != nil {
+		return false, err
+	}
+	_, ok := dirInModule(m.Path, m.Path, root, isLocal)
+	return ok, nil
+}

src/cmd/go/testdata/script/mod_get_indirect.txt

 env GO111MODULE=on
 
-# get -u should find quote v1.5.2
+# get -u should not upgrade anything, since the package
+# in the current directory doesn't import anything.
+go get -u
+go list -m all
+stdout 'quote v1.5.1$'
+grep 'rsc.io/quote v1.5.1$' go.mod
+
+# get -u should find quote v1.5.2 once there is a use.
+cp $WORK/tmp/usequote.go x.go
 go get -u
 go list -m all
 stdout 'quote v1.5.2$'
@@ -11,11 +19,10 @@ go list -m -f '{{.Path}} {{.Version}}{{if .Indirect}} // indirect{{end}}' all
 stdout '^golang.org/x/text [v0-9a-f\.-]+ // indirect'
 grep 'golang.org/x/text [v0-9a-f\.-]+ // indirect' go.mod
 
-# importing an empty module root as a package makes it direct.
-# TODO(bcmills): This doesn't seem correct. Fix is in the next change.
+# importing an empty module root as a package does not remove indirect tag.
 cp $WORK/tmp/usetext.go x.go
 go list -e
-grep 'golang.org/x/text [v0-9a-f\.-]+ // indirect' go.mod
+grep 'golang.org/x/text v0.3.0 // indirect$' go.mod
 
 # indirect tag should be removed upon seeing direct import.
 cp $WORK/tmp/uselang.go x.go

src/cmd/go/testdata/script/mod_get_local.txt

@@ -22,28 +22,24 @@ cp go.mod.orig go.mod
 go get -u -m local@patch
 cmp go.mod go.mod.implicitmod
 
-# 'go get -u -d' in the empty root of the main module should update the
-# dependencies of all packages in the module.
+# 'go get -u -d' in the empty root of the main module should fail.
+# 'go get -u -d .' should also fail.
 cp go.mod.orig go.mod
-go get -u -d
-cmp go.mod go.mod.implicitmod
+! go get -u -d
+! go get -u -d .
 
-# 'go get -u -d .' within a package in the main module updates all dependencies
-# of the main module.
-# TODO: Determine whether that behavior is a bug.
-# (https://golang.org/issue/26902)
+# 'go get -u -d .' within a package in the main module updates the dependencies
+# of that package.
 cp go.mod.orig go.mod
 cd uselang
 go get -u -d .
 cd ..
-grep 'rsc.io/quote.*v1.5.2' go.mod
+grep 'rsc.io/quote.*v1.3.0' go.mod
 grep 'golang.org/x/text.*v0.3.0' go.mod
 cp go.mod go.mod.dotpkg
 
-# 'go get -u -d' with an explicit package in the main module updates
-# all dependencies of the main module.
-# TODO: Determine whether that behavior is a bug.
-# (https://golang.org/issue/26902)
+# 'go get -u -d' with an explicit package in the main module updates the
+# dependencies of that package.
 cp go.mod.orig go.mod
 go get -u -d local/uselang
 cmp go.mod go.mod.dotpkg

src/cmd/go/testdata/script/mod_get_main.txt

 env GO111MODULE=on
 
-# @patch and @latest within the main module refer to the current version, and
-# are no-ops.
+# @patch and @latest within the main module refer to the current version.
+# The main module won't be upgraded, but missing dependencies will be added.
 cp go.mod.orig go.mod
 go get -m rsc.io@latest
+grep 'rsc.io/quote v1.5.2' go.mod
+cp go.mod.orig go.mod
 go get -m rsc.io@patch
-cmp go.mod go.mod.orig
+grep 'rsc.io/quote v1.5.2' go.mod
+cp go.mod.orig go.mod
 
 # The main module cannot be updated to a specific version.
-cp go.mod.orig go.mod
 ! go get -m rsc.io@v0.1.0
 stderr '^go get rsc.io@v0.1.0: can.t get a specific version of the main module$'
 ! go get -d rsc.io/x@v0.1.0

src/cmd/go/testdata/script/mod_get_upgrade.txt

@@ -5,8 +5,8 @@ go list -m all
 stdout 'rsc.io/quote v1.5.1'
 grep 'rsc.io/quote v1.5.1$' go.mod
 
-# get -u should update all dependencies
-go get -u
+# get -m -u should update all dependencies
+go get -m -u
 grep 'rsc.io/quote v1.5.2$' go.mod
 grep 'golang.org/x/text [v0-9a-f\.-]+ // indirect' go.mod
 
@@ -39,3 +39,8 @@ require rsc.io/quote v1.1.0
 -- go.mod-v1.5.1 --
 module x
 require rsc.io/quote v1.5.1
+
+-- use.go --
+package use
+
+import _ "rsc.io/quote"

src/cmd/go/testdata/script/mod_load_badchain.txt

@@ -16,7 +16,7 @@ cmp go.mod go.mod.orig
 
 # Try to update the main module. This updates everything, including
 # modules that aren't direct requirements, so the error stack is shorter.
-! go get -u
+! go get -m -u
 cmp stderr update-main-expected
 cmp go.mod go.mod.orig
 

src/cmd/go/testdata/script/mod_load_badzip.txt

@@ -3,7 +3,11 @@ env GO111MODULE=on
 
 ! go get -d rsc.io/badzip
 stderr 'zip for rsc.io/badzip@v1.0.0 has unexpected file rsc.io/badzip@v1.0.0.txt'
+! grep rsc.io/badzip go.mod
 
+# TODO(golang.org/issue/31730): 'go build' should print the error below if the
+# requirement is not present.
+go mod edit -require rsc.io/badzip@v1.0.0
 ! go build rsc.io/badzip
 stderr 'zip for rsc.io/badzip@v1.0.0 has unexpected file rsc.io/badzip@v1.0.0.txt'
 

src/cmd/go/testdata/script/mod_upgrade_patch_mod.txt

@@ -9,15 +9,12 @@ stdout '^patch.example.com/indirect v1.0.0'
 
 # get -m -u=patch, with no arguments, should patch-update all dependencies,
 # pulling in transitive dependencies and also patching those.
-#
-# TODO(golang.org/issue/26902): We should not update transitive dependencies
-# that don't affect the transitive import graph of the main module in any way.
 cp go.mod.orig go.mod
 go get -m -u=patch
 go list -m all
 stdout '^patch.example.com/direct v1.0.1'
 stdout '^patch.example.com/indirect v1.0.1'
-stdout '^patch.example.com/depofdirectpatch v1.0.1' # TODO: leave at v1.0.0
+stdout '^patch.example.com/depofdirectpatch v1.0.0'
 
 # 'get -m all@patch' should be equivalent to 'get -u=patch -m all'
 cp go.mod.orig go.mod
@@ -34,7 +31,7 @@ go get -m -u=patch patch.example.com/direct
 go list -m all
 stdout '^patch.example.com/direct v1.0.1'
 stdout '^patch.example.com/indirect v1.0.1'
-stdout '^patch.example.com/depofdirectpatch v1.0.1' # TODO: leave at v1.0.0
+stdout '^patch.example.com/depofdirectpatch v1.0.1'
 
 # Requesting only the indirect dependency should not update the direct one.
 cp go.mod.orig go.mod

src/cmd/go/testdata/script/mod_upgrade_patch_pkg.txt

@@ -7,17 +7,15 @@ stdout '^patch.example.com/direct v1.0.0'
 stdout '^patch.example.com/indirect v1.0.0'
 ! stdout '^patch.example.com/depofdirectpatch'
 
-# get -u=patch, with no arguments, should patch-update all dependencies,
-# pulling in transitive dependencies and also patching those.
-#
-# TODO(golang.org/issue/26902): We should not update dependencies
-# that don't affect the transitive import graph of the main module in any way.
+# get -u=patch, with no arguments, should patch-update all dependencies
+# of the package in the current directory, pulling in transitive dependencies
+# and also patching those.
 cp go.mod.orig go.mod
 go get -u=patch
 go list -m all
 stdout '^patch.example.com/direct v1.0.1'
 stdout '^patch.example.com/indirect v1.0.1'
-stdout '^patch.example.com/depofdirectpatch v1.0.1' # TODO: leave at v1.0.0
+stdout '^patch.example.com/depofdirectpatch v1.0.0'
 
 # 'get all@patch' should be equivalent to 'get -u=patch all'
 cp go.mod.orig go.mod
@@ -34,7 +32,7 @@ go get -u=patch patch.example.com/direct
 go list -m all
 stdout '^patch.example.com/direct v1.0.1'
 stdout '^patch.example.com/indirect v1.0.1'
-stdout '^patch.example.com/depofdirectpatch v1.0.1' # TODO: leave at v1.0.0
+stdout '^patch.example.com/depofdirectpatch v1.0.0'
 
 # Requesting only the indirect dependency should not update the direct one.
 cp go.mod.orig go.mod
@@ -59,7 +57,7 @@ go get -u patch.example.com/direct@patch
 go list -m all
 stdout '^patch.example.com/direct v1.0.1'
 stdout '^patch.example.com/indirect v1.1.0'
-stdout '^patch.example.com/depofdirectpatch v1.0.1'
+stdout '^patch.example.com/depofdirectpatch v1.0.0'
 
 # An explicit @latest should override a general -u=patch.
 cp go.mod.orig go.mod
@@ -75,7 +73,7 @@ cp go.mod.orig go.mod
 stderr 'cannot use pattern .* with explicit version'
 
 # However, standard-library packages without explicit versions are fine.
-go get -u=patch -d cmd/get
+go get -u=patch -d cmd/go
 
 
 -- go.mod --