cmd/go: query modules in parallel

Refactor modload.QueryPackage and modload.QueryPattern to share code.

Fine-tune error reporting and make it consistent between QueryPackage and QueryPattern.

Expand tests for pattern errors.

Update a TODO in modget/get.go and add a test case that demonstrates it.

Updates #26232

Change-Id: I900ca8de338ef9a51b7f85ed93d8bcf837621646
Reviewed-on: https://go-review.googlesource.com/c/go/+/173017
Run-TryBot: Bryan C. Mills <bcmills@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>

src/cmd/go/internal/modget/get.go

@@ -19,6 +19,7 @@ import (
 	"cmd/go/internal/semver"
 	"cmd/go/internal/str"
 	"cmd/go/internal/work"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -351,10 +352,17 @@ func runGet(cmd *base.Command, args []string) {
 			match := search.MatchPattern(path)
 			matched := false
 			for _, m := range modload.BuildList() {
-				// TODO(bcmills): Patterns that don't contain the module path but do
-				// contain partial package paths will not match here. For example,
-				// ...html/... would not match html/template or golang.org/x/net/html.
-				// Related golang.org/issue/26902.
+				// TODO: If we have matching packages already in the build list and we
+				// know which module(s) they are in, then we should not upgrade the
+				// modules that do *not* contain those packages, even if the module path
+				// is a prefix of the pattern.
+				//
+				// For example, if we have modules golang.org/x/tools and
+				// golang.org/x/tools/playground, and all of the packages matching
+				// golang.org/x/tools/playground... are in the
+				// golang.org/x/tools/playground module, then we should not *also* try
+				// to upgrade golang.org/x/tools if the user says 'go get
+				// golang.org/x/tools/playground...@latest'.
 				if match(m.Path) || str.HasPathPrefix(path, m.Path) {
 					tasks = append(tasks, &task{arg: arg, path: m.Path, vers: vers, prevM: m, forceModulePath: true})
 					matched = true
@@ -650,29 +658,31 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo
 		}
 	}
 
-	// If the path has a wildcard, search for a module that matches the pattern.
-	if strings.Contains(path, "...") {
-		if forceModulePath {
-			panic("forceModulePath is true for path with wildcard " + path)
+	if forceModulePath || *getM || !strings.Contains(path, "...") {
+		if path == modload.Target.Path {
+			if vers != "latest" {
+				return module.Version{}, fmt.Errorf("can't get a specific version of the main module")
+			}
+		}
+
+		// If the path doesn't contain a wildcard, try interpreting it as a module path.
+		info, err := modload.Query(path, vers, modload.Allowed)
+		if err == nil {
+			return module.Version{Path: path, Version: info.Version}, nil
 		}
-		_, m, _, err := modload.QueryPattern(path, vers, modload.Allowed)
-		return m, err
-	}
 
-	// Try interpreting the path as a module path.
-	info, err := modload.Query(path, vers, modload.Allowed)
-	if err == nil {
-		return module.Version{Path: path, Version: info.Version}, nil
+		// If the query fails, and the path must be a real module, report the query error.
+		if forceModulePath || *getM {
+			return module.Version{}, err
+		}
 	}
 
-	// If the query fails, and the path must be a real module, report the query error.
-	if forceModulePath || *getM {
+	// Otherwise, try a package path or pattern.
+	results, err := modload.QueryPattern(path, vers, modload.Allowed)
+	if err != nil {
 		return module.Version{}, err
 	}
-
-	// Otherwise, try a package path.
-	m, _, err := modload.QueryPackage(path, vers, modload.Allowed)
-	return m, err
+	return results[0].Mod, nil
 }
 
 // An upgrader adapts an underlying mvs.Reqs to apply an
@@ -736,7 +746,8 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) {
 		// even report the error. Because Query does not consider pseudo-versions,
 		// it may happen that we have a pseudo-version but during -u=patch
 		// the query v0.0 matches no versions (not even the one we're using).
-		if !strings.Contains(err.Error(), "no matching versions") {
+		var noMatch *modload.NoMatchingVersionError
+		if !errors.As(err, &noMatch) {
 			base.Errorf("go get: upgrading %s@%s: %v", m.Path, m.Version, err)
 		}
 		return m, nil

src/cmd/go/internal/modload/import.go

@@ -186,22 +186,31 @@ func Import(path string) (m module.Version, dir string, err error) {
 		}
 	}
 
-	m, _, err = QueryPackage(path, "latest", Allowed)
+	candidates, err := QueryPackage(path, "latest", Allowed)
 	if err != nil {
 		if _, ok := err.(*codehost.VCSError); ok {
 			return module.Version{}, "", err
 		}
 		return module.Version{}, "", &ImportMissingError{ImportPath: path}
 	}
+	m = candidates[0].Mod
 	newMissingVersion := ""
-	for _, bm := range buildList {
-		if bm.Path == m.Path && semver.Compare(bm.Version, m.Version) > 0 {
-			// This typically happens when a package is present at the "@latest"
-			// version (e.g., v1.0.0) of a module, but we have a newer version
-			// of the same module in the build list (e.g., v1.0.1-beta), and
-			// the package is not present there.
-			newMissingVersion = bm.Version
-			break
+	for _, c := range candidates {
+		cm := c.Mod
+		for _, bm := range buildList {
+			if bm.Path == cm.Path && semver.Compare(bm.Version, cm.Version) > 0 {
+				// QueryPackage proposed that we add module cm to provide the package,
+				// but we already depend on a newer version of that module (and we don't
+				// have the package).
+				//
+				// This typically happens when a package is present at the "@latest"
+				// version (e.g., v1.0.0) of a module, but we have a newer version
+				// of the same module in the build list (e.g., v1.0.1-beta), and
+				// the package is not present there.
+				m = cm
+				newMissingVersion = bm.Version
+				break
+			}
 		}
 	}
 	return m, "", &ImportMissingError{ImportPath: path, Module: m, newMissingVersion: newMissingVersion}

src/cmd/go/testdata/script/mod_get_local.txt

@@ -17,6 +17,11 @@ cp go.mod.orig go.mod
 go get -u -m local
 cmp go.mod go.mod.implicitmod
 
+# For the main module, @patch should be a no-op.
+cp go.mod.orig go.mod
+go get -u -m local@patch
+cmp go.mod go.mod.implicitmod
+
 # 'go get -u -d' in the empty root of the main module should update the
 # dependencies of all packages in the module.
 cp go.mod.orig go.mod
@@ -43,7 +48,6 @@ cp go.mod.orig go.mod
 go get -u -d local/uselang
 cmp go.mod go.mod.dotpkg
 
-
 -- go.mod --
 module local
 

src/cmd/go/testdata/script/mod_get_main.txt

+env GO111MODULE=on
+
+# @patch and @latest within the main module refer to the current version, and
+# are no-ops.
+cp go.mod.orig go.mod
+go get -m rsc.io@latest
+go get -m rsc.io@patch
+cmp go.mod go.mod.orig
+
+# The main module cannot be updated to a specific version.
+cp go.mod.orig go.mod
+! go get -m rsc.io@v0.1.0
+stderr '^go get rsc.io@v0.1.0: can.t get a specific version of the main module$'
+! go get -d rsc.io/x@v0.1.0
+stderr '^go get rsc.io/x@v0.1.0: can.t query specific version for package rsc.io/x in the main module \(rsc.io\)$'
+
+# TODO: upgrading a package pattern not contained in the main module should not
+# attempt to upgrade the main module.
+! go get rsc.io/quote/...@v1.5.1
+
+-- go.mod.orig --
+module rsc.io
+
+go 1.13
+-- x/x.go --
+package x
+
+import _ "rsc.io/quote"

src/cmd/go/testdata/script/mod_get_patterns.txt

@@ -15,11 +15,11 @@ grep 'require rsc.io/quote' go.mod
 
 cp go.mod.orig go.mod
 ! go get -d rsc.io/quote/x...
-stderr 'go get rsc.io/quote/x...: no matching packages in module rsc.io/quote@v1.5.2'
+stderr 'go get rsc.io/quote/x...: module rsc.io/quote@latest \(v1.5.2\) found, but does not contain packages matching rsc.io/quote/x...'
 ! grep 'require rsc.io/quote' go.mod
 
 ! go get -d rsc.io/quote/x/...
-stderr 'go get rsc.io/quote/x/...: no matching packages in module rsc.io/quote@v1.5.2'
+stderr 'go get rsc.io/quote/x/...: module rsc.io/quote@latest \(v1.5.2\) found, but does not contain packages matching rsc.io/quote/x/...'
 ! grep 'require rsc.io/quote' go.mod
 
 -- go.mod.orig --

src/cmd/go/testdata/script/mod_load_badchain.txt

@@ -4,6 +4,7 @@ env GO111MODULE=on
 # Download everything to avoid "finding" messages in stderr later.
 cp go.mod.orig go.mod
 go mod download
+go mod download example.com@v1.0.0
 go mod download example.com/badchain/a@v1.1.0
 go mod download example.com/badchain/b@v1.1.0
 go mod download example.com/badchain/c@v1.1.0