Commit c66ab9b1 authored by Bryan C. Mills's avatar Bryan C. Mills

cmd/go: query modules in parallel

Refactor modload.QueryPackage and modload.QueryPattern to share code.

Fine-tune error reporting and make it consistent between QueryPackage and QueryPattern.

Expand tests for pattern errors.

Update a TODO in modget/get.go and add a test case that demonstrates it.

Updates #26232

Change-Id: I900ca8de338ef9a51b7f85ed93d8bcf837621646
Reviewed-on: https://go-review.googlesource.com/c/go/+/173017
Run-TryBot: Bryan C. Mills <bcmills@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: default avatarJay Conrod <jayconrod@google.com>
parent 8e4f1a71
...@@ -19,6 +19,7 @@ import ( ...@@ -19,6 +19,7 @@ import (
"cmd/go/internal/semver" "cmd/go/internal/semver"
"cmd/go/internal/str" "cmd/go/internal/str"
"cmd/go/internal/work" "cmd/go/internal/work"
"errors"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
...@@ -351,10 +352,17 @@ func runGet(cmd *base.Command, args []string) { ...@@ -351,10 +352,17 @@ func runGet(cmd *base.Command, args []string) {
match := search.MatchPattern(path) match := search.MatchPattern(path)
matched := false matched := false
for _, m := range modload.BuildList() { for _, m := range modload.BuildList() {
// TODO(bcmills): Patterns that don't contain the module path but do // TODO: If we have matching packages already in the build list and we
// contain partial package paths will not match here. For example, // know which module(s) they are in, then we should not upgrade the
// ...html/... would not match html/template or golang.org/x/net/html. // modules that do *not* contain those packages, even if the module path
// Related golang.org/issue/26902. // is a prefix of the pattern.
//
// For example, if we have modules golang.org/x/tools and
// golang.org/x/tools/playground, and all of the packages matching
// golang.org/x/tools/playground... are in the
// golang.org/x/tools/playground module, then we should not *also* try
// to upgrade golang.org/x/tools if the user says 'go get
// golang.org/x/tools/playground...@latest'.
if match(m.Path) || str.HasPathPrefix(path, m.Path) { if match(m.Path) || str.HasPathPrefix(path, m.Path) {
tasks = append(tasks, &task{arg: arg, path: m.Path, vers: vers, prevM: m, forceModulePath: true}) tasks = append(tasks, &task{arg: arg, path: m.Path, vers: vers, prevM: m, forceModulePath: true})
matched = true matched = true
...@@ -650,29 +658,31 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo ...@@ -650,29 +658,31 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo
} }
} }
// If the path has a wildcard, search for a module that matches the pattern. if forceModulePath || *getM || !strings.Contains(path, "...") {
if strings.Contains(path, "...") { if path == modload.Target.Path {
if forceModulePath { if vers != "latest" {
panic("forceModulePath is true for path with wildcard " + path) return module.Version{}, fmt.Errorf("can't get a specific version of the main module")
}
}
// If the path doesn't contain a wildcard, try interpreting it as a module path.
info, err := modload.Query(path, vers, modload.Allowed)
if err == nil {
return module.Version{Path: path, Version: info.Version}, nil
} }
_, m, _, err := modload.QueryPattern(path, vers, modload.Allowed)
return m, err
}
// Try interpreting the path as a module path. // If the query fails, and the path must be a real module, report the query error.
info, err := modload.Query(path, vers, modload.Allowed) if forceModulePath || *getM {
if err == nil { return module.Version{}, err
return module.Version{Path: path, Version: info.Version}, nil }
} }
// If the query fails, and the path must be a real module, report the query error. // Otherwise, try a package path or pattern.
if forceModulePath || *getM { results, err := modload.QueryPattern(path, vers, modload.Allowed)
if err != nil {
return module.Version{}, err return module.Version{}, err
} }
return results[0].Mod, nil
// Otherwise, try a package path.
m, _, err := modload.QueryPackage(path, vers, modload.Allowed)
return m, err
} }
// An upgrader adapts an underlying mvs.Reqs to apply an // An upgrader adapts an underlying mvs.Reqs to apply an
...@@ -736,7 +746,8 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { ...@@ -736,7 +746,8 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) {
// even report the error. Because Query does not consider pseudo-versions, // even report the error. Because Query does not consider pseudo-versions,
// it may happen that we have a pseudo-version but during -u=patch // it may happen that we have a pseudo-version but during -u=patch
// the query v0.0 matches no versions (not even the one we're using). // the query v0.0 matches no versions (not even the one we're using).
if !strings.Contains(err.Error(), "no matching versions") { var noMatch *modload.NoMatchingVersionError
if !errors.As(err, &noMatch) {
base.Errorf("go get: upgrading %s@%s: %v", m.Path, m.Version, err) base.Errorf("go get: upgrading %s@%s: %v", m.Path, m.Version, err)
} }
return m, nil return m, nil
......
...@@ -186,22 +186,31 @@ func Import(path string) (m module.Version, dir string, err error) { ...@@ -186,22 +186,31 @@ func Import(path string) (m module.Version, dir string, err error) {
} }
} }
m, _, err = QueryPackage(path, "latest", Allowed) candidates, err := QueryPackage(path, "latest", Allowed)
if err != nil { if err != nil {
if _, ok := err.(*codehost.VCSError); ok { if _, ok := err.(*codehost.VCSError); ok {
return module.Version{}, "", err return module.Version{}, "", err
} }
return module.Version{}, "", &ImportMissingError{ImportPath: path} return module.Version{}, "", &ImportMissingError{ImportPath: path}
} }
m = candidates[0].Mod
newMissingVersion := "" newMissingVersion := ""
for _, bm := range buildList { for _, c := range candidates {
if bm.Path == m.Path && semver.Compare(bm.Version, m.Version) > 0 { cm := c.Mod
// This typically happens when a package is present at the "@latest" for _, bm := range buildList {
// version (e.g., v1.0.0) of a module, but we have a newer version if bm.Path == cm.Path && semver.Compare(bm.Version, cm.Version) > 0 {
// of the same module in the build list (e.g., v1.0.1-beta), and // QueryPackage proposed that we add module cm to provide the package,
// the package is not present there. // but we already depend on a newer version of that module (and we don't
newMissingVersion = bm.Version // have the package).
break //
// This typically happens when a package is present at the "@latest"
// version (e.g., v1.0.0) of a module, but we have a newer version
// of the same module in the build list (e.g., v1.0.1-beta), and
// the package is not present there.
m = cm
newMissingVersion = bm.Version
break
}
} }
} }
return m, "", &ImportMissingError{ImportPath: path, Module: m, newMissingVersion: newMissingVersion} return m, "", &ImportMissingError{ImportPath: path, Module: m, newMissingVersion: newMissingVersion}
......
This diff is collapsed.
...@@ -17,6 +17,11 @@ cp go.mod.orig go.mod ...@@ -17,6 +17,11 @@ cp go.mod.orig go.mod
go get -u -m local go get -u -m local
cmp go.mod go.mod.implicitmod cmp go.mod go.mod.implicitmod
# For the main module, @patch should be a no-op.
cp go.mod.orig go.mod
go get -u -m local@patch
cmp go.mod go.mod.implicitmod
# 'go get -u -d' in the empty root of the main module should update the # 'go get -u -d' in the empty root of the main module should update the
# dependencies of all packages in the module. # dependencies of all packages in the module.
cp go.mod.orig go.mod cp go.mod.orig go.mod
...@@ -43,7 +48,6 @@ cp go.mod.orig go.mod ...@@ -43,7 +48,6 @@ cp go.mod.orig go.mod
go get -u -d local/uselang go get -u -d local/uselang
cmp go.mod go.mod.dotpkg cmp go.mod go.mod.dotpkg
-- go.mod -- -- go.mod --
module local module local
......
env GO111MODULE=on
# @patch and @latest within the main module refer to the current version, and
# are no-ops.
cp go.mod.orig go.mod
go get -m rsc.io@latest
go get -m rsc.io@patch
cmp go.mod go.mod.orig
# The main module cannot be updated to a specific version.
cp go.mod.orig go.mod
! go get -m rsc.io@v0.1.0
stderr '^go get rsc.io@v0.1.0: can.t get a specific version of the main module$'
! go get -d rsc.io/x@v0.1.0
stderr '^go get rsc.io/x@v0.1.0: can.t query specific version for package rsc.io/x in the main module \(rsc.io\)$'
# TODO: upgrading a package pattern not contained in the main module should not
# attempt to upgrade the main module.
! go get rsc.io/quote/...@v1.5.1
-- go.mod.orig --
module rsc.io
go 1.13
-- x/x.go --
package x
import _ "rsc.io/quote"
...@@ -15,11 +15,11 @@ grep 'require rsc.io/quote' go.mod ...@@ -15,11 +15,11 @@ grep 'require rsc.io/quote' go.mod
cp go.mod.orig go.mod cp go.mod.orig go.mod
! go get -d rsc.io/quote/x... ! go get -d rsc.io/quote/x...
stderr 'go get rsc.io/quote/x...: no matching packages in module rsc.io/quote@v1.5.2' stderr 'go get rsc.io/quote/x...: module rsc.io/quote@latest \(v1.5.2\) found, but does not contain packages matching rsc.io/quote/x...'
! grep 'require rsc.io/quote' go.mod ! grep 'require rsc.io/quote' go.mod
! go get -d rsc.io/quote/x/... ! go get -d rsc.io/quote/x/...
stderr 'go get rsc.io/quote/x/...: no matching packages in module rsc.io/quote@v1.5.2' stderr 'go get rsc.io/quote/x/...: module rsc.io/quote@latest \(v1.5.2\) found, but does not contain packages matching rsc.io/quote/x/...'
! grep 'require rsc.io/quote' go.mod ! grep 'require rsc.io/quote' go.mod
-- go.mod.orig -- -- go.mod.orig --
......
...@@ -4,6 +4,7 @@ env GO111MODULE=on ...@@ -4,6 +4,7 @@ env GO111MODULE=on
# Download everything to avoid "finding" messages in stderr later. # Download everything to avoid "finding" messages in stderr later.
cp go.mod.orig go.mod cp go.mod.orig go.mod
go mod download go mod download
go mod download example.com@v1.0.0
go mod download example.com/badchain/a@v1.1.0 go mod download example.com/badchain/a@v1.1.0
go mod download example.com/badchain/b@v1.1.0 go mod download example.com/badchain/b@v1.1.0
go mod download example.com/badchain/c@v1.1.0 go mod download example.com/badchain/c@v1.1.0
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment