cmd/go: make commands other than 'tidy' prune go.mod less agressively

Updates #31870
Updates #33326
Fixes #34822

Change-Id: I1337f171133c20800eacc6b0955ede5a394ea7eb
Reviewed-on: https://go-review.googlesource.com/c/go/+/204878
Run-TryBot: Bryan C. Mills <bcmills@google.com>
Reviewed-by: Jay Conrod <jayconrod@google.com>

src/cmd/go/internal/modcmd/tidy.go

@@ -7,9 +7,6 @@
 package modcmd
 
 import (
-	"fmt"
-	"os"
-
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/modfetch"
@@ -45,28 +42,8 @@ func runTidy(cmd *base.Command, args []string) {
 		base.Fatalf("go mod tidy: no arguments allowed")
 	}
 
-	// LoadALL adds missing modules.
-	// Remove unused modules.
-	used := make(map[module.Version]bool)
-	for _, pkg := range modload.LoadALL() {
-		used[modload.PackageModule(pkg)] = true
-	}
-	used[modload.Target] = true // note: LoadALL initializes Target
-
-	inGoMod := make(map[string]bool)
-	for _, r := range modload.ModFile().Require {
-		inGoMod[r.Mod.Path] = true
-	}
-
-	var keep []module.Version
-	for _, m := range modload.BuildList() {
-		if used[m] {
-			keep = append(keep, m)
-		} else if cfg.BuildV && inGoMod[m.Path] {
-			fmt.Fprintf(os.Stderr, "unused %s\n", m.Path)
-		}
-	}
-	modload.SetBuildList(keep)
+	modload.LoadALL()
+	modload.TidyBuildList()
 	modTidyGoSum() // updates memory copy; WriteGoMod on next line flushes it out
 	modload.WriteGoMod()
 }

src/cmd/go/internal/modload/load.go

@@ -428,6 +428,37 @@ func SetBuildList(list []module.Version) {
 	buildList = append([]module.Version{}, list...)
 }
 
+// TidyBuildList trims the build list to the minimal requirements needed to
+// retain the same versions of all packages from the preceding Load* or
+// ImportPaths* call.
+func TidyBuildList() {
+	used := map[module.Version]bool{Target: true}
+	for _, pkg := range loaded.pkgs {
+		used[pkg.mod] = true
+	}
+
+	keep := []module.Version{Target}
+	var direct []string
+	for _, m := range buildList[1:] {
+		if used[m] {
+			keep = append(keep, m)
+			if loaded.direct[m.Path] {
+				direct = append(direct, m.Path)
+			}
+		} else if cfg.BuildV {
+			if _, ok := index.require[m]; ok {
+				fmt.Fprintf(os.Stderr, "unused %s\n", m.Path)
+			}
+		}
+	}
+
+	min, err := mvs.Req(Target, direct, &mvsReqs{buildList: keep})
+	if err != nil {
+		base.Fatalf("go: %v", err)
+	}
+	buildList = append([]module.Version{Target}, min...)
+}
+
 // ImportMap returns the actual package import path
 // for an import path found in source code.
 // If the given import path does not appear in the source code
@@ -966,21 +997,15 @@ func WhyDepth(path string) int {
 // If there is no replacement for mod, Replacement returns
 // a module.Version with Path == "".
 func Replacement(mod module.Version) module.Version {
-	if modFile == nil {
-		// Happens during testing and if invoking 'go get' or 'go list' outside a module.
-		return module.Version{}
-	}
-
-	var found *modfile.Replace
-	for _, r := range modFile.Replace {
-		if r.Old.Path == mod.Path && (r.Old.Version == "" || r.Old.Version == mod.Version) {
-			found = r // keep going
+	if index != nil {
+		if r, ok := index.replace[mod]; ok {
+			return r
+		}
+		if r, ok := index.replace[module.Version{Path: mod.Path}]; ok {
+			return r
 		}
 	}
-	if found == nil {
-		return module.Version{}
-	}
-	return found.New
+	return module.Version{}
 }
 
 // mvsReqs implements mvs.Reqs for module semantic versions,
@@ -1013,15 +1038,17 @@ func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) {
 			return cached{nil, err}
 		}
 		for i, mv := range list {
-			for excluded[mv] {
-				mv1, err := r.next(mv)
-				if err != nil {
-					return cached{nil, err}
-				}
-				if mv1.Version == "none" {
-					return cached{nil, fmt.Errorf("%s(%s) depends on excluded %s(%s) with no newer version available", mod.Path, mod.Version, mv.Path, mv.Version)}
+			if index != nil {
+				for index.exclude[mv] {
+					mv1, err := r.next(mv)
+					if err != nil {
+						return cached{nil, err}
+					}
+					if mv1.Version == "none" {
+						return cached{nil, fmt.Errorf("%s(%s) depends on excluded %s(%s) with no newer version available", mod.Path, mod.Version, mv.Path, mv.Version)}
+					}
+					mv = mv1
 				}
-				mv = mv1
 			}
 			list[i] = mv
 		}

src/cmd/go/testdata/script/mod_readonly.txt

@@ -2,9 +2,6 @@ env GO111MODULE=on
 [short] skip
 
 # -mod=readonly must not resolve missing modules nor update go.mod
-#
-# TODO(bcmills): 'go list' should suffice, but today it does not fail due to
-# unresolved imports. When that is fixed, use 'go list' instead of 'go list all'.
 env GOFLAGS=-mod=readonly
 go mod edit -fmt
 cp go.mod go.mod.empty
@@ -27,6 +24,7 @@ grep rsc.io/quote go.mod
 # update go.mod - go mod tidy allowed
 cp go.mod.empty go.mod
 go mod tidy
+cp go.mod go.mod.tidy
 
 # -mod=readonly must succeed once go.mod is up-to-date...
 go list all
@@ -43,6 +41,19 @@ cp go.mod go.mod.inconsistent
 stderr 'go: updates to go.mod needed, disabled by -mod=readonly'
 cmp go.mod go.mod.inconsistent
 
+# However, it should not reject files missing a 'go' directive,
+# since that was not always required.
+cp go.mod.nogo go.mod
+go list all
+
+# Nor should it reject files with redundant (not incorrect)
+# requirements.
+cp go.mod.redundant go.mod
+go list all
+
+cp go.mod.indirect go.mod
+go list all
+
 -- go.mod --
 module m
 
@@ -51,3 +62,30 @@ go 1.20
 -- x.go --
 package x
 import _ "rsc.io/quote"
+-- go.mod.nogo --
+module m
+
+require (
+	rsc.io/quote v1.5.2
+	rsc.io/testonly v1.0.0 // indirect
+)
+-- go.mod.redundant --
+module m
+
+go 1.20
+
+require (
+	rsc.io/quote v1.5.2
+	rsc.io/sampler v1.3.0 // indirect
+	rsc.io/testonly v1.0.0 // indirect
+)
+-- go.mod.indirect --
+module m
+
+go 1.20
+
+require (
+	rsc.io/quote v1.5.2 // indirect
+	rsc.io/sampler v1.3.0 // indirect
+	rsc.io/testonly v1.0.0 // indirect
+)

src/cmd/go/testdata/script/mod_retention.txt

+# Regression test for golang.org/issue/34822: the 'go' command should prefer not
+# to update the go.mod file if the changes only affect formatting, and should only
+# remove redundant requirements in 'go mod tidy'.
+
+env GO111MODULE=on
+[short] skip
+
+# Control case: verify that go.mod.tidy is actually tidy.
+cp go.mod.tidy go.mod
+go list all
+cmp go.mod go.mod.tidy
+
+
+# If the only difference in the go.mod file is the line endings,
+# it should not be overwritten automatically.
+cp go.mod.crlf go.mod
+go list all
+cmp go.mod go.mod.crlf
+
+# However, 'go mod tidy' should fix whitespace even if there are no other changes.
+go mod tidy
+cmp go.mod go.mod.tidy
+
+
+# Out-of-order requirements should not be overwritten automatically...
+cp go.mod.unsorted go.mod
+go list all
+cmp go.mod go.mod.unsorted
+
+# ...but 'go mod edit -fmt' should sort them.
+go mod edit -fmt
+cmp go.mod go.mod.tidy
+
+
+# "// indirect" comments should be removed if direct dependencies are seen.
+# changes.
+cp go.mod.indirect go.mod
+go list all
+cmp go.mod go.mod.tidy
+
+# "// indirect" comments should be added if appropriate.
+cp go.mod.toodirect go.mod
+go list all
+cmp go.mod go.mod.toodirect
+go mod vendor # loads everything, so adds "// indirect" comments.
+cmp go.mod go.mod.tidy
+rm -r vendor
+
+
+# Redundant requirements should be preserved...
+cp go.mod.redundant go.mod
+go list all
+cmp go.mod go.mod.redundant
+go mod vendor
+cmp go.mod go.mod.redundant
+rm -r vendor
+
+# ...except by 'go mod tidy'.
+go mod tidy
+cmp go.mod go.mod.tidy
+
+
+# A missing "go" version directive should be added.
+# However, that should not remove other redundant requirements.
+cp go.mod.nogo go.mod
+go list all
+cmp go.mod go.mod.redundant
+
+
+-- go.mod.tidy --
+module m
+
+go 1.14
+
+require (
+	rsc.io/quote v1.5.2
+	rsc.io/testonly v1.0.0 // indirect
+)
+-- x.go --
+package x
+import _ "rsc.io/quote"
+-- go.mod.crlf --
+module m
+
+go 1.14
+
+require (
+	rsc.io/quote v1.5.2
+	rsc.io/testonly v1.0.0 // indirect
+)
+-- go.mod.unsorted --
+module m
+
+go 1.14
+
+require (
+	rsc.io/testonly v1.0.0 // indirect
+	rsc.io/quote v1.5.2
+)
+-- go.mod.indirect --
+module m
+
+go 1.14
+
+require (
+	rsc.io/quote v1.5.2 // indirect
+	rsc.io/testonly v1.0.0 // indirect
+)
+-- go.mod.toodirect --
+module m
+
+go 1.14
+
+require (
+	rsc.io/quote v1.5.2
+	rsc.io/testonly v1.0.0
+)
+-- go.mod.redundant --
+module m
+
+go 1.14
+
+require (
+	rsc.io/quote v1.5.2
+	rsc.io/sampler v1.3.0 // indirect
+	rsc.io/testonly v1.0.0 // indirect
+)
+-- go.mod.nogo --
+module m
+
+require (
+	rsc.io/quote v1.5.2
+	rsc.io/sampler v1.3.0 // indirect
+	rsc.io/testonly v1.0.0 // indirect
+)

src/cmd/go/testdata/script/mod_tidy.txt

@@ -5,7 +5,6 @@ go mod tidy -v
 stderr '^unused y.1'
 ! stderr '^unused [^y]'
 
-# tidy should not touch existing go line
 grep 'go 1.10' go.mod
 
 go list -m all