cmd/go: switch to entirely content-based staleness determination

This CL changes the go command to base all its rebuilding decisions
on the content of the files being processed and not their file system
modification times. It also eliminates the special handling of release
toolchains, which were previously considered always up-to-date
because modification time order could not be trusted when unpacking
a pre-built release.

The go command previously tracked "build IDs" as a backup to
modification times, to catch changes not reflected in modification times.
For example, if you remove one .go file in a package with multiple .go
files, there is no modification time remaining in the system that indicates
that the installed package is out of date. The old build ID was the hash
of a list of file names and a few other factors, expected to change if
those factors changed.

This CL moves to using this kind of build ID as the only way to
detect staleness, making sure that the build ID hash includes all
possible factors that need to influence the rebuild decision.

One such factor is the compiler flags. As of this CL, if you run

	go build -gcflags -N cmd/gofmt

you will get a gofmt where every package is built with -N,
regardless of what may or may not be installed already.

Another such factor is the linker flags. As of this CL, if you run

	go install myprog
	go install -ldflags=-s myprog

the second go install will now correctly build a new myprog with
the updated linker flags. (Previously the installed myprog appeared
up-to-date, because the ldflags were not included in the build ID.)

Because we have more precise information we can also validate whether
the target of a "go test -c" operation is already the right binary and
therefore can avoid a rebuild.

This CL sets us up for having a more general build artifact cache,
maybe even a step toward not having a pkg directory with .a files,
but this CL does not take that step. For now the result of go install
is the same as it ever was; we just do a better job of what needs to
be installed.

This CL does slow down builds a small amount by reading all the
dependent source files in full. (The go command already read the
beginning of every dependent source file to discover build tags
and imports.) On my MacBook Pro, before this CL all.bash takes
3m58s, while after this CL and a few optimizations stacked above it
all.bash takes 4m28s. Given that CL 73850 cut 1m43s off the all.bash
time earlier today, we can afford adding 30s back for now.
More optimizations are planned that should make the go command
more efficient than it was even before this CL.

Fixes #15799.
Fixes #18369.
Fixes #19340.
Fixes #21477.

Change-Id: I10d7ca0e31ca3f58aabb9b1f11e2e3d9d18f0bc9
Reviewed-on: https://go-review.googlesource.com/73212
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: David Crawshaw <crawshaw@golang.org>

misc/cgo/testshared/shared_test.go

@@ -598,6 +598,7 @@ func TestThreeGopathShlibs(t *testing.T) {
 // If gccgo is not available or not new enough call t.Skip. Otherwise,
 // return a build.Context that is set up for gccgo.
 func prepGccgo(t *testing.T) build.Context {
+	t.Skip("golang.org/issue/22472")
 	gccgoName := os.Getenv("GCCGO")
 	if gccgoName == "" {
 		gccgoName = "gccgo"

src/cmd/dist/build.go

@@ -1018,7 +1018,9 @@ func cmdenv() {
 // that setting, not the new one.
 func cmdbootstrap() {
 	var noBanner bool
+	var debug bool
 	flag.BoolVar(&rebuildall, "a", rebuildall, "rebuild all")
+	flag.BoolVar(&debug, "d", debug, "enable debugging of bootstrap process")
 	flag.BoolVar(&noBanner, "no-banner", noBanner, "do not print banner")
 
 	xflagparse(0)
@@ -1055,31 +1057,7 @@ func cmdbootstrap() {
 	os.Setenv("GOARCH", goarch)
 	os.Setenv("GOOS", goos)
 
-	// TODO(rsc): Enable when appropriate.
-	// This step is only needed if we believe that the Go compiler built from Go 1.4
-	// will produce different object files than the Go compiler built from itself.
-	// In the absence of bugs, that should not happen.
-	// And if there are bugs, they're more likely in the current development tree
-	// than in a standard release like Go 1.4, so don't do this rebuild by default.
-	if false {
-		xprintf("##### Building Go toolchain using itself.\n")
-		for _, dir := range buildlist {
-			installed[dir] = make(chan struct{})
-		}
-		var wg sync.WaitGroup
-		for _, dir := range builddeps["cmd/go"] {
-			wg.Add(1)
-			dir := dir
-			go func() {
-				defer wg.Done()
-				install(dir)
-			}()
-		}
-		wg.Wait()
-		xprintf("\n")
-	}
-
-	xprintf("##### Building go_bootstrap for host, %s/%s.\n", gohostos, gohostarch)
+	xprintf("##### Building go_bootstrap.\n")
 	for _, dir := range buildlist {
 		installed[dir] = make(chan struct{})
 	}
@@ -1091,20 +1069,97 @@ func cmdbootstrap() {
 
 	gogcflags = os.Getenv("GO_GCFLAGS") // we were using $BOOT_GO_GCFLAGS until now
 	goldflags = os.Getenv("GO_LDFLAGS")
-
-	// Build full toolchain for host and (if different) for target.
-	if goos != oldgoos || goarch != oldgoarch {
-		os.Setenv("CC", defaultcc)
-		buildAll()
-		xprintf("\n")
+	goBootstrap := pathf("%s/go_bootstrap", tooldir)
+	cmdGo := pathf("%s/go", gobin)
+	if debug {
+		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
+		copyfile(pathf("%s/compile1", tooldir), pathf("%s/compile", tooldir), writeExec)
+	}
+
+	// To recap, so far we have built the new toolchain
+	// (cmd/asm, cmd/cgo, cmd/compile, cmd/link)
+	// using Go 1.4's toolchain and go command.
+	// Then we built the new go command (as go_bootstrap)
+	// using the new toolchain and our own build logic (above).
+	//
+	//	toolchain1 = mk(new toolchain, go1.4 toolchain, go1.4 cmd/go)
+	//	go_bootstrap = mk(new cmd/go, toolchain1, cmd/dist)
+	//
+	// The toolchain1 we built earlier is built from the new sources,
+	// but because it was built using cmd/go it has no build IDs.
+	// The eventually installed toolchain needs build IDs, so we need
+	// to do another round:
+	//
+	//	toolchain2 = mk(new toolchain, toolchain1, go_bootstrap)
+	//
+	xprintf("\n##### Building Go toolchain2 using go_bootstrap and Go toolchain1.\n")
+	os.Setenv("CC", defaultcc)
+	if goos == oldgoos && goarch == oldgoarch {
+		// Host and target are same, and we have historically
+		// chosen $CC_FOR_TARGET in this case.
+		os.Setenv("CC", defaultcctarget)
+	}
+	toolchain := []string{"cmd/asm", "cmd/cgo", "cmd/compile", "cmd/link", "cmd/buildid"}
+	goInstall(toolchain...)
+	if debug {
+		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
+		run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/../../darwin_amd64/runtime/internal/sys.a", tooldir))
+		copyfile(pathf("%s/compile2", tooldir), pathf("%s/compile", tooldir), writeExec)
+	}
+
+	// Toolchain2 should be semantically equivalent to toolchain1,
+	// but it was built using the new compilers instead of the Go 1.4 compilers,
+	// so it should at the least run faster. Also, toolchain1 had no build IDs
+	// in the binaries, while toolchain2 does. In non-release builds, the
+	// toolchain's build IDs feed into constructing the build IDs of built targets,
+	// so in non-release builds, everything now looks out-of-date due to
+	// toolchain2 having build IDs - that is, due to the go command seeing
+	// that there are new compilers. In release builds, the toolchain's reported
+	// version is used in place of the build ID, and the go command does not
+	// see that change from toolchain1 to toolchain2, so in release builds,
+	// nothing looks out of date.
+	// To keep the behavior the same in both non-release and release builds,
+	// we force-install everything here.
+	//
+	//	toolchain3 = mk(new toolchain, toolchain2, go_bootstrap)
+	//
+	xprintf("\n##### Building Go toolchain3 using go_bootstrap and Go toolchain2.\n")
+	goInstall(append([]string{"-a"}, toolchain...)...)
+	if debug {
+		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
+		run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/../../darwin_amd64/runtime/internal/sys.a", tooldir))
+		copyfile(pathf("%s/compile3", tooldir), pathf("%s/compile", tooldir), writeExec)
+	}
+	checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...)
+
+	if goos == oldgoos && goarch == oldgoarch {
+		// Common case - not setting up for cross-compilation.
+		xprintf("\n##### Building packages and commands for %s/%s\n", goos, goarch)
+	} else {
+		// GOOS/GOARCH does not match GOHOSTOS/GOHOSTARCH.
+		// Finish GOHOSTOS/GOHOSTARCH installation and then
+		// run GOOS/GOARCH installation.
+		xprintf("\n##### Building packages and commands for host, %s/%s\n", goos, goarch)
+		goInstall("std", "cmd")
+		checkNotStale(goBootstrap, "std", "cmd")
+		checkNotStale(cmdGo, "std", "cmd")
+
+		xprintf("\n##### Building packages and commands for target, %s/%s\n", goos, goarch)
 		goos = oldgoos
 		goarch = oldgoarch
 		os.Setenv("GOOS", goos)
 		os.Setenv("GOARCH", goarch)
+		os.Setenv("CC", defaultcctarget)
+	}
+	goInstall("std", "cmd")
+	checkNotStale(goBootstrap, "std", "cmd")
+	checkNotStale(cmdGo, "std", "cmd")
+	if debug {
+		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
+		run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/../../darwin_amd64/runtime/internal/sys.a", tooldir))
+		checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...)
+		copyfile(pathf("%s/compile4", tooldir), pathf("%s/compile", tooldir), writeExec)
 	}
-
-	os.Setenv("CC", defaultcctarget)
-	buildAll()
 
 	// Check that there are no new files in $GOROOT/bin other than
 	// go and gofmt and $GOOS_$GOARCH (target bin when cross-compiling).
@@ -1129,21 +1184,34 @@ func cmdbootstrap() {
 	}
 }
 
-func buildAll() {
-	desc := ""
-	if oldgoos != goos || oldgoarch != goarch {
-		desc = " host,"
-	}
-	xprintf("##### Building packages and commands for%s %s/%s.\n", desc, goos, goarch)
-	go_bootstrap := pathf("%s/go_bootstrap", tooldir)
-	go_install := []string{go_bootstrap, "install", "-v", "-gcflags=" + gogcflags, "-ldflags=" + goldflags}
+func goInstall(args ...string) {
+	installCmd := []string{pathf("%s/go_bootstrap", tooldir), "install", "-v", "-gcflags=" + gogcflags, "-ldflags=" + goldflags}
 
 	// Force only one process at a time on vx32 emulation.
 	if gohostos == "plan9" && os.Getenv("sysname") == "vx32" {
-		go_install = append(go_install, "-p=1")
+		installCmd = append(installCmd, "-p=1")
 	}
 
-	run(pathf("%s/src", goroot), ShowOutput|CheckExit, append(go_install, "std", "cmd")...)
+	run(goroot, ShowOutput|CheckExit, append(installCmd, args...)...)
+}
+
+func checkNotStale(goBinary string, targets ...string) {
+	out := run(goroot, CheckExit,
+		append([]string{
+			goBinary,
+			"list", "-gcflags=" + gogcflags, "-ldflags=" + goldflags,
+			"-f={{if .Stale}}\t{{.ImportPath}}: {{.StaleReason}}{{end}}",
+		}, targets...)...)
+	if out != "" {
+		os.Setenv("GOCMDDEBUGHASH", "1")
+		for _, target := range []string{"runtime/internal/sys", "cmd/dist", "cmd/link"} {
+			if strings.Contains(out, target) {
+				run(goroot, ShowOutput|CheckExit, goBinary, "list", "-f={{.ImportPath}} {{.Stale}}", target)
+				break
+			}
+		}
+		fatalf("unexpected stale targets reported by %s list -gcflags=\"%s\" -ldflags=\"%s\" for %v:\n%s", goBinary, gogcflags, goldflags, targets, out)
+	}
 }
 
 // Cannot use go/build directly because cmd/dist for a new release

src/cmd/dist/buildtool.go

@@ -106,7 +106,7 @@ func bootstrapBuildTools() {
 	if goroot_bootstrap == "" {
 		goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
 	}
-	xprintf("##### Building Go toolchain using %s.\n", goroot_bootstrap)
+	xprintf("##### Building Go toolchain1 using %s.\n", goroot_bootstrap)
 
 	mkzbootstrap(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 

src/cmd/dist/deps.go

@@ -84,6 +84,14 @@ var builddeps = map[string][]string{
 		"strings",       // cmd/go/internal/bug
 	},
 
+	"cmd/go/internal/cache": {
+		"crypto/sha256", // cmd/go/internal/cache
+		"fmt",           // cmd/go/internal/cache
+		"hash",          // cmd/go/internal/cache
+		"io",            // cmd/go/internal/cache
+		"os",            // cmd/go/internal/cache
+	},
+
 	"cmd/go/internal/cfg": {
 		"cmd/internal/objabi", // cmd/go/internal/cfg
 		"fmt",           // cmd/go/internal/cfg
@@ -224,21 +232,18 @@ var builddeps = map[string][]string{
 		"cmd/go/internal/base", // cmd/go/internal/load
 		"cmd/go/internal/cfg",  // cmd/go/internal/load
 		"cmd/go/internal/str",  // cmd/go/internal/load
-		"cmd/internal/buildid", // cmd/go/internal/load
-		"crypto/sha1",          // cmd/go/internal/load
-		"fmt",                  // cmd/go/internal/load
-		"go/build",             // cmd/go/internal/load
-		"go/token",             // cmd/go/internal/load
-		"io/ioutil",            // cmd/go/internal/load
-		"log",                  // cmd/go/internal/load
-		"os",                   // cmd/go/internal/load
-		"path",                 // cmd/go/internal/load
-		"path/filepath",        // cmd/go/internal/load
-		"regexp",               // cmd/go/internal/load
-		"runtime",              // cmd/go/internal/load
-		"sort",                 // cmd/go/internal/load
-		"strings",              // cmd/go/internal/load
-		"unicode",              // cmd/go/internal/load
+		"fmt",           // cmd/go/internal/load
+		"go/build",      // cmd/go/internal/load
+		"go/token",      // cmd/go/internal/load
+		"io/ioutil",     // cmd/go/internal/load
+		"log",           // cmd/go/internal/load
+		"os",            // cmd/go/internal/load
+		"path",          // cmd/go/internal/load
+		"path/filepath", // cmd/go/internal/load
+		"regexp",        // cmd/go/internal/load
+		"sort",          // cmd/go/internal/load
+		"strings",       // cmd/go/internal/load
+		"unicode",       // cmd/go/internal/load
 	},
 
 	"cmd/go/internal/run": {
@@ -324,35 +329,34 @@ var builddeps = map[string][]string{
 	},
 
 	"cmd/go/internal/work": {
-		"bufio",                // cmd/go/internal/work
-		"bytes",                // cmd/go/internal/work
-		"cmd/go/internal/base", // cmd/go/internal/work
-		"cmd/go/internal/cfg",  // cmd/go/internal/work
-		"cmd/go/internal/load", // cmd/go/internal/work
-		"cmd/go/internal/str",  // cmd/go/internal/work
-		"cmd/internal/buildid", // cmd/go/internal/work
-		"container/heap",       // cmd/go/internal/work
-		"crypto/sha1",          // cmd/go/internal/work
-		"crypto/sha256",        // cmd/go/internal/work
-		"debug/elf",            // cmd/go/internal/work
-		"encoding/json",        // cmd/go/internal/work
-		"errors",               // cmd/go/internal/work
-		"flag",                 // cmd/go/internal/work
-		"fmt",                  // cmd/go/internal/work
-		"go/build",             // cmd/go/internal/work
-		"io",                   // cmd/go/internal/work
-		"io/ioutil",            // cmd/go/internal/work
-		"log",                  // cmd/go/internal/work
-		"os",                   // cmd/go/internal/work
-		"os/exec",              // cmd/go/internal/work
-		"path",                 // cmd/go/internal/work
-		"path/filepath",        // cmd/go/internal/work
-		"regexp",               // cmd/go/internal/work
-		"runtime",              // cmd/go/internal/work
-		"strconv",              // cmd/go/internal/work
-		"strings",              // cmd/go/internal/work
-		"sync",                 // cmd/go/internal/work
-		"time",                 // cmd/go/internal/work
+		"bufio",                 // cmd/go/internal/work
+		"bytes",                 // cmd/go/internal/work
+		"cmd/go/internal/base",  // cmd/go/internal/work
+		"cmd/go/internal/cache", // cmd/go/internal/work
+		"cmd/go/internal/cfg",   // cmd/go/internal/work
+		"cmd/go/internal/load",  // cmd/go/internal/work
+		"cmd/go/internal/str",   // cmd/go/internal/work
+		"cmd/internal/buildid",  // cmd/go/internal/work
+		"container/heap",        // cmd/go/internal/work
+		"crypto/sha1",           // cmd/go/internal/work
+		"debug/elf",             // cmd/go/internal/work
+		"encoding/json",         // cmd/go/internal/work
+		"errors",                // cmd/go/internal/work
+		"flag",                  // cmd/go/internal/work
+		"fmt",                   // cmd/go/internal/work
+		"go/build",              // cmd/go/internal/work
+		"io",                    // cmd/go/internal/work
+		"io/ioutil",             // cmd/go/internal/work
+		"log",                   // cmd/go/internal/work
+		"os",                    // cmd/go/internal/work
+		"os/exec",               // cmd/go/internal/work
+		"path",                  // cmd/go/internal/work
+		"path/filepath",         // cmd/go/internal/work
+		"regexp",                // cmd/go/internal/work
+		"runtime",               // cmd/go/internal/work
+		"strconv",               // cmd/go/internal/work
+		"strings",               // cmd/go/internal/work
+		"sync",                  // cmd/go/internal/work
 	},
 
 	"cmd/internal/buildid": {

src/cmd/dist/test.go

@@ -23,6 +23,8 @@ import (
 )
 
 func cmdtest() {
+	gogcflags = os.Getenv("GO_GCFLAGS")
+
 	var t tester
 	var noRebuild bool
 	flag.BoolVar(&t.listMode, "list", false, "list available tests")
@@ -272,7 +274,7 @@ func (t *tester) registerStdTest(pkg string) {
 				"-short",
 				t.tags(),
 				t.timeout(180),
-				"-gcflags=" + os.Getenv("GO_GCFLAGS"),
+				"-gcflags=" + gogcflags,
 			}
 			if t.race {
 				args = append(args, "-race")
@@ -936,6 +938,7 @@ func (t *tester) cgoTest(dt *distTest) error {
 // running in parallel with earlier tests, or if it has some other reason
 // for needing the earlier tests to be done.
 func (t *tester) runPending(nextTest *distTest) {
+	checkNotStale("go", "std", "cmd")
 	worklist := t.worklist
 	t.worklist = nil
 	for _, w := range worklist {
@@ -985,6 +988,7 @@ func (t *tester) runPending(nextTest *distTest) {
 			log.Printf("Failed: %v", w.err)
 			t.failed = true
 		}
+		checkNotStale("go", "std", "cmd")
 	}
 	if t.failed && !t.keepGoing {
 		log.Fatal("FAILED")

src/cmd/go/go_test.go

@@ -739,6 +739,7 @@ func TestBuildComplex(t *testing.T) {
 	tg.run("build", "-x", "-o", os.DevNull, "complex")
 
 	if _, err := exec.LookPath("gccgo"); err == nil {
+		t.Skip("golang.org/issue/22472")
 		tg.run("build", "-x", "-o", os.DevNull, "-compiler=gccgo", "complex")
 	}
 }
@@ -841,40 +842,36 @@ func TestNewReleaseRebuildsStalePackagesInGOPATH(t *testing.T) {
 		}
 	}
 
-	tg.setenv("TESTGO_IS_GO_RELEASE", "1")
-
 	tg.tempFile("d1/src/p1/p1.go", `package p1`)
 	tg.setenv("GOPATH", tg.path("d1"))
 	tg.run("install", "-a", "p1")
-	tg.wantNotStale("p1", "", "./testgo list claims p1 is stale, incorrectly")
-	tg.sleep()
+	tg.wantNotStale("p1", "", "./testgo list claims p1 is stale, incorrectly, before any changes")
 
-	// Changing mtime and content of runtime/internal/sys/sys.go
-	// should have no effect: we're in a release, which doesn't rebuild
-	// for general mtime or content changes.
+	// Changing mtime of runtime/internal/sys/sys.go
+	// should have no effect: only the content matters.
+	// In fact this should be true even outside a release branch.
 	sys := runtime.GOROOT() + "/src/runtime/internal/sys/sys.go"
+	tg.sleep()
 	restore := addNL(sys)
-	defer restore()
-	tg.wantNotStale("p1", "", "./testgo list claims p1 is stale, incorrectly, after updating runtime/internal/sys/sys.go")
 	restore()
-	tg.wantNotStale("p1", "", "./testgo list claims p1 is stale, incorrectly, after restoring runtime/internal/sys/sys.go")
+	tg.wantNotStale("p1", "", "./testgo list claims p1 is stale, incorrectly, after updating mtime of runtime/internal/sys/sys.go")
 
-	// But changing runtime/internal/sys/zversion.go should have an effect:
-	// that's how we tell when we flip from one release to another.
-	zversion := runtime.GOROOT() + "/src/runtime/internal/sys/zversion.go"
-	restore = addNL(zversion)
+	// But changing content of any file should have an effect.
+	// Previously zversion.go was the only one that mattered;
+	// now they all matter, so keep using sys.go.
+	restore = addNL(sys)
 	defer restore()
-	tg.wantStale("p1", "build ID mismatch", "./testgo list claims p1 is NOT stale, incorrectly, after changing to new release")
+	tg.wantStale("p1", "stale dependency: runtime/internal/sys", "./testgo list claims p1 is NOT stale, incorrectly, after changing sys.go")
 	restore()
 	tg.wantNotStale("p1", "", "./testgo list claims p1 is stale, incorrectly, after changing back to old release")
-	addNL(zversion)
-	tg.wantStale("p1", "build ID mismatch", "./testgo list claims p1 is NOT stale, incorrectly, after changing again to new release")
+	addNL(sys)
+	tg.wantStale("p1", "stale dependency: runtime/internal/sys", "./testgo list claims p1 is NOT stale, incorrectly, after changing sys.go again")
 	tg.run("install", "p1")
 	tg.wantNotStale("p1", "", "./testgo list claims p1 is stale after building with new release")
 
 	// Restore to "old" release.
 	restore()
-	tg.wantStale("p1", "build ID mismatch", "./testgo list claims p1 is NOT stale, incorrectly, after changing to old release after new build")
+	tg.wantStale("p1", "stale dependency: runtime/internal/sys", "./testgo list claims p1 is NOT stale, incorrectly, after restoring sys.go")
 	tg.run("install", "p1")
 	tg.wantNotStale("p1", "", "./testgo list claims p1 is stale after building with old release")
 
@@ -971,8 +968,8 @@ func TestGoInstallRebuildsStalePackagesInOtherGOPATH(t *testing.T) {
 	} else {
 		tg.must(f.Close())
 	}
-	tg.wantStale("p2", "newer source file", "./testgo list claims p2 is NOT stale, incorrectly")
-	tg.wantStale("p1", "stale dependency", "./testgo list claims p1 is NOT stale, incorrectly")
+	tg.wantStale("p2", "build ID mismatch", "./testgo list claims p2 is NOT stale, incorrectly")
+	tg.wantStale("p1", "stale dependency: p2", "./testgo list claims p1 is NOT stale, incorrectly")
 
 	tg.run("install", "p1")
 	tg.wantNotStale("p2", "", "./testgo list claims p2 is stale after reinstall, incorrectly")
@@ -1578,14 +1575,9 @@ func TestPackageNotStaleWithTrailingSlash(t *testing.T) {
 	goroot := runtime.GOROOT()
 	tg.setenv("GOROOT", goroot+"/")
 
-	want := ""
-	if isGoRelease {
-		want = "standard package in Go release distribution"
-	}
-
-	tg.wantNotStale("runtime", want, "with trailing slash in GOROOT, runtime listed as stale")
-	tg.wantNotStale("os", want, "with trailing slash in GOROOT, os listed as stale")
-	tg.wantNotStale("io", want, "with trailing slash in GOROOT, io listed as stale")
+	tg.wantNotStale("runtime", "", "with trailing slash in GOROOT, runtime listed as stale")
+	tg.wantNotStale("os", "", "with trailing slash in GOROOT, os listed as stale")
+	tg.wantNotStale("io", "", "with trailing slash in GOROOT, io listed as stale")
 }
 
 // With $GOBIN set, binaries get installed to $GOBIN.
@@ -2667,6 +2659,7 @@ func TestIssue7573(t *testing.T) {
 	if _, err := exec.LookPath("gccgo"); err != nil {
 		t.Skip("skipping because no gccgo compiler found")
 	}
+	t.Skip("golang.org/issue/22472")
 
 	tg := testgo(t)
 	defer tg.cleanup()
@@ -3710,9 +3703,9 @@ func TestBinaryOnlyPackages(t *testing.T) {
 
 		package p1
 	`)
-	tg.wantStale("p1", "cannot access install target", "p1 is binary-only but has no binary, should be stale")
+	tg.wantStale("p1", "missing or invalid binary-only package", "p1 is binary-only but has no binary, should be stale")
 	tg.runFail("install", "p1")
-	tg.grepStderr("missing or invalid package binary", "did not report attempt to compile binary-only package")
+	tg.grepStderr("missing or invalid binary-only package", "did not report attempt to compile binary-only package")
 
 	tg.tempFile("src/p1/p1.go", `
 		package p1
@@ -3738,7 +3731,7 @@ func TestBinaryOnlyPackages(t *testing.T) {
 		import _ "fmt"
 		func G()
 	`)
-	tg.wantNotStale("p1", "no source code", "should NOT want to rebuild p1 (first)")
+	tg.wantNotStale("p1", "binary-only package", "should NOT want to rebuild p1 (first)")
 	tg.run("install", "-x", "p1") // no-op, up to date
 	tg.grepBothNot("/compile", "should not have run compiler")
 	tg.run("install", "p2") // does not rebuild p1 (or else p2 will fail)
@@ -3750,7 +3743,7 @@ func TestBinaryOnlyPackages(t *testing.T) {
 		package p1
 		func H()
 	`)
-	tg.wantNotStale("p1", "no source code", "should NOT want to rebuild p1 (second)")
+	tg.wantNotStale("p1", "binary-only package", "should NOT want to rebuild p1 (second)")
 	tg.wantNotStale("p2", "", "should NOT want to rebuild p2")
 
 	tg.tempFile("src/p3/p3.go", `
@@ -4417,7 +4410,7 @@ func main() {}`)
 			before()
 			tg.run("install", "mycmd")
 			after()
-			tg.wantStale("mycmd", "build ID mismatch", "should be stale after environment variable change")
+			tg.wantStale("mycmd", "stale dependency: runtime/internal/sys", "should be stale after environment variable change")
 		}
 	}
 

src/cmd/go/internal/cache/hash.go

@@ -12,7 +12,7 @@ import (
 	"os"
 )
 
-const debugHash = false
+var debugHash = os.Getenv("GOCMDDEBUGHASH") == "1"
 
 // HashSize is the number of bytes in a hash.
 const HashSize = 32

src/cmd/go/internal/list/list.go

@@ -194,12 +194,29 @@ func runList(cmd *base.Command, args []string) {
 		}
 	}
 
-	loadpkgs := load.Packages
+	var pkgs []*load.Package
 	if *listE {
-		loadpkgs = load.PackagesAndErrors
+		pkgs = load.PackagesAndErrors(args)
+	} else {
+		pkgs = load.Packages(args)
+	}
+
+	// Estimate whether staleness information is needed,
+	// since it's a little bit of work to compute.
+	needStale := *listJson || strings.Contains(*listFmt, ".Stale")
+	if needStale {
+		var b work.Builder
+		b.Init()
+		b.ComputeStaleOnly = true
+		a := &work.Action{}
+		// TODO: Use pkgsFilter?
+		for _, p := range pkgs {
+			a.Deps = append(a.Deps, b.AutoAction(work.ModeInstall, work.ModeInstall, p))
+		}
+		b.Do(a)
 	}
 
-	for _, pkg := range loadpkgs(args) {
+	for _, pkg := range pkgs {
 		// Show vendor-expanded paths in listing
 		pkg.TestImports = pkg.Vendored(pkg.TestImports)
 		pkg.XTestImports = pkg.Vendored(pkg.XTestImports)

src/cmd/go/internal/load/testgo.go

-// Copyright 2014 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This file contains extra hooks for testing the go command.
-// It is compiled into the Go binary only when building the
-// test copy; it does not get compiled into the standard go
-// command, so these testing hooks are not present in the
-// go command that everyone uses.
-
-// +build testgo
-
-package load
-
-import "os"
-
-func init() {
-	if v := os.Getenv("TESTGO_IS_GO_RELEASE"); v != "" {
-		isGoRelease = v == "1"
-	}
-}

src/cmd/go/internal/test/test.go

@@ -560,8 +560,6 @@ func runTest(cmd *base.Command, args []string) {
 			if p.ImportPath == "unsafe" {
 				continue
 			}
-			p.Stale = true // rebuild
-			p.StaleReason = "rebuild for coverage"
 			p.Internal.CoverMode = testCoverMode
 			var coverFiles []string
 			coverFiles = append(coverFiles, p.GoFiles...)
@@ -633,13 +631,12 @@ func ensureImport(p *load.Package, pkg string) {
 		}
 	}
 
-	a := load.LoadPackage(pkg, &load.ImportStack{})
-	if a.Error != nil {
-		base.Fatalf("load %s: %v", pkg, a.Error)
+	p1 := load.LoadPackage(pkg, &load.ImportStack{})
+	if p1.Error != nil {
+		base.Fatalf("load %s: %v", pkg, p1.Error)
 	}
-	load.ComputeStale(a)
 
-	p.Internal.Imports = append(p.Internal.Imports, a)
+	p.Internal.Imports = append(p.Internal.Imports, p1)
 }
 
 var windowsBadWords = []string{
@@ -741,8 +738,6 @@ func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, prin
 		ptest.Imports = str.StringList(p.Imports, p.TestImports)
 		ptest.Internal.Imports = append(append([]*load.Package{}, p.Internal.Imports...), imports...)
 		ptest.Internal.ForceLibrary = true
-		ptest.Stale = true
-		ptest.StaleReason = "rebuild for test"
 		ptest.Internal.Build = new(build.Package)
 		*ptest.Internal.Build = *p.Internal.Build
 		m := map[string][]token.Position{}
@@ -775,7 +770,6 @@ func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, prin
 				Dir:        p.Dir,
 				GoFiles:    p.XTestGoFiles,
 				Imports:    p.XTestImports,
-				Stale:      true,
 			},
 			Internal: load.PackageInternal{
 				LocalPrefix: p.Internal.LocalPrefix,
@@ -803,7 +797,6 @@ func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, prin
 			GoFiles:    []string{"_testmain.go"},
 			ImportPath: p.ImportPath + " (testmain)",
 			Root:       p.Root,
-			Stale:      true,
 		},
 		Internal: load.PackageInternal{
 			Build:     &build.Package{Name: "main"},
@@ -894,8 +887,6 @@ func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, prin
 		}
 	}
 
-	load.ComputeStale(pmain)
-
 	a := b.LinkAction(work.ModeBuild, work.ModeBuild, pmain)
 	a.Target = testDir + testBinary + cfg.ExeSuffix
 	if cfg.Goos == "windows" {
@@ -939,6 +930,7 @@ func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, prin
 				target = filepath.Join(base.Cwd, target)
 			}
 		}
+		pmain.Target = target
 		buildAction = &work.Action{
 			Mode:    "test build",
 			Func:    work.BuildInstallFunc,
@@ -1018,8 +1010,6 @@ func recompileForTest(pmain, preal, ptest *load.Package) {
 			copy(p1.Internal.Imports, p.Internal.Imports)
 			p = p1
 			p.Target = ""
-			p.Stale = true
-			p.StaleReason = "depends on package being tested"
 		}
 
 		// Update p.Internal.Imports to use test copies.

src/cmd/go/internal/work/action.go

@@ -18,7 +18,6 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
-	"time"
 
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
@@ -36,6 +35,8 @@ type Builder struct {
 	flagCache   map[[2]string]bool   // a cache of supported compiler flags
 	Print       func(args ...interface{}) (int, error)
 
+	ComputeStaleOnly bool // compute staleness for go list; no actual build
+
 	objdirSeq int // counter for NewObjdir
 	pkgSeq    int
 
@@ -45,6 +46,11 @@ type Builder struct {
 	exec      sync.Mutex
 	readySema chan bool
 	ready     actionQueue
+
+	id            sync.Mutex
+	toolIDCache   map[string]string // tool name -> tool ID
+	buildIDCache  map[string]string // file name -> build ID
+	fileHashCache map[string]string // file name -> content hash
 }
 
 // NOTE: Much of Action would not need to be exported if not for test.
@@ -61,12 +67,12 @@ type Action struct {
 	Args       []string                      // additional args for runProgram
 
 	triggers []*Action // inverse of deps
-	buildID  string
 
 	// Generated files, directories.
-	Objdir string // directory for intermediate objects
-	Target string // goal of the action: the created package or executable
-	built  string // the actual created package or executable
+	Objdir  string // directory for intermediate objects
+	Target  string // goal of the action: the created package or executable
+	built   string // the actual created package or executable
+	buildID string // build ID of action output
 
 	// Execution state.
 	pending  int  // number of deps yet to complete
@@ -184,6 +190,9 @@ func (b *Builder) Init() {
 	}
 	b.actionCache = make(map[cacheKey]*Action)
 	b.mkdirCache = make(map[string]bool)
+	b.toolIDCache = make(map[string]string)
+	b.buildIDCache = make(map[string]string)
+	b.fileHashCache = make(map[string]string)
 
 	if cfg.BuildN {
 		b.WorkDir = "$WORK"
@@ -261,6 +270,8 @@ func readpkglist(shlibpath string) (pkgs []*load.Package) {
 
 // cacheAction looks up {mode, p} in the cache and returns the resulting action.
 // If the cache has no such action, f() is recorded and returned.
+// TODO(rsc): Change the second key from *load.Package to interface{},
+// to make the caching in linkShared less awkward?
 func (b *Builder) cacheAction(mode string, p *load.Package, f func() *Action) *Action {
 	a := b.actionCache[cacheKey{mode, p}]
 	if a == nil {
@@ -327,15 +338,6 @@ func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Actio
 			}
 		}
 
-		if !p.Stale && p.Target != "" && p.Name != "main" {
-			// p.Stale==false implies that p.Target is up-to-date.
-			// Record target name for use by actions depending on this one.
-			a.Mode = "use installed"
-			a.Target = p.Target
-			a.Func = nil
-			a.built = a.Target
-			return a
-		}
 		return a
 	})
 
@@ -358,16 +360,6 @@ func (b *Builder) LinkAction(mode, depMode BuildMode, p *load.Package) *Action {
 			Package: p,
 		}
 
-		if !p.Stale && p.Target != "" {
-			// p.Stale==false implies that p.Target is up-to-date.
-			// Record target name for use by actions depending on this one.
-			a.Mode = "use installed"
-			a.Func = nil
-			a.Target = p.Target
-			a.built = a.Target
-			return a
-		}
-
 		a1 := b.CompileAction(ModeBuild, depMode, p)
 		a.Func = (*Builder).link
 		a.Deps = []*Action{a1}
@@ -396,6 +388,15 @@ func (b *Builder) LinkAction(mode, depMode BuildMode, p *load.Package) *Action {
 		a.Target = a.Objdir + filepath.Join("exe", name) + cfg.ExeSuffix
 		a.built = a.Target
 		b.addTransitiveLinkDeps(a, a1, "")
+
+		// Sequence the build of the main package (a1) strictly after the build
+		// of all other dependencies that go into the link. It is likely to be after
+		// them anyway, but just make sure. This is required by the build ID-based
+		// shortcut in (*Builder).useCache(a1), which will call b.linkActionID(a).
+		// In order for that linkActionID call to compute the right action ID, all the
+		// dependencies of a (except a1) must have completed building and have
+		// recorded their build IDs.
+		a1.Deps = append(a1.Deps, &Action{Mode: "nop", Deps: a.Deps[1:]})
 		return a
 	})
 
@@ -426,6 +427,7 @@ func (b *Builder) installAction(a1 *Action) *Action {
 			Target:  p.Target,
 			built:   p.Target,
 		}
+
 		b.addInstallHeaderAction(a)
 		return a
 	})
@@ -566,7 +568,6 @@ func (b *Builder) linkSharedAction(mode, depMode BuildMode, shlib string, a1 *Ac
 				if p.Error != nil {
 					base.Fatalf("load %s: %v", pkg, p.Error)
 				}
-				load.ComputeStale(p)
 				// Assume that if pkg (runtime/cgo or math)
 				// is already accounted for in a different shared library,
 				// then that shared library also contains runtime,
@@ -581,78 +582,12 @@ func (b *Builder) linkSharedAction(mode, depMode BuildMode, shlib string, a1 *Ac
 				add("math")
 			}
 		}
-
-		// Determine the eventual install target and compute staleness.
-		// TODO(rsc): This doesn't belong here and should be with the
-		// other staleness code. When we move to content-based staleness
-		// determination, that will happen for us.
-
-		// The install target is root/pkg/shlib, where root is the source root
-		// in which all the packages lie.
-		// TODO(rsc): Perhaps this cross-root check should apply to the full
-		// transitive package dependency list, not just the ones named
-		// on the command line?
-		pkgDir := a1.Deps[0].Package.Internal.Build.PkgTargetRoot
-		for _, a2 := range a1.Deps {
-			if dir := a2.Package.Internal.Build.PkgTargetRoot; dir != pkgDir {
-				// TODO(rsc): Misuse of base.Fatalf?
-				base.Fatalf("installing shared library: cannot use packages %s and %s from different roots %s and %s",
-					a1.Deps[0].Package.ImportPath,
-					a2.Package.ImportPath,
-					pkgDir,
-					dir)
-			}
-		}
-		// TODO(rsc): Find out and explain here why gccgo is different.
-		if cfg.BuildToolchainName == "gccgo" {
-			pkgDir = filepath.Join(pkgDir, "shlibs")
-		}
-		target := filepath.Join(pkgDir, shlib)
-
-		// The install target is stale if it doesn't exist or if it is older than
-		// any of the .a files that are written into it.
-		// TODO(rsc): This computation does not detect packages that
-		// have been removed from a wildcard used to construct the package list
-		// but are still present in the installed list.
-		// It would be possible to detect this by reading the pkg list
-		// out of any installed target, but content-based staleness
-		// determination should discover that too.
-		var built time.Time
-		if fi, err := os.Stat(target); err == nil {
-			built = fi.ModTime()
-		}
-		stale := cfg.BuildA
-		if !stale {
-			for _, a2 := range a1.Deps {
-				if a2.Target == "" {
-					continue
-				}
-				if a2.Func != nil {
-					// a2 is going to be rebuilt (reuse of existing target would have Func==nil).
-					stale = true
-					break
-				}
-				info, err := os.Stat(a2.Target)
-				if err != nil || info.ModTime().After(built) {
-					stale = true
-					break
-				}
-			}
-		}
-		if !stale {
-			return &Action{
-				Mode:   "use installed buildmode=shared",
-				Target: target,
-				Deps:   []*Action{a1},
-			}
-		}
 		// Link packages into a shared library.
 		a := &Action{
 			Mode:   "go build -buildmode=shared",
 			Objdir: b.NewObjdir(),
 			Func:   (*Builder).linkShared,
 			Deps:   []*Action{a1},
-			Args:   []string{target}, // awful side-channel for install action
 		}
 		a.Target = filepath.Join(a.Objdir, shlib)
 		b.addTransitiveLinkDeps(a, a1, shlib)
@@ -662,13 +597,36 @@ func (b *Builder) linkSharedAction(mode, depMode BuildMode, shlib string, a1 *Ac
 	// Install result.
 	if mode == ModeInstall && a.Func != nil {
 		buildAction := a
+
 		a = b.cacheAction("install-shlib "+shlib, nil, func() *Action {
+			// Determine the eventual install target.
+			// The install target is root/pkg/shlib, where root is the source root
+			// in which all the packages lie.
+			// TODO(rsc): Perhaps this cross-root check should apply to the full
+			// transitive package dependency list, not just the ones named
+			// on the command line?
+			pkgDir := a1.Deps[0].Package.Internal.Build.PkgTargetRoot
+			for _, a2 := range a1.Deps {
+				if dir := a2.Package.Internal.Build.PkgTargetRoot; dir != pkgDir {
+					base.Fatalf("installing shared library: cannot use packages %s and %s from different roots %s and %s",
+						a1.Deps[0].Package.ImportPath,
+						a2.Package.ImportPath,
+						pkgDir,
+						dir)
+				}
+			}
+			// TODO(rsc): Find out and explain here why gccgo is different.
+			if cfg.BuildToolchainName == "gccgo" {
+				pkgDir = filepath.Join(pkgDir, "shlibs")
+			}
+			target := filepath.Join(pkgDir, shlib)
+
 			a := &Action{
 				Mode:   "go install -buildmode=shared",
 				Objdir: buildAction.Objdir,
 				Func:   BuildInstallFunc,
 				Deps:   []*Action{buildAction},
-				Target: buildAction.Args[0],
+				Target: target,
 			}
 			for _, a2 := range buildAction.Deps[0].Deps {
 				p := a2.Package

src/cmd/go/internal/work/build.go

@@ -394,9 +394,6 @@ func BuildModeInit() {
 			cfg.BuildContext.InstallSuffix += codegenArg[1:]
 		}
 	}
-	if strings.HasPrefix(runtimeVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") {
-		buildGcflags = append(buildGcflags, "-goversion", runtimeVersion)
-	}
 }
 
 var runtimeVersion = runtime.Version()

src/cmd/go/internal/work/gc.go

@@ -47,7 +47,7 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, a
 
 	pkgpath := p.ImportPath
 	if cfg.BuildBuildmode == "plugin" {
-		pkgpath = pluginPath(p)
+		pkgpath = pluginPath(a)
 	} else if p.Name == "main" {
 		pkgpath = "main"
 	}
@@ -86,6 +86,9 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, a
 	if p.Internal.OmitDebug || platform == "nacl/amd64p32" || platform == "darwin/arm" || platform == "darwin/arm64" || cfg.Goos == "plan9" {
 		gcargs = append(gcargs, "-dwarf=false")
 	}
+	if strings.HasPrefix(runtimeVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") {
+		gcargs = append(gcargs, "-goversion", runtimeVersion)
+	}
 
 	gcflags := buildGcflags
 	if compilingRuntime {
@@ -102,6 +105,7 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, a
 			}
 		}
 	}
+
 	args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", b.WorkDir, gcflags, gcargs, "-D", p.Internal.LocalPrefix}
 	if importcfg != nil {
 		if err := b.writeFile(objdir+"importcfg", importcfg); err != nil {
@@ -367,12 +371,13 @@ func setextld(ldflags []string, compiler []string) []string {
 // combine the package build ID with the contents of the main package
 // source files. This allows us to identify two different plugins
 // built from two source files with the same name.
-func pluginPath(p *load.Package) string {
+func pluginPath(a *Action) string {
+	p := a.Package
 	if p.ImportPath != "command-line-arguments" {
 		return p.ImportPath
 	}
 	h := sha1.New()
-	fmt.Fprintf(h, "build ID: %s\n", p.Internal.BuildID)
+	fmt.Fprintf(h, "build ID: %s\n", a.buildID)
 	for _, file := range str.StringList(p.GoFiles, p.CgoFiles, p.SFiles) {
 		data, err := ioutil.ReadFile(filepath.Join(p.Dir, file))
 		if err != nil {
@@ -398,7 +403,7 @@ func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string)
 		ldflags = append(ldflags, "-s", "-w")
 	}
 	if cfg.BuildBuildmode == "plugin" {
-		ldflags = append(ldflags, "-pluginpath", pluginPath(root.Package))
+		ldflags = append(ldflags, "-pluginpath", pluginPath(root))
 	}
 
 	// TODO(rsc): This is probably wrong - see golang.org/issue/22155.

src/cmd/internal/objabi/flag.go

@@ -47,6 +47,7 @@ func (versionFlag) Set(s string) error {
 	name := os.Args[0]
 	name = name[strings.LastIndex(name, `/`)+1:]
 	name = name[strings.LastIndex(name, `\`)+1:]
+	name = strings.TrimSuffix(name, ".exe")
 	p := Expstring()
 	if p == DefaultExpstring() {
 		p = ""

test/inline_callers.go

@@ -7,7 +7,7 @@
 package main
 
 import (
-	"log"
+	"fmt"
 	"runtime"
 )
 
@@ -56,8 +56,8 @@ func testCallersFrames(skp int) (frames []string) {
 }
 
 var expectedFrames [][]string = [][]string{
-	0: {"runtime.Callers", "main.testCallers", "main.main"},
-	1: {"main.testCallers", "main.main"},
+	0: {"main.testCallers", "main.main"},
+	1: {"main.testCallers", "runtime.skipPleaseUseCallersFrames", "main.main"},
 	2: {"main.testCallers", "runtime.skipPleaseUseCallersFrames", "main.main"},
 	3: {"main.testCallers", "runtime.skipPleaseUseCallersFrames", "main.main"},
 	4: {"main.testCallers", "runtime.skipPleaseUseCallersFrames", "main.main"},
@@ -83,13 +83,13 @@ func main() {
 		frames := testCallers(i)
 		expected := expectedFrames[i]
 		if !same(frames, expected) {
-			log.Fatalf("testCallers(%d):\n got %v\n want %v", i, frames, expected)
+			fmt.Printf("testCallers(%d):\n got %v\n want %v", i, frames, expected)
 		}
 
 		frames = testCallersFrames(i)
 		expected = allFrames[i:]
 		if !same(frames, expected) {
-			log.Fatalf("testCallersFrames(%d):\n got %v\n want %v", i, frames, expected)
+			fmt.Printf("testCallersFrames(%d):\n got %v\n want %v", i, frames, expected)
 		}
 	}
 }