cmd/go: implement Go checksum database support

This CL adds support for consulting the Go checksum database
when downloading a module that is not already listed in go.sum.
The overall system is described at golang.org/design/25530-sumdb,
and this CL implements the functionality described specifically in
golang.org/design/25530-sumdb#command-client.

Although the eventual plan is to set GOPROXY and GOSUMDB to
default to a Google-run proxy serving the public Go ecosystem,
this CL leaves them off by default.

Fixes #30601.

Change-Id: Ie46140f93c6cc2d85573fbce0878a258819ff44d
Reviewed-on: https://go-review.googlesource.com/c/go/+/173951
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Emmanuel Odeke <emm.odeke@gmail.com>
Reviewed-by: Jay Conrod <jayconrod@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/cmd/go/alldocs.go

@@ -2742,33 +2742,46 @@
 // you want to use the same code you used yesterday.
 //
 // If a downloaded module is not yet included in go.sum and it is a publicly
-// available module, the go command consults the Go notary server to fetch
+// available module, the go command consults the Go checksum database to fetch
 // the expected go.sum lines. If the downloaded code does not match those
 // lines, the go command reports the mismatch and exits. Note that the
-// notary is not consulted for module versions already listed in go.sum.
+// database is not consulted for module versions already listed in go.sum.
 //
-// The GONOVERIFY environment variable is a comma-separated list of
+// If a go.sum mismatch is reported, it is always worth investigating why
+// the code downloaded today differs from what was downloaded yesterday.
+//
+// The GOSUMDB environment variable identifies the name of checksum database
+// to use and optionally its public key and URL, as in:
+//
+// 	GOSUMDB="sum.golang.org"
+// 	GOSUMDB="sum.golang.org+<publickey>"
+// 	GOSUMDB="sum.golang.org+<publickey> https://sum.golang.org"
+//
+// The go command knows the public key of sum.golang.org; use of any other
+// database requires giving the public key explicitly. The URL defaults to
+// "https://" followed by the database name.
+//
+// GOSUMDB defaults to "sum.golang.org" when GOPROXY="https://proxy.golang.org"
+// and otherwise defaults to "off". NOTE: The GOSUMDB will later default to
+// "sum.golang.org" unconditionally.
+//
+// If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag,
+// the checksum database is never consulted, but at the cost of giving up the
+// security guarantee of verified repeatable downloads for all modules.
+// A better way to bypass the checksum database for specific modules is
+// to use the GONOSUMDB environment variable.
+//
+// The GONOSUMDB environment variable is a comma-separated list of
 // patterns (in the syntax of Go's path.Match) of module path prefixes
-// that should not be verified using the notary. For example,
+// that should not be compared against the checksum database.
+// For example,
 //
-// 	GONOVERIFY=*.corp.example.com,rsc.io/private
+// 	GONOSUMDB=*.corp.example.com,rsc.io/private
 //
-// disables notary verification for modules with path prefixes matching
+// disables checksum database lookups for modules with path prefixes matching
 // either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private",
 // and "rsc.io/private/quux".
 //
-// As a special case, if GONOVERIFY is set to "off", or if "go get" was invoked
-// with the -insecure flag, the notary is never consulted, but note that this
-// defeats the security provided by the notary. A better course of action is
-// to set a narrower GONOVERIFY and, in the case of go.sum mismatches,
-// investigate why the code downloaded code differs from what was
-// downloaded yesterday.
-//
-// NOTE: Early in the Go 1.13 dev cycle, the notary is being simulated by
-// a whitelist of known hashes for popular Go modules, to expose any
-// problems arising from knowing the expected hashes.
-// TODO(rsc): This note should be removed once the real notary is used instead. See #30601.
-//
 //
 // Testing flags
 //

src/cmd/go/internal/cfg/cfg.go

@@ -265,12 +265,14 @@ var knownEnv = `
 	GOHOSTOS
 	GOMIPS
 	GOMIPS64
-	GONOVERIFY
+	GONOPROXY
+	GONOSUMDB
 	GOOS
 	GOPATH
 	GOPPC64
 	GOPROXY
 	GOROOT
+	GOSUMDB
 	GOTMPDIR
 	GOTOOLDIR
 	GOWASM
@@ -293,8 +295,45 @@ var (
 	GOMIPS64 = envOr("GOMIPS64", objabi.GOMIPS64)
 	GOPPC64  = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", objabi.GOPPC64))
 	GOWASM   = envOr("GOWASM", fmt.Sprint(objabi.GOWASM))
+
+	GOPROXY   = goproxy()
+	GOSUMDB   = gosumdb()
+	GONOPROXY = Getenv("GONOPROXY")
+	GONOSUMDB = Getenv("GONOSUMDB")
 )
 
+func goproxy() string {
+	v := Getenv("GOPROXY")
+	if v != "" {
+		return v
+	}
+
+	// Proxy is off by default for now.
+	// TODO(rsc): Remove this condition, turning it on always.
+	// (But do NOT do this without approval from rsc.)
+	if true {
+		return "direct"
+	}
+
+	return "https://proxy.golang.org"
+}
+
+func gosumdb() string {
+	v := Getenv("GOSUMDB")
+	if v != "" {
+		return v
+	}
+
+	// Checksum database is off by default except when GOPROXY is proxy.golang.org.
+	// TODO(rsc): Remove this condition, turning it on always.
+	// (But do NOT do this without approval from rsc.)
+	if !strings.HasPrefix(GOPROXY, "https://proxy.golang.org") {
+		return "off"
+	}
+
+	return "sum.golang.org"
+}
+
 // GetArchEnv returns the name and setting of the
 // GOARCH-specific architecture environment variable.
 // If the current architecture has no GOARCH-specific variable,

src/cmd/go/internal/envcmd/env.go

@@ -8,13 +8,13 @@ package envcmd
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
-	"unicode/utf8"
-	"io/ioutil"
-	"sort"
 	"runtime"
+	"sort"
 	"strings"
+	"unicode/utf8"
 
 	"cmd/go/internal/base"
 	"cmd/go/internal/cache"
@@ -74,10 +74,13 @@ func MkEnv() []cfg.EnvVar {
 		{Name: "GOFLAGS", Value: cfg.Getenv("GOFLAGS")},
 		{Name: "GOHOSTARCH", Value: runtime.GOARCH},
 		{Name: "GOHOSTOS", Value: runtime.GOOS},
+		{Name: "GONOPROXY", Value: cfg.GONOPROXY},
+		{Name: "GONOSUMDB", Value: cfg.GONOSUMDB},
 		{Name: "GOOS", Value: cfg.Goos},
 		{Name: "GOPATH", Value: cfg.BuildContext.GOPATH},
-		{Name: "GOPROXY", Value: cfg.Getenv("GOPROXY")},
+		{Name: "GOPROXY", Value: cfg.GOPROXY},
 		{Name: "GOROOT", Value: cfg.GOROOT},
+		{Name: "GOSUMDB", Value: cfg.GOSUMDB},
 		{Name: "GOTMPDIR", Value: cfg.Getenv("GOTMPDIR")},
 		{Name: "GOTOOLDIR", Value: base.ToolDir},
 	}
@@ -387,7 +390,7 @@ func updateEnvFile(add map[string]string, del map[string]bool) {
 		}
 	}
 	for key, val := range add {
-		lines = append(lines, key + "=" + val + "\n")
+		lines = append(lines, key+"="+val+"\n")
 	}
 
 	// Delete requested variables (go env -u).
@@ -404,7 +407,7 @@ func updateEnvFile(add map[string]string, del map[string]bool) {
 	for i := 0; i <= len(lines); i++ {
 		if i == len(lines) || lineToKey(lines[i]) == "" {
 			sortKeyValues(lines[start:i])
-			start = i+1
+			start = i + 1
 		}
 	}
 

src/cmd/go/internal/modfetch/bootstrap.go

+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build cmd_go_bootstrap
+
+package modfetch
+
+import "cmd/go/internal/module"
+
+func useSumDB(mod module.Version) bool {
+	return false
+}
+
+func lookupSumDB(mod module.Version) (string, []string, error) {
+	panic("bootstrap")
+}

src/cmd/go/internal/modfetch/fetch.go

@@ -48,7 +48,7 @@ func Download(mod module.Version) (dir string, err error) {
 		if err := download(mod, dir); err != nil {
 			return cached{"", err}
 		}
-		checkSum(mod)
+		checkMod(mod)
 		return cached{dir, nil}
 	}).(cached)
 	return c.dir, c.err
@@ -246,7 +246,7 @@ func downloadZip(mod module.Version, zipfile string) (err error) {
 	if err != nil {
 		return err
 	}
-	checkOneSum(mod, hash)
+	checkModSum(mod, hash)
 
 	if err := renameio.WriteFile(zipfile+"hash", []byte(hash)); err != nil {
 		return err
@@ -305,7 +305,7 @@ func initGoSum() bool {
 		readGoSum(migrate, alt, data)
 		for mod, sums := range migrate {
 			for _, sum := range sums {
-				checkOneSumLocked(mod, sum)
+				addModSumLocked(mod, sum)
 			}
 		}
 		goSum.modverify = alt
@@ -348,8 +348,8 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) {
 	}
 }
 
-// checkSum checks the given module's checksum.
-func checkSum(mod module.Version) {
+// checkMod checks the given module's checksum.
+func checkMod(mod module.Version) {
 	if PkgMod == "" {
 		// Do not use current directory.
 		return
@@ -373,7 +373,7 @@ func checkSum(mod module.Version) {
 		base.Fatalf("verifying %s@%s: unexpected ziphash: %q", mod.Path, mod.Version, h)
 	}
 
-	checkOneSum(mod, h)
+	checkModSum(mod, h)
 }
 
 // goModSum returns the checksum for the go.mod contents.
@@ -391,24 +391,46 @@ func checkGoMod(path, version string, data []byte) {
 		base.Fatalf("verifying %s %s go.mod: %v", path, version, err)
 	}
 
-	checkOneSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
+	checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
 }
 
-// checkOneSum checks that the recorded hash for mod is h.
-func checkOneSum(mod module.Version, h string) {
+// checkModSum checks that the recorded checksum for mod is h.
+func checkModSum(mod module.Version, h string) {
+	// We lock goSum when manipulating it,
+	// but we arrange to release the lock when calling checkSumDB,
+	// so that parallel calls to checkModHash can execute parallel calls
+	// to checkSumDB.
+
+	// Check whether mod+h is listed in go.sum already. If so, we're done.
 	goSum.mu.Lock()
-	defer goSum.mu.Unlock()
-	if initGoSum() {
-		checkOneSumLocked(mod, h)
-	} else if useNotary(mod) {
-		checkNotarySum(mod, h)
+	inited := initGoSum()
+	done := inited && haveModSumLocked(mod, h)
+	goSum.mu.Unlock()
+
+	if done {
+		return
+	}
+
+	// Not listed, so we want to add them.
+	// Consult checksum database if appropriate.
+	if useSumDB(mod) {
+		// Calls base.Fatalf if mismatch detected.
+		checkSumDB(mod, h)
+	}
+
+	// Add mod+h to go.sum, if it hasn't appeared already.
+	if inited {
+		goSum.mu.Lock()
+		addModSumLocked(mod, h)
+		goSum.mu.Unlock()
 	}
 }
 
-func checkOneSumLocked(mod module.Version, h string) {
+// haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
+// If it finds a conflicting pair instead, it calls base.Fatalf.
+// goSum.mu must be locked.
+func haveModSumLocked(mod module.Version, h string) bool {
 	goSum.checked[modSum{mod, h}] = true
-
-	checkGoSum := func() bool {
 	for _, vh := range goSum.m[mod] {
 		if h == vh {
 			return true
@@ -418,25 +440,14 @@ func checkOneSumLocked(mod module.Version, h string) {
 		}
 	}
 	return false
-	}
-
-	if checkGoSum() {
-		return
-	}
-
-	if useNotary(mod) {
-		goSum.mu.Unlock()
-		checkNotarySum(mod, h) // dies if h is wrong
-		goSum.mu.Lock()
+}
 
-		// Because we dropped the lock, a racing goroutine
-		// may have already added this entry to go.sum.
-		// Check again.
-		if checkGoSum() {
+// addModSumLocked adds the pair mod,h to go.sum.
+// goSum.mu must be locked.
+func addModSumLocked(mod module.Version, h string) {
+	if haveModSumLocked(mod, h) {
 		return
 	}
-	}
-
 	if len(goSum.m[mod]) > 0 {
 		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
 	}
@@ -444,16 +455,22 @@ func checkOneSumLocked(mod module.Version, h string) {
 	goSum.dirty = true
 }
 
-// checkNotarySum checks the mod, h pair against the Go notary.
+// checkSumDB checks the mod, h pair against the Go checksum database.
 // It calls base.Fatalf if the hash is to be rejected.
-func checkNotarySum(mod module.Version, h string) {
-	hashes := notaryHashes(mod)
-	for _, vh := range hashes {
-		if h == vh {
+func checkSumDB(mod module.Version, h string) {
+	db, lines, err := lookupSumDB(mod)
+	if err != nil {
+		base.Fatalf("verifying %s@%s: %v", mod.Path, mod.Version, err)
+	}
+
+	have := mod.Path + " " + mod.Version + " " + h
+	prefix := mod.Path + " " + mod.Version + " h1:"
+	for _, line := range lines {
+		if line == have {
 			return
 		}
-		if strings.HasPrefix(vh, "h1:") {
-			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tnotary:     %v"+notarySumMismatch, mod.Path, mod.Version, h, vh)
+		if strings.HasPrefix(line, prefix) {
+			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, mod.Path, mod.Version, h, db, line[len(prefix)-len("h1:"):])
 		}
 	}
 }
@@ -526,7 +543,8 @@ func WriteGoSum() {
 		goSum.m = make(map[module.Version][]string, len(goSum.m))
 		readGoSum(goSum.m, GoSumFile, data)
 		for ms := range goSum.checked {
-			checkOneSumLocked(ms.mod, ms.sum)
+			addModSumLocked(ms.mod, ms.sum)
+			goSum.dirty = true
 		}
 	}
 
@@ -587,10 +605,10 @@ have intercepted the download attempt.
 For more information, see 'go help module-auth'.
 `
 
-const notarySumMismatch = `
+const sumdbMismatch = `
 
 SECURITY ERROR
-This download does NOT match the expected download known to the notary.
+This download does NOT match the one reported by the checksum server.
 The bits may have been replaced on the origin server, or an attacker may
 have intercepted the download attempt.
 
@@ -662,31 +680,44 @@ go.sum is wrong or the downloaded code is wrong. Usually go.sum is right:
 you want to use the same code you used yesterday.
 
 If a downloaded module is not yet included in go.sum and it is a publicly
-available module, the go command consults the Go notary server to fetch
+available module, the go command consults the Go checksum database to fetch
 the expected go.sum lines. If the downloaded code does not match those
 lines, the go command reports the mismatch and exits. Note that the
-notary is not consulted for module versions already listed in go.sum.
+database is not consulted for module versions already listed in go.sum.
+
+If a go.sum mismatch is reported, it is always worth investigating why
+the code downloaded today differs from what was downloaded yesterday.
+
+The GOSUMDB environment variable identifies the name of checksum database
+to use and optionally its public key and URL, as in:
+
+	GOSUMDB="sum.golang.org"
+	GOSUMDB="sum.golang.org+<publickey>"
+	GOSUMDB="sum.golang.org+<publickey> https://sum.golang.org"
 
-The GONOVERIFY environment variable is a comma-separated list of
+The go command knows the public key of sum.golang.org; use of any other
+database requires giving the public key explicitly. The URL defaults to
+"https://" followed by the database name.
+
+GOSUMDB defaults to "sum.golang.org" when GOPROXY="https://proxy.golang.org"
+and otherwise defaults to "off". NOTE: The GOSUMDB will later default to
+"sum.golang.org" unconditionally.
+
+If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag,
+the checksum database is never consulted, but at the cost of giving up the
+security guarantee of verified repeatable downloads for all modules.
+A better way to bypass the checksum database for specific modules is
+to use the GONOSUMDB environment variable.
+
+The GONOSUMDB environment variable is a comma-separated list of
 patterns (in the syntax of Go's path.Match) of module path prefixes
-that should not be verified using the notary. For example,
+that should not be compared against the checksum database.
+For example,
 
-	GONOVERIFY=*.corp.example.com,rsc.io/private
+	GONOSUMDB=*.corp.example.com,rsc.io/private
 
-disables notary verification for modules with path prefixes matching
+disables checksum database lookups for modules with path prefixes matching
 either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private",
 and "rsc.io/private/quux".
-
-As a special case, if GONOVERIFY is set to "off", or if "go get" was invoked
-with the -insecure flag, the notary is never consulted, but note that this
-defeats the security provided by the notary. A better course of action is
-to set a narrower GONOVERIFY and, in the case of go.sum mismatches,
-investigate why the code downloaded code differs from what was
-downloaded yesterday.
-
-NOTE: Early in the Go 1.13 dev cycle, the notary is being simulated by
-a whitelist of known hashes for popular Go modules, to expose any
-problems arising from knowing the expected hashes.
-TODO(rsc): This note should be removed once the real notary is used instead. See #30601.
 `,
 }

src/cmd/go/internal/modfetch/key.go

+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package modfetch
+
+var knownGOSUMDB = map[string]string{
+	"sum.golang.org": "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8",
+}

src/cmd/go/internal/modfetch/notary.go

-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package modfetch
-
-import (
-	"fmt"
-	pathpkg "path"
-	"strings"
-
-	"cmd/go/internal/base"
-	"cmd/go/internal/cfg"
-	"cmd/go/internal/get"
-	"cmd/go/internal/module"
-)
-
-// notaryShouldVerify reports whether the notary should be used for path,
-// given the GONOVERIFY setting.
-func notaryShouldVerify(path, GONOVERIFY string) (bool, error) {
-	if GONOVERIFY == "off" {
-		return false, nil
-	}
-	for GONOVERIFY != "" {
-		var pattern string
-		i := strings.Index(GONOVERIFY, ",")
-		if i < 0 {
-			pattern, GONOVERIFY = GONOVERIFY, ""
-		} else {
-			pattern, GONOVERIFY = GONOVERIFY[:i], GONOVERIFY[i+1:]
-		}
-		if pattern == "" {
-			continue
-		}
-		n := strings.Count(pattern, "/") + 1
-		prefix := path
-		for i := 0; i < len(prefix); i++ {
-			if prefix[i] == '/' {
-				n--
-				if n == 0 {
-					prefix = prefix[:i]
-					break
-				}
-			}
-		}
-		if n > 1 {
-			continue
-		}
-		matched, err := pathpkg.Match(pattern, prefix)
-		if err != nil {
-			// Note that path.Match does not guarantee to detect
-			// pattern errors. It usually depends on whether the
-			// given text (prefix in this case) matches enough of
-			// the pattern to reach the error. So this will only
-			// trigger on malformed patterns that are "close enough" to prefix.
-			return false, fmt.Errorf("malformed GONOVERIFY pattern: %s", pattern)
-		}
-		if matched {
-			return false, nil
-		}
-	}
-	return true, nil
-}
-
-// useNotary reports whether to use the notary for the given module.
-func useNotary(mod module.Version) bool {
-	if get.Insecure {
-		return false
-	}
-	wantNotary, err := notaryShouldVerify(mod.Path, cfg.Getenv("GONOVERIFY"))
-	if err != nil {
-		base.Fatalf("%v", err)
-	}
-
-	// TODO(rsc): return wantNotary. See #30601.
-	//
-	// This code must be deleted when goSumPin is deleted.
-	// goSumPin is only a partial notary simulation, so we don't return true from
-	// useNotary when we don't have an entry for that module.
-	// This differs from the real notary, which will be authoritative
-	// for everything it is asked for. When goSumPin is removed,
-	// this function body should end here with "return wantNotary".
-
-	_ = goSumPin // read TODO above if goSumPin is gone
-	return wantNotary && notaryHashes(mod) != nil
-}
-
-// notaryHashes fetches hashes for mod from the notary.
-// The caller must have checked that useNotary(mod) is true.
-func notaryHashes(mod module.Version) []string {
-	// For testing, hard-code this result.
-	if mod.Path == "rsc.io/badsum" {
-		switch mod.Version {
-		case "v1.0.0":
-			return []string{"h1:6/o+QJfe6mFSNuegDihphabcvR94anXQk/qq7Enr19U="}
-		case "v1.0.0/go.mod":
-			return []string{"h1:avOsLUJaHavllihBU9qCTW37z64ypkZjqZg8O16JLVY="}
-		case "v1.0.1":
-			return []string{"h1:S7G9Ikksx7htnFivDrUOv8xI0kIdAf15gLt97Gy//Zk="}
-		case "v1.0.1/go.mod":
-			return []string{"h1:avOsLUJaHavllihBU9qCTW37z64ypkZjqZg8O16JLVY="}
-		}
-	}
-
-	// Until the notary is ready, simulate contacting the notary by
-	// looking in the known hash list goSumPin in pin.go.
-	// Entries not listed in goSumPin are treated as "not for the notary",
-	// but once the real notary is added, they should be treated as
-	// "failed to verify".
-	//
-	// TODO(rsc): Once the notary is ready, this function should be
-	// rewritten to use it. See #30601.
-	i := strings.Index(goSumPin, "\n"+mod.Path+"\n")
-	if i < 0 {
-		return nil
-	}
-	wantGoSum := false
-	if strings.HasSuffix(mod.Version, "/go.mod") {
-		wantGoSum = true
-		mod.Version = strings.TrimSuffix(mod.Version, "/go.mod")
-	}
-	versions := goSumPin[i+1+len(mod.Path)+1:]
-	var lastSum, lastGoSum string
-	for {
-		i := strings.Index(versions, "\n")
-		if i < 0 {
-			break
-		}
-		line := versions[:i]
-		versions = versions[i+1:]
-		if !strings.HasPrefix(line, " ") {
-			break
-		}
-		f := strings.Fields(line)
-		if len(f) < 3 {
-			break
-		}
-		if f[1] == "-" {
-			f[1] = lastSum
-		} else {
-			lastSum = f[1]
-		}
-		if f[2] == "-" {
-			f[2] = lastGoSum
-		} else {
-			lastGoSum = f[2]
-		}
-		if f[0] == mod.Version {
-			if wantGoSum {
-				return []string{f[2]}
-			}
-			return []string{f[1]}
-		}
-	}
-	return nil
-}

src/cmd/go/internal/modfetch/notary_test.go

-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package modfetch
-
-import (
-	"testing"
-)
-
-var notaryShouldVerifyTests = []struct {
-	modPath    string
-	GONOVERIFY string
-	result     int // -1 = bad GONOVERIFY, 0 = wantNotary=false, 1 = wantNotary=true
-}{
-	{"anything", "off", 0},
-	{"anything", "", 1},
-	{"anything", ",", 1},
-	{"anything", ",foo,", 1},
-	{"anything", "[malformed", -1},
-	{"anything", "malformed[", 1},
-	{"my.corp.example.com", "*.[c]orp.*", 0},
-	{"my.corp.example.com/foo", "*.c[^a]rp.*", 0},
-	{"my.corp.example.com", "*.corp.*,bar.com", 0},
-	{"my.corp.example.com/foo", "*.corp.*,bar.com", 0},
-	{"my.corp.example.com", "bar.com,*.corp.*", 0},
-	{"my.corp.example.com/foo", "bar.com,*.corp.*", 0},
-	{"bar.com", "*.corp.*", 1},
-	{"bar.com/foo", "*.corp.*", 1},
-	{"bar.com", "*.corp.*,bar.com", 0},
-	{"bar.com/foo", "*.corp.*,bar.com", 0},
-	{"bar.com", "bar.com,*.corp.*", 0},
-	{"bar.com/foo", "bar.com,*.corp.*", 0},
-}
-
-func TestNotaryShouldVerify(t *testing.T) {
-	for _, tt := range notaryShouldVerifyTests {
-		wantNotary, err := notaryShouldVerify(tt.modPath, tt.GONOVERIFY)
-		if wantNotary != (tt.result > 0) || (err != nil) != (tt.result < 0) {
-			wantErr := "nil"
-			if tt.result < 0 {
-				wantErr = "non-nil error"
-			}
-			t.Errorf("notaryShouldVerify(%q, %q) = %v, %v, want %v, %s", tt.modPath, tt.GONOVERIFY, wantNotary, err, tt.result > 0, wantErr)
-		}
-	}
-}

src/cmd/go/internal/modfetch/proxy.go

@@ -10,7 +10,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	url "net/url"
+	"net/url"
 	"os"
 	pathpkg "path"
 	"path/filepath"

src/cmd/go/internal/modfetch/repo.go

@@ -17,6 +17,7 @@ import (
 	"cmd/go/internal/modfetch/codehost"
 	"cmd/go/internal/par"
 	"cmd/go/internal/semver"
+	"cmd/go/internal/str"
 	web "cmd/go/internal/web"
 )
 
@@ -205,10 +206,9 @@ func lookup(path string) (r Repo, err error) {
 	if proxyURL == "off" {
 		return nil, fmt.Errorf("module lookup disabled by GOPROXY=%s", proxyURL)
 	}
-	if proxyURL != "" && proxyURL != "direct" {
+	if proxyURL != "" && proxyURL != "direct" && !str.GlobsMatchPath(cfg.GONOPROXY, path) {
 		return lookupProxy(path)
 	}
-
 	return lookupDirect(path)
 }
 

src/cmd/go/internal/modfetch/sumdb.go

+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Go checksum database lookup
+
+// +build !cmd_go_bootstrap
+
+package modfetch
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"cmd/go/internal/get"
+	"cmd/go/internal/lockedfile"
+	"cmd/go/internal/module"
+	"cmd/go/internal/note"
+	"cmd/go/internal/str"
+	"cmd/go/internal/sumweb"
+	"cmd/go/internal/web"
+)
+
+// useSumDB reports whether to use the Go checksum database for the given module.
+func useSumDB(mod module.Version) bool {
+	return cfg.GOSUMDB != "off" && !get.Insecure && !str.GlobsMatchPath(cfg.GONOSUMDB, mod.Path)
+}
+
+// lookupSumDB returns the Go checksum database's go.sum lines for the given module,
+// along with the name of the database.
+func lookupSumDB(mod module.Version) (dbname string, lines []string, err error) {
+	dbOnce.Do(func() {
+		dbName, db, dbErr = dbDial()
+	})
+	if dbErr != nil {
+		return "", nil, dbErr
+	}
+	lines, err = db.Lookup(mod.Path, mod.Version)
+	return dbName, lines, err
+}
+
+var (
+	dbOnce sync.Once
+	dbName string
+	db     *sumweb.Conn
+	dbErr  error
+)
+
+func dbDial() (dbName string, db *sumweb.Conn, err error) {
+	// $GOSUMDB can be "key" or "key url",
+	// and the key can be a full verifier key
+	// or a host on our list of known keys.
+	key := strings.Fields(cfg.Getenv("GOSUMDB"))
+	if len(key) >= 1 {
+		if k := knownGOSUMDB[key[0]]; k != "" {
+			key[0] = k
+		}
+	}
+	if len(key) == 0 {
+		return "", nil, fmt.Errorf("missing GOSUMDB")
+	}
+	if len(key) > 2 {
+		return "", nil, fmt.Errorf("invalid GOSUMDB: too many fields")
+	}
+	vkey, err := note.NewVerifier(key[0])
+	if err != nil {
+		return "", nil, fmt.Errorf("invalid GOSUMDB: %v", err)
+	}
+	name := vkey.Name()
+
+	// No funny business in the database name.
+	direct, err := url.Parse("https://" + name)
+	if err != nil || strings.HasSuffix(name, "/") || *direct != (url.URL{Scheme: "https", Host: direct.Host, Path: direct.Path, RawPath: direct.RawPath}) || direct.RawPath != "" || direct.Host == "" {
+		return "", nil, fmt.Errorf("invalid sumdb name (must be host[/path]): %s %+v", name, *direct)
+	}
+
+	// Determine how to get to database.
+	var base *url.URL
+	if len(key) >= 2 {
+		// Use explicit alternate URL listed in $GOSUMDB,
+		// bypassing both the default URL derivation and any proxies.
+		u, err := url.Parse(key[1])
+		if err != nil {
+			return "", nil, fmt.Errorf("invalid GOSUMDB URL: %v", err)
+		}
+		base = u
+	}
+
+	return name, sumweb.NewConn(&dbClient{key: key[0], name: name, direct: direct, base: base}), nil
+}
+
+type dbClient struct {
+	key    string
+	name   string
+	direct *url.URL
+
+	once    sync.Once
+	base    *url.URL
+	baseErr error
+}
+
+func (c *dbClient) ReadRemote(path string) ([]byte, error) {
+	c.once.Do(c.initBase)
+	if c.baseErr != nil {
+		return nil, c.baseErr
+	}
+
+	var data []byte
+	start := time.Now()
+	targ := web.Join(c.base, path)
+	data, err := web.GetBytes(targ)
+	if false {
+		fmt.Fprintf(os.Stderr, "%.3fs %s\n", time.Since(start).Seconds(), web.Redacted(targ))
+	}
+	return data, err
+}
+
+// initBase determines the base URL for connecting to the database.
+// Determining the URL requires sending network traffic to proxies,
+// so this work is delayed until we need to download something from
+// the database. If everything we need is in the local cache and
+// c.ReadRemote is never called, we will never do this work.
+func (c *dbClient) initBase() {
+	if c.base != nil {
+		return
+	}
+
+	// Try proxies in turn until we find out how to connect to this database.
+	urls, err := proxyURLs()
+	if err != nil {
+		c.baseErr = err
+		return
+	}
+	for _, proxyURL := range urls {
+		if proxyURL == "direct" {
+			break
+		}
+		proxy, err := url.Parse(proxyURL)
+		if err != nil {
+			c.baseErr = err
+			return
+		}
+		// Quoting https://golang.org/design/25530-sumdb#proxying-a-checksum-database:
+		//
+		// Before accessing any checksum database URL using a proxy,
+		// the proxy client should first fetch <proxyURL>/sumdb/<sumdb-name>/supported.
+		// If that request returns a successful (HTTP 200) response, then the proxy supports
+		// proxying checksum database requests. In that case, the client should use
+		// the proxied access method only, never falling back to a direct connection to the database.
+		// If the /sumdb/<sumdb-name>/supported check fails with a “not found” (HTTP 404)
+		// or “gone” (HTTP 410) response, the proxy is unwilling to proxy the checksum database,
+		// and the client should connect directly to the database.
+		// Any other response is treated as the database being unavailable.
+		_, err = web.GetBytes(web.Join(proxy, "sumdb/"+c.name+"/supported"))
+		if err == nil {
+			// Success! This proxy will help us.
+			c.base = web.Join(proxy, "sumdb/"+c.name)
+			return
+		}
+		// If the proxy serves a non-404/410, give up.
+		if !errors.Is(err, os.ErrNotExist) {
+			c.baseErr = err
+			return
+		}
+	}
+
+	// No proxies, or all proxies said 404, or we reached an explicit "direct".
+	c.base = c.direct
+}
+
+// ReadConfig reads the key from c.key
+// and otherwise reads the config (a latest tree head) from GOPATH/pkg/sumdb/<file>.
+func (c *dbClient) ReadConfig(file string) (data []byte, err error) {
+	if file == "key" {
+		return []byte(c.key), nil
+	}
+
+	// GOPATH/pkg is PkgMod/..
+	targ := filepath.Join(PkgMod, "../sumdb/"+file)
+	data, err = lockedfile.Read(targ)
+	if errors.Is(err, os.ErrNotExist) {
+		// Treat non-existent as empty, to bootstrap the "latest" file
+		// the first time we connect to a given database.
+		return []byte{}, nil
+	}
+	return data, err
+}
+
+// WriteConfig rewrites the latest tree head.
+func (*dbClient) WriteConfig(file string, old, new []byte) error {
+	if file == "key" {
+		// Should not happen.
+		return fmt.Errorf("cannot write key")
+	}
+	targ := filepath.Join(PkgMod, "../sumdb/"+file)
+	os.MkdirAll(filepath.Dir(targ), 0777)
+	f, err := lockedfile.Edit(targ)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return err
+	}
+	if len(data) > 0 && !bytes.Equal(data, old) {
+		return sumweb.ErrWriteConflict
+	}
+	if _, err := f.Seek(0, 0); err != nil {
+		return err
+	}
+	if err := f.Truncate(0); err != nil {
+		return err
+	}
+	if _, err := f.Write(new); err != nil {
+		return err
+	}
+	return f.Close()
+}
+
+// ReadCache reads cached lookups or tiles from
+// GOPATH/pkg/mod/download/cache/sumdb,
+// which will be deleted by "go clean -modcache".
+func (*dbClient) ReadCache(file string) ([]byte, error) {
+	targ := filepath.Join(PkgMod, "download/cache/sumdb", file)
+	return lockedfile.Read(targ)
+}
+
+// WriteCache updates cached lookups or tiles.
+func (*dbClient) WriteCache(file string, data []byte) {
+	targ := filepath.Join(PkgMod, "download/cache/sumdb", file)
+	os.MkdirAll(filepath.Dir(targ), 0777)
+	lockedfile.Write(targ, bytes.NewReader(data), 0666)
+}
+
+func (*dbClient) Log(msg string) {
+	// nothing for now
+}
+
+func (*dbClient) SecurityError(msg string) {
+	base.Fatalf("%s", msg)
+}

src/cmd/go/internal/sumweb/server.go

@@ -59,7 +59,7 @@ var Paths = []string{
 	"/tile/",
 }
 
-var modVerRE = regexp.MustCompile(`^[^@]+@v[0-9]+\.[0-9]+\.[0-9]+(-[^@]*)?$`)
+var modVerRE = regexp.MustCompile(`^[^@]+@v[0-9]+\.[0-9]+\.[0-9]+(-[^@]*)?(\+incompatible)?$`)
 
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	ctx, err := h.Server.NewContext(r)

src/cmd/go/internal/web/api.go

@@ -15,6 +15,7 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
+	"strings"
 )
 
 // SecurityMode specifies whether a function should make network
@@ -118,3 +119,15 @@ func Redacted(u *url.URL) string {
 func OpenBrowser(url string) (opened bool) {
 	return openBrowser(url)
 }
+
+// Join returns the result of adding the slash-separated
+// path elements to the end of u's path.
+func Join(u *url.URL, path string) *url.URL {
+	j := *u
+	if path == "" {
+		return &j
+	}
+	j.Path = strings.TrimSuffix(u.Path, "/") + "/" + strings.TrimPrefix(path, "/")
+	j.RawPath = strings.TrimSuffix(u.RawPath, "/") + "/" + strings.TrimPrefix(path, "/")
+	return &j
+}

src/cmd/go/proxy_test.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net"
@@ -21,11 +22,13 @@ import (
 	"sync"
 	"testing"
 
+	"cmd/go/internal/dirhash"
 	"cmd/go/internal/modfetch"
 	"cmd/go/internal/modfetch/codehost"
 	"cmd/go/internal/module"
 	"cmd/go/internal/par"
 	"cmd/go/internal/semver"
+	"cmd/go/internal/sumweb"
 	"cmd/go/internal/txtar"
 )
 
@@ -43,7 +46,6 @@ var proxyOnce sync.Once
 // The proxy serves from testdata/mod. See testdata/mod/README.
 func StartProxy() {
 	proxyOnce.Do(func() {
-		fmt.Fprintf(os.Stderr, "go test proxy starting\n")
 		readModList()
 		addr := *proxyAddr
 		if addr == "" {
@@ -59,6 +61,11 @@ func StartProxy() {
 		go func() {
 			log.Fatalf("go proxy: http.Serve: %v", http.Serve(l, http.HandlerFunc(proxyHandler)))
 		}()
+
+		// Prepopulate main sumdb.
+		for _, mod := range modList {
+			sumdbHandler.Server.Lookup(nil, mod.Path+"@"+mod.Version)
+		}
 	})
 }
 
@@ -82,7 +89,9 @@ func readModList() {
 		encPath := strings.ReplaceAll(name[:i], "_", "/")
 		path, err := module.DecodePath(encPath)
 		if err != nil {
+			if encPath != "example.com/invalidpath/v1" {
 				fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
+			}
 			continue
 		}
 		encVers := name[i+1:]
@@ -97,6 +106,15 @@ func readModList() {
 
 var zipCache par.Cache
 
+const (
+	testSumDBName        = "localhost.localdev/sumdb"
+	testSumDBVerifierKey = "localhost.localdev/sumdb+00000c67+AcTrnkbUA+TU4heY3hkjiSES/DSQniBqIeQ/YppAUtK6"
+	testSumDBSignerKey   = "PRIVATE+KEY+localhost.localdev/sumdb+00000c67+AXu6+oaVaOYuQOFrf1V59JK1owcFlJcHwwXHDfDGxSPk"
+)
+
+var sumdbHandler = &sumweb.Handler{Server: sumweb.NewTestServer(testSumDBSignerKey, proxyGoSum)}
+var sumdbWrongHandler = &sumweb.Handler{Server: sumweb.NewTestServer(testSumDBSignerKey, proxyGoSumWrong)}
+
 // proxyHandler serves the Go module proxy protocol.
 // See the proxy section of https://research.swtch.com/vgo-module.
 func proxyHandler(w http.ResponseWriter, r *http.Request) {
@@ -104,17 +122,66 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) {
 		http.NotFound(w, r)
 		return
 	}
-	path := strings.TrimPrefix(r.URL.Path, "/mod/")
+	path := r.URL.Path[len("/mod/"):]
+
+	// /mod/quiet/ does not print errors.
+	quiet := false
+	if strings.HasPrefix(path, "quiet/") {
+		path = path[len("quiet/"):]
+		quiet = true
+	}
 
-	// If asked for 404/abc, serve a 404.
+	// Next element may opt into special behavior.
 	if j := strings.Index(path, "/"); j >= 0 {
 		n, err := strconv.Atoi(path[:j])
 		if err == nil && n >= 200 {
 			w.WriteHeader(n)
 			return
 		}
+		if strings.HasPrefix(path, "sumdb-") {
+			n, err := strconv.Atoi(path[len("sumdb-"):j])
+			if err == nil && n >= 200 {
+				if strings.HasPrefix(path[j:], "/sumdb/") {
+					w.WriteHeader(n)
+					return
+				}
+				path = path[j+1:]
+			}
+		}
+	}
+
+	// Request for $GOPROXY/sumdb-direct is direct sumdb access.
+	// (Client thinks it is talking directly to a sumdb.)
+	if strings.HasPrefix(path, "sumdb-direct/") {
+		r.URL.Path = path[len("sumdb-direct"):]
+		sumdbHandler.ServeHTTP(w, r)
+		return
+	}
+
+	// Request for $GOPROXY/sumdb-wrong is direct sumdb access
+	// but all the hashes are wrong.
+	// (Client thinks it is talking directly to a sumdb.)
+	if strings.HasPrefix(path, "sumdb-wrong/") {
+		r.URL.Path = path[len("sumdb-wrong"):]
+		sumdbWrongHandler.ServeHTTP(w, r)
+		return
+	}
+
+	// Request for $GOPROXY/sumdb/<name>/supported
+	// is checking whether it's OK to access sumdb via the proxy.
+	if path == "sumdb/"+testSumDBName+"/supported" {
+		w.WriteHeader(200)
+		return
 	}
 
+	// Request for $GOPROXY/sumdb/<name>/... goes to sumdb.
+	if sumdbPrefix := "sumdb/" + testSumDBName + "/"; strings.HasPrefix(path, sumdbPrefix) {
+		r.URL.Path = path[len(sumdbPrefix)-1:]
+		sumdbHandler.ServeHTTP(w, r)
+		return
+	}
+
+	// Module proxy request: /mod/path/@v/version[.suffix]
 	i := strings.Index(path, "/@v/")
 	if i < 0 {
 		http.NotFound(w, r)
@@ -123,7 +190,9 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) {
 	enc, file := path[:i], path[i+len("/@v/"):]
 	path, err := module.DecodePath(enc)
 	if err != nil {
+		if !quiet {
 			fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
+		}
 		http.NotFound(w, r)
 		return
 	}
@@ -179,9 +248,11 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	a := readArchive(path, vers)
-	if a == nil {
-		fmt.Fprintf(os.Stderr, "go proxy: no archive %s %s\n", path, vers)
+	a, err := readArchive(path, vers)
+	if err != nil {
+		if !quiet {
+			fmt.Fprintf(os.Stderr, "go proxy: no archive %s %s: %v\n", path, vers, err)
+		}
 		http.Error(w, "cannot load archive", 500)
 		return
 	}
@@ -229,7 +300,9 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) {
 		}).(cached)
 
 		if c.err != nil {
+			if !quiet {
 				fmt.Fprintf(os.Stderr, "go proxy: %v\n", c.err)
+			}
 			http.Error(w, c.err.Error(), 500)
 			return
 		}
@@ -241,8 +314,8 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func findHash(m module.Version) string {
-	a := readArchive(m.Path, m.Version)
-	if a == nil {
+	a, err := readArchive(m.Path, m.Version)
+	if err != nil {
 		return ""
 	}
 	var data []byte
@@ -261,16 +334,14 @@ var archiveCache par.Cache
 
 var cmdGoDir, _ = os.Getwd()
 
-func readArchive(path, vers string) *txtar.Archive {
+func readArchive(path, vers string) (*txtar.Archive, error) {
 	enc, err := module.EncodePath(path)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "go proxy: %v\n", err)
-		return nil
+		return nil, err
 	}
 	encVers, err := module.EncodeVersion(vers)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "go proxy: %v\n", err)
-		return nil
+		return nil, err
 	}
 
 	prefix := strings.ReplaceAll(enc, "/", "_")
@@ -285,5 +356,51 @@ func readArchive(path, vers string) *txtar.Archive {
 		}
 		return a
 	}).(*txtar.Archive)
-	return a
+	if a == nil {
+		return nil, os.ErrNotExist
+	}
+	return a, nil
+}
+
+// proxyGoSum returns the two go.sum lines for path@vers.
+func proxyGoSum(path, vers string) ([]byte, error) {
+	a, err := readArchive(path, vers)
+	if err != nil {
+		return nil, err
+	}
+	var names []string
+	files := make(map[string][]byte)
+	var gomod []byte
+	for _, f := range a.Files {
+		if strings.HasPrefix(f.Name, ".") {
+			if f.Name == ".mod" {
+				gomod = f.Data
+			}
+			continue
+		}
+		name := path + "@" + vers + "/" + f.Name
+		names = append(names, name)
+		files[name] = f.Data
+	}
+	h1, err := dirhash.Hash1(names, func(name string) (io.ReadCloser, error) {
+		data := files[name]
+		return ioutil.NopCloser(bytes.NewReader(data)), nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	h1mod, err := dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
+		return ioutil.NopCloser(bytes.NewReader(gomod)), nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	data := []byte(fmt.Sprintf("%s %s %s\n%s %s/go.mod %s\n", path, vers, h1, path, vers, h1mod))
+	return data, nil
+}
+
+// proxyGoSumWrong returns the wrong lines.
+func proxyGoSumWrong(path, vers string) ([]byte, error) {
+	data := []byte(fmt.Sprintf("%s %s %s\n%s %s/go.mod %s\n", path, vers, "h1:wrong", path, vers, "h1:wrong"))
+	return data, nil
 }

src/cmd/go/script_test.go

@@ -110,7 +110,7 @@ func (ts *testScript) setup() {
 		"GOPATH=" + filepath.Join(ts.workdir, "gopath"),
 		"GOPROXY=" + proxyURL,
 		"GOROOT=" + testGOROOT,
-		"GONOVERIFY=*",
+		"GOSUMDB=" + testSumDBVerifierKey,
 		tempEnvName() + "=" + filepath.Join(ts.workdir, "tmp"),
 		"devnull=" + os.DevNull,
 		"goversion=" + goVersion(ts),

src/cmd/go/testdata/script/mod_auth.txt

@@ -2,6 +2,7 @@
 
 env GO111MODULE=on
 env GOPROXY=direct
+env GOSUMDB=off
 
 # Without credentials, downloading a module from a path that requires HTTPS
 # basic auth should fail.

src/cmd/go/testdata/script/mod_download.txt

 env GO111MODULE=on
+env GOPROXY=$GOPROXY/quiet
 
 # download with version should print nothing
 go mod download rsc.io/quote@v1.5.0

src/cmd/go/testdata/script/mod_download_hash.txt

@@ -3,7 +3,8 @@ env GO111MODULE=on
 # Testing mod download with non semantic versions; turn off proxy.
 [!net] skip
 [!exec:git] skip
-env GOPROXY=
+env GOPROXY=direct
+env GOSUMDB=off
 
 go mod download rsc.io/quote@a91498bed0a73d4bb9c1fb2597925f7883bc40a7
 exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v0.0.0-20180709162918-a91498bed0a7.info

src/cmd/go/testdata/script/mod_get_commit.txt

 env GO111MODULE=on
+env GOPROXY=$GOPROXY/quiet
 
 # @commit should resolve
 

src/cmd/go/testdata/script/mod_get_moved.txt

 env GO111MODULE=on
+env GOPROXY=$GOPROXY/quiet
 
 # A 'go get' that worked at a previous version should continue to work at that version,
 # even if the package was subsequently moved into a submodule.

src/cmd/go/testdata/script/mod_get_pseudo.txt

@@ -3,7 +3,8 @@ env GO111MODULE=on
 # Testing git->module converter's generation of +incompatible tags; turn off proxy.
 [!net] skip
 [!exec:git] skip
-env GOPROXY=
+env GOPROXY=direct
+env GOSUMDB=off
 
 # We can resolve the @master branch without unshallowing the local repository
 # (even with older gits), so try that before we do anything else.

src/cmd/go/testdata/script/mod_gopkg_unstable.txt

@@ -9,7 +9,8 @@ go list
 
 [!net] skip
 
-env GOPROXY=
+env GOPROXY=direct
+env GOSUMDB=off
 go get gopkg.in/macaroon-bakery.v2-unstable/bakery
 go list -m all
 stdout 'gopkg.in/macaroon-bakery.v2-unstable v2.0.0-[0-9]+-[0-9a-f]+$'

src/cmd/go/testdata/script/mod_notary.txt

-env GO111MODULE=on
-env GONOVERIFY=  # default in test scripts is *
-
-# notary should reject rsc.io/badsum v1.0.0 with good go.mod, bad tree
-cp go.mod.orig go.mod
-! go get rsc.io/badsum@v1.0.0
-stderr 'verifying rsc.io/badsum@v1.0.0: checksum mismatch'
-stderr 'notary: +h1:6/o\+QJfe6mFSNuegDihphabcvR94anXQk/qq7Enr19U='
-stderr 'SECURITY ERROR'
-stderr 'NOT match the expected download known to the notary'
-stderr 'go help module-auth'
-grep 'rsc.io/badsum v1.0.0/go.mod' go.sum
-! grep 'rsc.io/badsum v1.0.0 ' go.sum
-rm go.sum
-
-# notary should reject rsc.io/badsum v1.0.1 with bad go.mod, good tree
-cp go.mod.orig go.mod
-! go get rsc.io/badsum@v1.0.1
-stderr 'verifying rsc.io/badsum@v1.0.1/go.mod: checksum mismatch'
-stderr 'notary: +h1:avOsLUJaHavllihBU9qCTW37z64ypkZjqZg8O16JLVY='
-stderr 'SECURITY ERROR'
-stderr 'NOT match the expected download known to the notary'
-stderr 'go help module-auth'
-! exists go.sum # failed at go.mod, did not get to the tree
-
-# notary checks should run even without explicit go.mod
-rm go.mod
-rm go.sum
-! go get rsc.io/badsum@v1.0.0
-stderr 'verifying rsc.io/badsum@v1.0.0: checksum mismatch'
-stderr 'notary: +h1:6/o\+QJfe6mFSNuegDihphabcvR94anXQk/qq7Enr19U='
-stderr 'SECURITY ERROR'
-stderr 'NOT match the expected download known to the notary'
-stderr 'go help module-auth'
-! go get rsc.io/badsum@v1.0.1
-stderr 'verifying rsc.io/badsum@v1.0.1/go.mod: checksum mismatch'
-stderr 'notary: +h1:avOsLUJaHavllihBU9qCTW37z64ypkZjqZg8O16JLVY='
-stderr 'SECURITY ERROR'
-stderr 'NOT match the expected download known to the notary'
-stderr 'go help module-auth'
-! exists go.mod
-! exists go.sum
-
-# go get -insecure should skip notary without go.mod
-go get -insecure rsc.io/badsum@v1.0.0
-go get -insecure rsc.io/badsum@v1.0.1
-! exists go.mod
-! exists go.sum
-
-# GONOVERIFY should skip notary too
-env GONOVERIFY=rsc.i[aeiou]
-go get rsc.io/badsum@v1.0.0
-go get rsc.io/badsum@v1.0.1
-! exists go.mod
-! exists go.sum
-env GONOVERIFY=
-
-# go get -insecure should skip notary with go.mod and update go.sum
-cp go.mod.orig go.mod
-go get -insecure rsc.io/badsum@v1.0.0
-go get -insecure rsc.io/badsum@v1.0.1
-
-# go.sum should override notary
-cp go.mod.orig go.mod
-cp go.sum.wrong go.sum
-go get rsc.io/badsum@v1.0.0
-go get rsc.io/badsum@v1.0.1
-
--- go.mod.orig --
-module m
--- go.sum.wrong --
-rsc.io/badsum v1.0.0 h1:kZaLRhqz4LgsHPQddRMr+124lTgJfm28AxghGw3vLB0=
-rsc.io/badsum v1.0.1/go.mod h1:xwKWaN8OFR80Kg5/vdt+V2b+2P5kaLH+wWo5CL+pwHs=

src/cmd/go/testdata/script/mod_query_exclude.txt

 env GO111MODULE=on
+env GOPROXY=$GOPROXY/quiet
 
 # get excluded version
 cp go.mod1 go.mod

src/cmd/go/testdata/script/mod_sumdb.txt

+env GO111MODULE=on
+env sumdb=$GOSUMDB
+env proxy=$GOPROXY
+env GOPROXY GONOPROXY GOSUMDB GONOSUMDB
+env dbname=localhost.localdev/sumdb
+
+# disagreeing with the sumdb produces security errors
+# (this also populates tiles on the sumdb server).
+cp go.mod.orig go.mod
+env GOSUMDB=$sumdb' '$proxy/sumdb-wrong
+! go get rsc.io/quote
+stderr 'verifying rsc.io/quote@v1.5.2/go.mod: checksum mismatch'
+stderr 'downloaded: h1:LzX7'
+stderr 'localhost.localdev/sumdb: h1:wrong'
+stderr 'SECURITY ERROR\nThis download does NOT match the one reported by the checksum server.'
+! go get rsc.io/sampler
+! go get golang.org/x/text
+rm go.sum
+
+# switching to truthful sumdb detects timeline inconsistency
+cp go.mod.orig go.mod
+env GOSUMDB=$sumdb
+! go get -m rsc.io/fortune
+stderr 'SECURITY ERROR\ngo.sum database server misbehavior detected!'
+stderr 'proof of misbehavior:'
+
+# removing the cached wrong tree head and cached tiles clears the bad data
+rm $GOPATH/pkg/sumdb/$dbname/latest
+go clean -modcache
+go get -m rsc.io/fortune
+
+-- go.mod.orig --
+module m

src/cmd/go/testdata/script/mod_sumdb_cache.txt

+env GO111MODULE=on
+env sumdb=$GOSUMDB
+env proxy=$GOPROXY
+env GOPROXY GONOPROXY GOSUMDB GONOSUMDB
+
+# rejected proxy fails verification
+cp go.mod.orig go.mod
+rm go.sum
+env GOPROXY=$proxy/sumdb-503
+! go get -m rsc.io/quote
+stderr 503
+
+# fetch through working proxy is OK
+cp go.mod.orig go.mod
+rm go.sum
+env GOPROXY=$proxy
+go get -m rsc.io/quote
+
+# repeated fetch works entirely from cache, does not consult sumdb
+cp go.mod.orig go.mod
+rm go.sum
+env GOPROXY=$proxy/sumdb-503
+go get -m rsc.io/quote
+rm go.sum
+
+# fetch specific module can work without proxy, using cache or go.sum
+cp go.mod.orig go.mod
+rm go.sum
+env GOPROXY=off
+go get -m rsc.io/quote@v1.5.2 # using cache
+rm $GOPATH/pkg/mod/download/cache/sumdb/localhost.localdev/sumdb/lookup/rsc.io/quote@v1.5.2
+go get -m rsc.io/quote@v1.5.2 # using go.sum
+
+# fetch fails once we lose access to both cache and go.sum
+rm go.sum
+env GOPROXY=$proxy/sumdb-504
+! go get -m rsc.io/quote@v1.5.2
+stderr 504
+
+# but -insecure bypasses the checksum lookup entirely
+go get -m -insecure rsc.io/quote@v1.5.2
+
+# and then it is in go.sum again
+go get -m rsc.io/quote@v1.5.2
+
+-- go.mod.orig --
+module m

src/cmd/go/testdata/script/mod_sumdb_golang.txt

+[!net] skip
+
+env GOPROXY=
+env GOSUMDB=
+go env GOPROXY
+stdout '^direct$'
+go env GOSUMDB
+stdout '^off$'
+env GOPROXY=https://proxy.golang.org
+go env GOSUMDB
+stdout '^sum.golang.org$'
+
+# download direct from github
+env GOSUMDB=sum.golang.org
+env GOPROXY=direct
+go get -m rsc.io/quote
+
+# download from proxy.golang.org
+go clean -modcache
+env GOSUMDB='sum.golang.org https://sum.golang.org' # TODO remove URL
+env GOPROXY=https://proxy.golang.org
+go get -m rsc.io/quote
+
+-- go.mod --
+module m

src/cmd/go/testdata/script/mod_sumdb_proxy.txt

+env GO111MODULE=on
+env sumdb=$GOSUMDB
+env proxy=$GOPROXY
+env GOPROXY GONOPROXY GOSUMDB GONOSUMDB
+
+# basic fetch (through proxy) works
+cp go.mod.orig go.mod
+go get -m rsc.io/fortune@v1.0.0 # note: must use test proxy, does not exist in real world
+rm $GOPATH/pkg/mod/download/cache/sumdb # rm sumdb cache but NOT package download cache
+rm go.sum
+
+# can fetch by explicit URL
+cp go.mod.orig go.mod
+env GOSUMDB=$sumdb' '$proxy/sumdb-direct
+go get -m rsc.io/fortune@v1.0.0
+rm $GOPATH/pkg/mod/download/cache/sumdb
+rm go.sum
+
+# direct access fails (because localhost.localdev does not exist)
+cp go.mod.orig go.mod
+env GOSUMDB=$sumdb
+env GOPROXY=direct
+! go get -m rsc.io/fortune@v1.0.0
+stderr 'verifying.*localhost.localdev.*no such host'
+rm $GOPATH/pkg/mod/download/cache/sumdb
+rm go.sum
+
+# proxy 404 falls back to direct access (which fails)
+cp go.mod.orig go.mod
+env GOSUMDB=$sumdb
+env GOPROXY=$proxy/sumdb-404
+! go get -m rsc.io/fortune@v1.0.0
+stderr 'verifying.*localhost.localdev.*no such host'
+rm $GOPATH/pkg/mod/download/cache/sumdb
+rm go.sum
+
+# proxy non-200/404/410 stops direct access
+cp go.mod.orig go.mod
+env GOSUMDB=$sumdb
+env GOPROXY=$proxy/sumdb-503
+! go get -m rsc.io/fortune@v1.0.0
+stderr '503 Service Unavailable'
+rm $GOPATH/pkg/mod/download/cache/sumdb
+rm go.sum
+
+-- go.mod.orig --
+module m