cmd/go: add GOPRIVATE environment variable

It is too confusing to have to set GONOSUMDB and GONOPROXY
in common use cases, but one cannot be guaranteed to be a
subset of the other.

This CL adds GOPRIVATE, which takes the same kind of pattern list
but is defined as "these patterns are private (non-public) modules".
Today the implication is that GOPRIVATE is the default setting for
GONOSUMDB and GONOPROXY. If there are other accommodations
to make for private packages in the future or in other tools,
having this clear statement of intent will let us do that.
(For example maybe an IDE integration would hyperlink an import
path to godoc.org; consulting GOPRIVATE would be a reasonable
way to decide not to do that for certain imports. In contrast,
consulting GONOPROXY or GONOSUMDB clearly would not.)

Fixes #32184.

Change-Id: If54c12d353c7a0a5c0e0273764140cce3c154a02
Reviewed-on: https://go-review.googlesource.com/c/go/+/181719
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
Reviewed-by: Jay Conrod <jayconrod@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/cmd/go/internal/cfg/cfg.go

@@ -265,6 +265,7 @@ var knownEnv = `
 	GOOS
 	GOPATH
 	GOPPC64
+	GOPRIVATE
 	GOPROXY
 	GOROOT
 	GOSUMDB
@@ -291,30 +292,13 @@ var (
 	GOPPC64  = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", objabi.GOPPC64))
 	GOWASM   = envOr("GOWASM", fmt.Sprint(objabi.GOWASM))
 
-	GOPROXY   = goproxy()
-	GOSUMDB   = gosumdb()
-	GONOPROXY = Getenv("GONOPROXY")
-	GONOSUMDB = Getenv("GONOSUMDB")
+	GOPROXY   = envOr("GOPROXY", "https://proxy.golang.org,direct")
+	GOSUMDB   = envOr("GOSUMDB", "sum.golang.org")
+	GOPRIVATE = Getenv("GOPRIVATE")
+	GONOPROXY = envOr("GONOPROXY", GOPRIVATE)
+	GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE)
 )
 
-func goproxy() string {
-	v := Getenv("GOPROXY")
-	if v != "" {
-		return v
-	}
-
-	return "https://proxy.golang.org,direct"
-}
-
-func gosumdb() string {
-	v := Getenv("GOSUMDB")
-	if v != "" {
-		return v
-	}
-
-	return "sum.golang.org"
-}
-
 // GetArchEnv returns the name and setting of the
 // GOARCH-specific architecture environment variable.
 // If the current architecture has no GOARCH-specific variable,

src/cmd/go/internal/envcmd/env.go

@@ -79,6 +79,7 @@ func MkEnv() []cfg.EnvVar {
 		{Name: "GONOSUMDB", Value: cfg.GONOSUMDB},
 		{Name: "GOOS", Value: cfg.Goos},
 		{Name: "GOPATH", Value: cfg.BuildContext.GOPATH},
+		{Name: "GOPRIVATE", Value: cfg.GOPRIVATE},
 		{Name: "GOPROXY", Value: cfg.GOPROXY},
 		{Name: "GOROOT", Value: cfg.GOROOT},
 		{Name: "GOSUMDB", Value: cfg.GOSUMDB},

src/cmd/go/internal/help/helpdoc.go

@@ -510,19 +510,16 @@ General-purpose environment variables:
 		For more details see: 'go help gopath'.
 	GOPROXY
 		URL of Go module proxy. See 'go help modules'.
-	GONOPROXY
+	GOPRIVATE, GONOPROXY, GONOSUMDB
 		Comma-separated list of glob patterns (in the syntax of Go's path.Match)
-		of module path prefixes that should always be fetched directly, ignoring
-		the GOPROXY setting. See 'go help modules'.
+		of module path prefixes that should always be fetched directly
+		or that should not be compared against the checksum database.
+		See 'go help module-private'.
+	GOROOT
+		The root of the go tree.
 	GOSUMDB
 		The name of checksum database to use and optionally its public key and
 		URL. See 'go help module-auth'.
-	GONOSUMDB
-		Comma-separated list of glob patterns (in the syntax of Go's path.Match)
-		of module path prefixes that should not be compared against the checksum
-		database. See 'go help module-auth'.
-	GOROOT
-		The root of the go tree.
 	GOTMPDIR
 		The directory where the go command will write
 		temporary source files, packages, and binaries.

src/cmd/go/internal/modfetch/fetch.go

@@ -631,7 +631,7 @@ For more information, see 'go help module-auth'.
 
 `
 
-var HelpSum = &base.Command{
+var HelpModuleAuth = &base.Command{
 	UsageLine: "module-auth",
 	Short:     "module authentication using go.sum",
 	Long: `
@@ -712,18 +712,56 @@ If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag,
 the checksum database is not consulted, and all unrecognized modules are
 accepted, at the cost of giving up the security guarantee of verified repeatable
 downloads for all modules. A better way to bypass the checksum database
-for specific modules is to use the GONOSUMDB environment variable.
+for specific modules is to use the GOPRIVATE or GONOSUMDB environment
+variables. See 'go help module-private' for details.
 
-The GONOSUMDB environment variable is a comma-separated list of
-glob patterns (in the syntax of Go's path.Match) of module path prefixes
-that should not be compared against the checksum database.
+The 'go env -w' command (see 'go help env') can be used to set these variables
+for future go command invocations.
+`,
+}
+
+var HelpModulePrivate = &base.Command{
+	UsageLine: "module-private",
+	Short:     "module configuration for non-public modules",
+	Long: `
+The go command defaults to downloading modules from the public Go module
+mirror at proxy.golang.org. It also defaults to validating downloaded modules,
+regardless of source, against the public Go checksum database at sum.golang.org.
+These defaults work well for publicly available source code.
+
+The GOPRIVATE environment variable controls which modules the go command
+considers to be private (not available publicly) and should therefore not use the
+proxy or checksum database. The variable is a comma-separated list of
+glob patterns (in the syntax of Go's path.Match) of module path prefixes.
 For example,
 
-	GONOSUMDB=*.corp.example.com,rsc.io/private
+	GOPRIVATE=*.corp.example.com,rsc.io/private
+
+causes the go command to treat as private any module with a path prefix
+matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
+and rsc.io/private/quux.
+
+The GOPRIVATE environment variable may be used by other tools as well to
+identify non-public modules. For example, an editor could use GOPRIVATE
+to decide whether to hyperlink a package import to a godoc.org page.
+
+For fine-grained control over module download and validation, the GONOPROXY
+and GONOSUMDB environment variables accept the same kind of glob list
+and override GOPRIVATE for the specific decision of whether to use the proxy
+and checksum database, respectively.
+
+For example, if a company ran a module proxy serving private modules,
+users would configure go using:
+
+	GOPRIVATE=*.corp.example.com
+	GOPROXY=proxy.example.com
+	GONOPROXY=none
 
-disables checksum database lookups for modules with path prefixes matching
-either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private",
-and "rsc.io/private/quux".
+This would tell the go comamnd and other tools that modules beginning with
+a corp.example.com subdomain are private but that the company proxy should
+be used for downloading both public and private modules, because
+GONOPROXY has been set to a pattern that won't match any modules,
+overriding GOPRIVATE.
 
 The 'go env -w' command (see 'go help env') can be used to set these variables
 for future go command invocations.

src/cmd/go/internal/modfetch/repo.go

@@ -241,8 +241,8 @@ func lookup(proxy, path string) (r Repo, err error) {
 var (
 	errModVendor       = errors.New("module lookup disabled by -mod=vendor")
 	errProxyOff        = errors.New("module lookup disabled by GOPROXY=off")
-	errNoproxy   error = notExistError("disabled by GONOPROXY")
-	errUseProxy  error = notExistError("path does not match GONOPROXY")
+	errNoproxy   error = notExistError("disabled by GOPRIVATE/GONOPROXY")
+	errUseProxy  error = notExistError("path does not match GOPRIVATE/GONOPROXY")
 )
 
 func lookupDirect(path string) (Repo, error) {

src/cmd/go/internal/modload/help.go

@@ -348,19 +348,8 @@ HTTP response. The string "direct" may appear in the proxy list,
 to cause a direct connection to be attempted at that point in the search.
 Any proxies listed after "direct" are never consulted.
 
-The GONOPROXY environment variable is a comma-separated list of
-glob patterns (in the syntax of Go's path.Match) of module path prefixes
-that should always be fetched directly, ignoring the GOPROXY setting.
-For example,
-
-	GONOPROXY=*.corp.example.com,rsc.io/private
-
-forces a direct connection to download modules with path prefixes matching
-either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private",
-and "rsc.io/private/quux".
-
-The 'go env -w' command (see 'go help env') can be used to set these variables
-for future go command invocations.
+The GOPRIVATE and GONOPROXY environment variables allow bypassing
+the proxy for selected modules. See 'go help module-private' for details.
 
 No matter the source of the modules, the go command checks downloads against
 known checksums, to detect unexpected changes in the content of any specific

src/cmd/go/main.go

@@ -71,8 +71,9 @@ func init() {
 		help.HelpImportPath,
 		modload.HelpModules,
 		modget.HelpModuleGet,
+		modfetch.HelpModuleAuth,
+		modfetch.HelpModulePrivate,
 		help.HelpPackages,
-		modfetch.HelpSum,
 		test.HelpTestflag,
 		test.HelpTestfunc,
 	}

src/cmd/go/script_test.go

@@ -111,6 +111,7 @@ func (ts *testScript) setup() {
 		"GOOS=" + runtime.GOOS,
 		"GOPATH=" + filepath.Join(ts.workdir, "gopath"),
 		"GOPROXY=" + proxyURL,
+		"GOPRIVATE=",
 		"GOROOT=" + testGOROOT,
 		"GOSUMDB=" + testSumDBVerifierKey,
 		"GONOPROXY=",

src/cmd/go/testdata/script/mod_gonoproxy.txt

 env GO111MODULE=on
 env sumdb=$GOSUMDB
 env proxy=$GOPROXY
-env GOPROXY GONOPROXY GOSUMDB GONOSUMDB
+env GOPRIVATE GOPROXY GONOPROXY GOSUMDB GONOSUMDB
 env dbname=localhost.localdev/sumdb
 
 # disagree with sumdb fails
@@ -13,20 +13,25 @@ stderr 'SECURITY ERROR'
 # but GONOSUMDB bypasses sumdb, for rsc.io/quote, rsc.io/sampler, golang.org/x/text
 env GONOSUMDB='*/quote,*/*mple*,golang.org/x'
 go get rsc.io/quote
+rm go.sum
+env GOPRIVATE='*/quote,*/*mple*,golang.org/x'
+env GONOPROXY=none # that is, proxy all despite GOPRIVATE
+go get rsc.io/quote
 
 # and GONOPROXY bypasses proxy
 [!net] skip
 [!exec:git] skip
+env GOPRIVATE=none
 env GONOPROXY='*/fortune'
 ! go get rsc.io/fortune # does not exist in real world, only on test proxy
 stderr 'git ls-remote'
 
 env GOSUMDB=
-env GONOPROXY='*/x'
+env GONOPROXY=
+env GOPRIVATE='*/x'
 go get golang.org/x/text
 go list -m all
 ! stdout 'text.*v0.0.0-2017' # should not have the version from the proxy
 
-
 -- go.mod.orig --
 module m