cmd/go: add GOPRIVATE environment variable

It is too confusing to have to set GONOSUMDB and GONOPROXY
in common use cases, but one cannot be guaranteed to be a
subset of the other.

This CL adds GOPRIVATE, which takes the same kind of pattern list
but is defined as "these patterns are private (non-public) modules".
Today the implication is that GOPRIVATE is the default setting for
GONOSUMDB and GONOPROXY. If there are other accommodations
to make for private packages in the future or in other tools,
having this clear statement of intent will let us do that.
(For example maybe an IDE integration would hyperlink an import
path to godoc.org; consulting GOPRIVATE would be a reasonable
way to decide not to do that for certain imports. In contrast,
consulting GONOPROXY or GONOSUMDB clearly would not.)

Fixes #32184.

Change-Id: If54c12d353c7a0a5c0e0273764140cce3c154a02
Reviewed-on: https://go-review.googlesource.com/c/go/+/181719
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
Reviewed-by: Jay Conrod <jayconrod@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/cmd/go/alldocs.go

@@ -47,8 +47,9 @@
 // 	importpath  import path syntax
 // 	modules     modules, module versions, and more
 // 	module-get  module-aware go get
-// 	packages    package lists and patterns
 // 	module-auth module authentication using go.sum
+// 	module-private module configuration for non-public modules
+// 	packages    package lists and patterns
 // 	testflag    testing flags
 // 	testfunc    testing functions
 //
@@ -1568,19 +1569,16 @@
 // 		For more details see: 'go help gopath'.
 // 	GOPROXY
 // 		URL of Go module proxy. See 'go help modules'.
-// 	GONOPROXY
+// 	GOPRIVATE, GONOPROXY, GONOSUMDB
 // 		Comma-separated list of glob patterns (in the syntax of Go's path.Match)
-// 		of module path prefixes that should always be fetched directly, ignoring
-// 		the GOPROXY setting. See 'go help modules'.
+// 		of module path prefixes that should always be fetched directly
+// 		or that should not be compared against the checksum database.
+// 		See 'go help module-private'.
+// 	GOROOT
+// 		The root of the go tree.
 // 	GOSUMDB
 // 		The name of checksum database to use and optionally its public key and
 // 		URL. See 'go help module-auth'.
-// 	GONOSUMDB
-// 		Comma-separated list of glob patterns (in the syntax of Go's path.Match)
-// 		of module path prefixes that should not be compared against the checksum
-// 		database. See 'go help module-auth'.
-// 	GOROOT
-// 		The root of the go tree.
 // 	GOTMPDIR
 // 		The directory where the go command will write
 // 		temporary source files, packages, and binaries.
@@ -2598,19 +2596,8 @@
 // to cause a direct connection to be attempted at that point in the search.
 // Any proxies listed after "direct" are never consulted.
 //
-// The GONOPROXY environment variable is a comma-separated list of
-// glob patterns (in the syntax of Go's path.Match) of module path prefixes
-// that should always be fetched directly, ignoring the GOPROXY setting.
-// For example,
-//
-// 	GONOPROXY=*.corp.example.com,rsc.io/private
-//
-// forces a direct connection to download modules with path prefixes matching
-// either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private",
-// and "rsc.io/private/quux".
-//
-// The 'go env -w' command (see 'go help env') can be used to set these variables
-// for future go command invocations.
+// The GOPRIVATE and GONOPROXY environment variables allow bypassing
+// the proxy for selected modules. See 'go help module-private' for details.
 //
 // No matter the source of the modules, the go command checks downloads against
 // known checksums, to detect unexpected changes in the content of any specific
@@ -2642,90 +2629,6 @@
 // are still ignored.
 //
 //
-// Package lists and patterns
-//
-// Many commands apply to a set of packages:
-//
-// 	go action [packages]
-//
-// Usually, [packages] is a list of import paths.
-//
-// An import path that is a rooted path or that begins with
-// a . or .. element is interpreted as a file system path and
-// denotes the package in that directory.
-//
-// Otherwise, the import path P denotes the package found in
-// the directory DIR/src/P for some DIR listed in the GOPATH
-// environment variable (For more details see: 'go help gopath').
-//
-// If no import paths are given, the action applies to the
-// package in the current directory.
-//
-// There are four reserved names for paths that should not be used
-// for packages to be built with the go tool:
-//
-// - "main" denotes the top-level package in a stand-alone executable.
-//
-// - "all" expands to all packages found in all the GOPATH
-// trees. For example, 'go list all' lists all the packages on the local
-// system. When using modules, "all" expands to all packages in
-// the main module and their dependencies, including dependencies
-// needed by tests of any of those.
-//
-// - "std" is like all but expands to just the packages in the standard
-// Go library.
-//
-// - "cmd" expands to the Go repository's commands and their
-// internal libraries.
-//
-// Import paths beginning with "cmd/" only match source code in
-// the Go repository.
-//
-// An import path is a pattern if it includes one or more "..." wildcards,
-// each of which can match any string, including the empty string and
-// strings containing slashes. Such a pattern expands to all package
-// directories found in the GOPATH trees with names matching the
-// patterns.
-//
-// To make common patterns more convenient, there are two special cases.
-// First, /... at the end of the pattern can match an empty string,
-// so that net/... matches both net and packages in its subdirectories, like net/http.
-// Second, any slash-separated pattern element containing a wildcard never
-// participates in a match of the "vendor" element in the path of a vendored
-// package, so that ./... does not match packages in subdirectories of
-// ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do.
-// Note, however, that a directory named vendor that itself contains code
-// is not a vendored package: cmd/vendor would be a command named vendor,
-// and the pattern cmd/... matches it.
-// See golang.org/s/go15vendor for more about vendoring.
-//
-// An import path can also name a package to be downloaded from
-// a remote repository. Run 'go help importpath' for details.
-//
-// Every package in a program must have a unique import path.
-// By convention, this is arranged by starting each path with a
-// unique prefix that belongs to you. For example, paths used
-// internally at Google all begin with 'google', and paths
-// denoting remote repositories begin with the path to the code,
-// such as 'github.com/user/repo'.
-//
-// Packages in a program need not have unique package names,
-// but there are two reserved package names with special meaning.
-// The name main indicates a command, not a library.
-// Commands are built into binaries and cannot be imported.
-// The name documentation indicates documentation for
-// a non-Go program in the directory. Files in package documentation
-// are ignored by the go command.
-//
-// As a special case, if the package list is a list of .go files from a
-// single directory, the command is applied to a single synthesized
-// package made up of exactly those files, ignoring any build constraints
-// in those files and ignoring any other files in the directory.
-//
-// Directory and file names that begin with "." or "_" are ignored
-// by the go tool, as are directories named "testdata".
-//
-//
 // Module authentication using go.sum
 //
 // The go command tries to authenticate every downloaded module,
@@ -2805,23 +2708,142 @@
 // the checksum database is not consulted, and all unrecognized modules are
 // accepted, at the cost of giving up the security guarantee of verified repeatable
 // downloads for all modules. A better way to bypass the checksum database
-// for specific modules is to use the GONOSUMDB environment variable.
+// for specific modules is to use the GOPRIVATE or GONOSUMDB environment
+// variables. See 'go help module-private' for details.
+//
+// The 'go env -w' command (see 'go help env') can be used to set these variables
+// for future go command invocations.
+//
+//
+// Module configuration for non-public modules
 //
-// The GONOSUMDB environment variable is a comma-separated list of
-// glob patterns (in the syntax of Go's path.Match) of module path prefixes
-// that should not be compared against the checksum database.
+// The go command defaults to downloading modules from the public Go module
+// mirror at proxy.golang.org. It also defaults to validating downloaded modules,
+// regardless of source, against the public Go checksum database at sum.golang.org.
+// These defaults work well for publicly available source code.
+//
+// The GOPRIVATE environment variable controls which modules the go command
+// considers to be private (not available publicly) and should therefore not use the
+// proxy or checksum database. The variable is a comma-separated list of
+// glob patterns (in the syntax of Go's path.Match) of module path prefixes.
 // For example,
 //
-// 	GONOSUMDB=*.corp.example.com,rsc.io/private
+// 	GOPRIVATE=*.corp.example.com,rsc.io/private
+//
+// causes the go command to treat as private any module with a path prefix
+// matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
+// and rsc.io/private/quux.
+//
+// The GOPRIVATE environment variable may be used by other tools as well to
+// identify non-public modules. For example, an editor could use GOPRIVATE
+// to decide whether to hyperlink a package import to a godoc.org page.
+//
+// For fine-grained control over module download and validation, the GONOPROXY
+// and GONOSUMDB environment variables accept the same kind of glob list
+// and override GOPRIVATE for the specific decision of whether to use the proxy
+// and checksum database, respectively.
+//
+// For example, if a company ran a module proxy serving private modules,
+// users would configure go using:
+//
+// 	GOPRIVATE=*.corp.example.com
+// 	GOPROXY=proxy.example.com
+// 	GONOPROXY=none
 //
-// disables checksum database lookups for modules with path prefixes matching
-// either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private",
-// and "rsc.io/private/quux".
+// This would tell the go comamnd and other tools that modules beginning with
+// a corp.example.com subdomain are private but that the company proxy should
+// be used for downloading both public and private modules, because
+// GONOPROXY has been set to a pattern that won't match any modules,
+// overriding GOPRIVATE.
 //
 // The 'go env -w' command (see 'go help env') can be used to set these variables
 // for future go command invocations.
 //
 //
+// Package lists and patterns
+//
+// Many commands apply to a set of packages:
+//
+// 	go action [packages]
+//
+// Usually, [packages] is a list of import paths.
+//
+// An import path that is a rooted path or that begins with
+// a . or .. element is interpreted as a file system path and
+// denotes the package in that directory.
+//
+// Otherwise, the import path P denotes the package found in
+// the directory DIR/src/P for some DIR listed in the GOPATH
+// environment variable (For more details see: 'go help gopath').
+//
+// If no import paths are given, the action applies to the
+// package in the current directory.
+//
+// There are four reserved names for paths that should not be used
+// for packages to be built with the go tool:
+//
+// - "main" denotes the top-level package in a stand-alone executable.
+//
+// - "all" expands to all packages found in all the GOPATH
+// trees. For example, 'go list all' lists all the packages on the local
+// system. When using modules, "all" expands to all packages in
+// the main module and their dependencies, including dependencies
+// needed by tests of any of those.
+//
+// - "std" is like all but expands to just the packages in the standard
+// Go library.
+//
+// - "cmd" expands to the Go repository's commands and their
+// internal libraries.
+//
+// Import paths beginning with "cmd/" only match source code in
+// the Go repository.
+//
+// An import path is a pattern if it includes one or more "..." wildcards,
+// each of which can match any string, including the empty string and
+// strings containing slashes. Such a pattern expands to all package
+// directories found in the GOPATH trees with names matching the
+// patterns.
+//
+// To make common patterns more convenient, there are two special cases.
+// First, /... at the end of the pattern can match an empty string,
+// so that net/... matches both net and packages in its subdirectories, like net/http.
+// Second, any slash-separated pattern element containing a wildcard never
+// participates in a match of the "vendor" element in the path of a vendored
+// package, so that ./... does not match packages in subdirectories of
+// ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do.
+// Note, however, that a directory named vendor that itself contains code
+// is not a vendored package: cmd/vendor would be a command named vendor,
+// and the pattern cmd/... matches it.
+// See golang.org/s/go15vendor for more about vendoring.
+//
+// An import path can also name a package to be downloaded from
+// a remote repository. Run 'go help importpath' for details.
+//
+// Every package in a program must have a unique import path.
+// By convention, this is arranged by starting each path with a
+// unique prefix that belongs to you. For example, paths used
+// internally at Google all begin with 'google', and paths
+// denoting remote repositories begin with the path to the code,
+// such as 'github.com/user/repo'.
+//
+// Packages in a program need not have unique package names,
+// but there are two reserved package names with special meaning.
+// The name main indicates a command, not a library.
+// Commands are built into binaries and cannot be imported.
+// The name documentation indicates documentation for
+// a non-Go program in the directory. Files in package documentation
+// are ignored by the go command.
+//
+// As a special case, if the package list is a list of .go files from a
+// single directory, the command is applied to a single synthesized
+// package made up of exactly those files, ignoring any build constraints
+// in those files and ignoring any other files in the directory.
+//
+// Directory and file names that begin with "." or "_" are ignored
+// by the go tool, as are directories named "testdata".
+//
+//
 // Testing flags
 //
 // The 'go test' command takes both flags that apply to 'go test' itself

src/cmd/go/internal/cfg/cfg.go

@@ -265,6 +265,7 @@ var knownEnv = `
 	GOOS
 	GOPATH
 	GOPPC64
+	GOPRIVATE
 	GOPROXY
 	GOROOT
 	GOSUMDB
@@ -291,30 +292,13 @@ var (
 	GOPPC64  = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", objabi.GOPPC64))
 	GOWASM   = envOr("GOWASM", fmt.Sprint(objabi.GOWASM))
 
-	GOPROXY   = goproxy()
-	GOSUMDB   = gosumdb()
-	GONOPROXY = Getenv("GONOPROXY")
-	GONOSUMDB = Getenv("GONOSUMDB")
+	GOPROXY   = envOr("GOPROXY", "https://proxy.golang.org,direct")
+	GOSUMDB   = envOr("GOSUMDB", "sum.golang.org")
+	GOPRIVATE = Getenv("GOPRIVATE")
+	GONOPROXY = envOr("GONOPROXY", GOPRIVATE)
+	GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE)
 )
 
-func goproxy() string {
-	v := Getenv("GOPROXY")
-	if v != "" {
-		return v
-	}
-
-	return "https://proxy.golang.org,direct"
-}
-
-func gosumdb() string {
-	v := Getenv("GOSUMDB")
-	if v != "" {
-		return v
-	}
-
-	return "sum.golang.org"
-}
-
 // GetArchEnv returns the name and setting of the
 // GOARCH-specific architecture environment variable.
 // If the current architecture has no GOARCH-specific variable,

src/cmd/go/internal/envcmd/env.go

@@ -79,6 +79,7 @@ func MkEnv() []cfg.EnvVar {
 		{Name: "GONOSUMDB", Value: cfg.GONOSUMDB},
 		{Name: "GOOS", Value: cfg.Goos},
 		{Name: "GOPATH", Value: cfg.BuildContext.GOPATH},
+		{Name: "GOPRIVATE", Value: cfg.GOPRIVATE},
 		{Name: "GOPROXY", Value: cfg.GOPROXY},
 		{Name: "GOROOT", Value: cfg.GOROOT},
 		{Name: "GOSUMDB", Value: cfg.GOSUMDB},

src/cmd/go/internal/help/helpdoc.go

@@ -510,19 +510,16 @@ General-purpose environment variables:
 		For more details see: 'go help gopath'.
 	GOPROXY
 		URL of Go module proxy. See 'go help modules'.
-	GONOPROXY
+	GOPRIVATE, GONOPROXY, GONOSUMDB
 		Comma-separated list of glob patterns (in the syntax of Go's path.Match)
-		of module path prefixes that should always be fetched directly, ignoring
-		the GOPROXY setting. See 'go help modules'.
+		of module path prefixes that should always be fetched directly
+		or that should not be compared against the checksum database.
+		See 'go help module-private'.
+	GOROOT
+		The root of the go tree.
 	GOSUMDB
 		The name of checksum database to use and optionally its public key and
 		URL. See 'go help module-auth'.
-	GONOSUMDB
-		Comma-separated list of glob patterns (in the syntax of Go's path.Match)
-		of module path prefixes that should not be compared against the checksum
-		database. See 'go help module-auth'.
-	GOROOT
-		The root of the go tree.
 	GOTMPDIR
 		The directory where the go command will write
 		temporary source files, packages, and binaries.

src/cmd/go/internal/modfetch/fetch.go

@@ -631,7 +631,7 @@ For more information, see 'go help module-auth'.
 
 `
 
-var HelpSum = &base.Command{
+var HelpModuleAuth = &base.Command{
 	UsageLine: "module-auth",
 	Short:     "module authentication using go.sum",
 	Long: `
@@ -712,18 +712,56 @@ If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag,
 the checksum database is not consulted, and all unrecognized modules are
 accepted, at the cost of giving up the security guarantee of verified repeatable
 downloads for all modules. A better way to bypass the checksum database
-for specific modules is to use the GONOSUMDB environment variable.
+for specific modules is to use the GOPRIVATE or GONOSUMDB environment
+variables. See 'go help module-private' for details.
 
-The GONOSUMDB environment variable is a comma-separated list of
-glob patterns (in the syntax of Go's path.Match) of module path prefixes
-that should not be compared against the checksum database.
+The 'go env -w' command (see 'go help env') can be used to set these variables
+for future go command invocations.
+`,
+}
+
+var HelpModulePrivate = &base.Command{
+	UsageLine: "module-private",
+	Short:     "module configuration for non-public modules",
+	Long: `
+The go command defaults to downloading modules from the public Go module
+mirror at proxy.golang.org. It also defaults to validating downloaded modules,
+regardless of source, against the public Go checksum database at sum.golang.org.
+These defaults work well for publicly available source code.
+
+The GOPRIVATE environment variable controls which modules the go command
+considers to be private (not available publicly) and should therefore not use the
+proxy or checksum database. The variable is a comma-separated list of
+glob patterns (in the syntax of Go's path.Match) of module path prefixes.
 For example,
 
-	GONOSUMDB=*.corp.example.com,rsc.io/private
+	GOPRIVATE=*.corp.example.com,rsc.io/private
+
+causes the go command to treat as private any module with a path prefix
+matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
+and rsc.io/private/quux.
+
+The GOPRIVATE environment variable may be used by other tools as well to
+identify non-public modules. For example, an editor could use GOPRIVATE
+to decide whether to hyperlink a package import to a godoc.org page.
+
+For fine-grained control over module download and validation, the GONOPROXY
+and GONOSUMDB environment variables accept the same kind of glob list
+and override GOPRIVATE for the specific decision of whether to use the proxy
+and checksum database, respectively.
+
+For example, if a company ran a module proxy serving private modules,
+users would configure go using:
+
+	GOPRIVATE=*.corp.example.com
+	GOPROXY=proxy.example.com
+	GONOPROXY=none
 
-disables checksum database lookups for modules with path prefixes matching
-either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private",
-and "rsc.io/private/quux".
+This would tell the go comamnd and other tools that modules beginning with
+a corp.example.com subdomain are private but that the company proxy should
+be used for downloading both public and private modules, because
+GONOPROXY has been set to a pattern that won't match any modules,
+overriding GOPRIVATE.
 
 The 'go env -w' command (see 'go help env') can be used to set these variables
 for future go command invocations.

src/cmd/go/internal/modfetch/repo.go

@@ -241,8 +241,8 @@ func lookup(proxy, path string) (r Repo, err error) {
 var (
 	errModVendor       = errors.New("module lookup disabled by -mod=vendor")
 	errProxyOff        = errors.New("module lookup disabled by GOPROXY=off")
-	errNoproxy   error = notExistError("disabled by GONOPROXY")
-	errUseProxy  error = notExistError("path does not match GONOPROXY")
+	errNoproxy   error = notExistError("disabled by GOPRIVATE/GONOPROXY")
+	errUseProxy  error = notExistError("path does not match GOPRIVATE/GONOPROXY")
 )
 
 func lookupDirect(path string) (Repo, error) {

src/cmd/go/internal/modload/help.go

@@ -348,19 +348,8 @@ HTTP response. The string "direct" may appear in the proxy list,
 to cause a direct connection to be attempted at that point in the search.
 Any proxies listed after "direct" are never consulted.
 
-The GONOPROXY environment variable is a comma-separated list of
-glob patterns (in the syntax of Go's path.Match) of module path prefixes
-that should always be fetched directly, ignoring the GOPROXY setting.
-For example,
-
-	GONOPROXY=*.corp.example.com,rsc.io/private
-
-forces a direct connection to download modules with path prefixes matching
-either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private",
-and "rsc.io/private/quux".
-
-The 'go env -w' command (see 'go help env') can be used to set these variables
-for future go command invocations.
+The GOPRIVATE and GONOPROXY environment variables allow bypassing
+the proxy for selected modules. See 'go help module-private' for details.
 
 No matter the source of the modules, the go command checks downloads against
 known checksums, to detect unexpected changes in the content of any specific

src/cmd/go/main.go

@@ -71,8 +71,9 @@ func init() {
 		help.HelpImportPath,
 		modload.HelpModules,
 		modget.HelpModuleGet,
+		modfetch.HelpModuleAuth,
+		modfetch.HelpModulePrivate,
 		help.HelpPackages,
-		modfetch.HelpSum,
 		test.HelpTestflag,
 		test.HelpTestfunc,
 	}

src/cmd/go/script_test.go

@@ -111,6 +111,7 @@ func (ts *testScript) setup() {
 		"GOOS=" + runtime.GOOS,
 		"GOPATH=" + filepath.Join(ts.workdir, "gopath"),
 		"GOPROXY=" + proxyURL,
+		"GOPRIVATE=",
 		"GOROOT=" + testGOROOT,
 		"GOSUMDB=" + testSumDBVerifierKey,
 		"GONOPROXY=",

src/cmd/go/testdata/script/mod_gonoproxy.txt

 env GO111MODULE=on
 env sumdb=$GOSUMDB
 env proxy=$GOPROXY
-env GOPROXY GONOPROXY GOSUMDB GONOSUMDB
+env GOPRIVATE GOPROXY GONOPROXY GOSUMDB GONOSUMDB
 env dbname=localhost.localdev/sumdb
 
 # disagree with sumdb fails
@@ -13,20 +13,25 @@ stderr 'SECURITY ERROR'
 # but GONOSUMDB bypasses sumdb, for rsc.io/quote, rsc.io/sampler, golang.org/x/text
 env GONOSUMDB='*/quote,*/*mple*,golang.org/x'
 go get rsc.io/quote
+rm go.sum
+env GOPRIVATE='*/quote,*/*mple*,golang.org/x'
+env GONOPROXY=none # that is, proxy all despite GOPRIVATE
+go get rsc.io/quote
 
 # and GONOPROXY bypasses proxy
 [!net] skip
 [!exec:git] skip
+env GOPRIVATE=none
 env GONOPROXY='*/fortune'
 ! go get rsc.io/fortune # does not exist in real world, only on test proxy
 stderr 'git ls-remote'
 
 env GOSUMDB=
-env GONOPROXY='*/x'
+env GONOPROXY=
+env GOPRIVATE='*/x'
 go get golang.org/x/text
 go list -m all
 ! stdout 'text.*v0.0.0-2017' # should not have the version from the proxy
 
-
 -- go.mod.orig --
 module m