cmd/go: respect default proxy setting, add direct fallback

Getenv("GOPROXY") says what the environment variable is
(including looking in the go env file), but it doesn't include
the default setting. This code needs to use cfg.GOPROXY
to get the actual default. Fix and test that.

Also, we forgot to include the fallback to direct for when
the proxy serves a 404. Add and test that too.

Also add HTTP fetch information to -x build flag output.
(It does not belong in the -v output, despite the GOPATH go get
command doing this.)

Change-Id: Ieab7ef13cda3e1ad041dbe04921af206e2232c9c
Reviewed-on: https://go-review.googlesource.com/c/go/+/178720
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
Reviewed-by: Hyang-Ah Hana Kim <hyangah@gmail.com>

src/cmd/go/internal/cfg/cfg.go

@@ -303,7 +303,7 @@ func goproxy() string {
 		return v
 	}
 
-	return "https://proxy.golang.org"
+	return "https://proxy.golang.org,direct"
 }
 
 func gosumdb() string {

src/cmd/go/internal/web/http.go

@@ -14,9 +14,11 @@ package web
 import (
 	"crypto/tls"
 	"fmt"
-	"log"
+	"io/ioutil"
 	"net/http"
 	urlpkg "net/url"
+	"os"
+	"strings"
 	"time"
 
 	"cmd/go/internal/auth"
@@ -50,9 +52,28 @@ var securityPreservingHTTPClient = &http.Client{
 }
 
 func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
+	start := time.Now()
+	if os.Getenv("TESTGOPROXY404") == "1" && url.Host == "proxy.golang.org" {
+		res := &Response{
+			URL:        Redacted(url),
+			Status:     "404 testing",
+			StatusCode: 404,
+			Header:     make(map[string][]string),
+			Body:       ioutil.NopCloser(strings.NewReader("")),
+		}
+		if cfg.BuildX {
+			fmt.Fprintf(os.Stderr, "# get %s: %v (%.3fs)\n", Redacted(url), res.Status, time.Since(start).Seconds())
+		}
+		return res, nil
+	}
+
 	fetch := func(url *urlpkg.URL) (*urlpkg.URL, *http.Response, error) {
-		if cfg.BuildV {
-			log.Printf("Fetching %s", url)
+		// Note: The -v build flag does not mean "print logging information",
+		// despite its historical misuse for this in GOPATH-based go get.
+		// We print extra logging in -x mode instead, which traces what
+		// commands are executed.
+		if cfg.BuildX {
+			fmt.Fprintf(os.Stderr, "# get %s\n", Redacted(url))
 		}
 
 		req, err := http.NewRequest("GET", url.String(), nil)
@@ -84,8 +105,8 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
 
 		fetched, res, err = fetch(secure)
 		if err != nil {
-			if cfg.BuildV {
-				log.Printf("https fetch failed: %v", err)
+			if cfg.BuildX {
+				fmt.Fprintf(os.Stderr, "# get %s: %v\n", Redacted(url), err)
 			}
 			if security != Insecure || url.Scheme == "https" {
 				// HTTPS failed, and we can't fall back to plain HTTP.
@@ -99,6 +120,9 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
 		switch url.Scheme {
 		case "http":
 			if security == SecureOnly {
+				if cfg.BuildX {
+					fmt.Fprintf(os.Stderr, "# get %s: insecure\n", Redacted(url))
+				}
 				return nil, fmt.Errorf("insecure URL: %s", Redacted(url))
 			}
 		case "":
@@ -106,6 +130,9 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
 				panic("should have returned after HTTPS failure")
 			}
 		default:
+			if cfg.BuildX {
+				fmt.Fprintf(os.Stderr, "# get %s: unsupported\n", Redacted(url))
+			}
 			return nil, fmt.Errorf("unsupported scheme: %s", Redacted(url))
 		}
 
@@ -113,11 +140,17 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
 		*insecure = *url
 		insecure.Scheme = "http"
 		if insecure.User != nil && security != Insecure {
+			if cfg.BuildX {
+				fmt.Fprintf(os.Stderr, "# get %s: insecure credentials\n", Redacted(url))
+			}
 			return nil, fmt.Errorf("refusing to pass credentials to insecure URL: %s", Redacted(insecure))
 		}
 
 		fetched, res, err = fetch(insecure)
 		if err != nil {
+			if cfg.BuildX {
+				fmt.Fprintf(os.Stderr, "# get %s: %v\n", Redacted(url), err)
+			}
 			// HTTP failed, and we already tried HTTPS if applicable.
 			// Report the error from the HTTP attempt.
 			return nil, err
@@ -126,8 +159,8 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
 
 	// Note: accepting a non-200 OK here, so people can serve a
 	// meta import in their http 404 page.
-	if cfg.BuildV {
-		log.Printf("reading from %s: status code %d", Redacted(fetched), res.StatusCode)
+	if cfg.BuildX {
+		fmt.Fprintf(os.Stderr, "# get %s: %v (%.3fs)\n", Redacted(url), res.Status, time.Since(start).Seconds())
 	}
 	r := &Response{
 		URL:        Redacted(fetched),

src/cmd/go/testdata/script/get_insecure_redirect.txt

@@ -3,7 +3,8 @@
 [!net] skip
 
 env GO111MODULE=on
-env GOPROXY=
+env GOPROXY=direct
+env GOSUMDB=off
 
 ! go get -d vcs-test.golang.org/insecure/go/insecure
 stderr 'redirected .* to insecure URL'

src/cmd/go/testdata/script/mod_get_private_vcs.txt

@@ -3,7 +3,7 @@ env GO111MODULE=on
 # Testing stderr for git ls-remote; turn off proxy.
 [!net] skip
 [!exec:git] skip
-env GOPROXY=
+env GOPROXY=direct
 
 ! go get github.com/golang/nonexist
 stderr 'Confirm the import path was entered correctly.'

src/cmd/go/testdata/script/mod_git_export_subst.txt

 env GO111MODULE=on
-env GOPROXY=
+env GOPROXY=direct
 
 # Testing that git export-subst is disabled
 [!net] skip

src/cmd/go/testdata/script/mod_sumdb_golang.txt

@@ -2,7 +2,7 @@
 env GOPROXY=
 env GOSUMDB=
 go env GOPROXY
-stdout '^https://proxy.golang.org$'
+stdout '^https://proxy.golang.org,direct$'
 go env GOSUMDB
 stdout '^sum.golang.org$'
 env GOPROXY=https://proxy.golang.org
@@ -15,11 +15,29 @@ env GOSUMDB=sum.golang.org
 env GOPROXY=direct
 go get -m rsc.io/quote
 
-# download from proxy.golang.org
+# download from proxy.golang.org with go.sum entry already
 go clean -modcache
-env GOSUMDB='sum.golang.org https://sum.golang.org' # TODO remove URL
-env GOPROXY=https://proxy.golang.org
-go get -m rsc.io/quote
+env GOSUMDB=
+env GOPROXY=
+go get -x -m rsc.io/quote
+! stderr github
+stderr proxy.golang.org/rsc.io/quote
+! stderr sum.golang.org/tile
+! stderr sum.golang.org/lookup/rsc.io/quote
+
+# download again, using checksum database to validate new go.sum lines
+rm go.sum
+go get -x -m rsc.io/quote
+! stderr github
+stderr proxy.golang.org/rsc.io/quote
+stderr sum.golang.org/tile
+stderr sum.golang.org/lookup/rsc.io/quote
+
+# test fallback to direct
+env TESTGOPROXY404=1
+go get -x -m rsc.io/quote
+stderr 'proxy.golang.org.*404 testing'
+stderr github.com/rsc
 
 -- go.mod --
 module m

src/cmd/go/testdata/script/mod_vcs_missing.txt

@@ -2,7 +2,7 @@
 [!net] skip
 
 env GO111MODULE=on
-env GOPROXY=
+env GOPROXY=direct
 
 cd empty
 ! go list launchpad.net/gocheck