Skip to content

G
go
Commit 58de7c6d authored by Bryan C. Mills's avatar Bryan C. Mills Committed by Russ Cox
Browse files
Options

cmd/go/internal/web: merge internal/web2 into web 

The cmd/go/internal/web package was forked in order to support direct
HTTPS fetches from widely-used hosting providers,¹ but direct fetches
were subsequently dropped in CL 107657. The forked web2 package, with
its GitHub-specific diagnostics and .netrc support, remained in use
for module proxy support, but was not used for the initial '?go-get=1'
path resolution, so the .netrc file was only used to fetch from
already-resolved module protocol servers.

This CL moves the .netrc support into its own (new) package,
cmd/go/internal/auth, and consolidates the web and web2 packages back
into just web. As a result, fetches via the web package now support
.netrc, and fetches that previously used web2 now enforce the same
security policies as web (such as prohibiting HTTPS-to-HTTP
redirects).

¹https://github.com/golang/vgo/commit/63138cb6ceed7d6d4e51a8cbd568c64bd3e2b132

Fixes #29591
Fixes #29888
Fixes #30610
Updates #26232

Change-Id: Ia3a13526e443679cf14a72a1f3db96f336ce5e73
Reviewed-on: https://go-review.googlesource.com/c/go/+/170879
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: default avatarRuss Cox <rsc@golang.org>
Reviewed-by: default avatarJay Conrod <jayconrod@google.com>
parent 807761f3
Hide whitespace changes
Inline Side-by-side
Showing with 509 additions and 534 deletions
src/cmd/go/internal/auth/auth.go 0 → 100644
View file @ 58de7c6d
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package auth provides access to user-provided authentication credentials.
package auth
import "net/http"
// AddCredentials fills in the user's credentials for req, if any.
// The return value reports whether any matching credentials were found.
func AddCredentials(req *http.Request) (added bool) {
// TODO(golang.org/issue/26232): Support arbitrary user-provided credentials.
netrcOnce.Do(readNetrc)
for _, l := range netrc {
if l.machine == req.URL.Host {
req.SetBasicAuth(l.login, l.password)
return true
}
}
return false
}
src/cmd/go/internal/web2/web.go → src/cmd/go/internal/auth/netrc.go
View file @ 58de7c6d
// Copyright 2018 The Go Authors. All rights reserved.
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package web2
package auth
import (
"bytes"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"runtime"
"runtime/debug"
"strings"
"sync"
)
var TraceGET = false
var webstack = false
func init() {
flag.BoolVar(&TraceGET, "webtrace", TraceGET, "trace GET requests")
flag.BoolVar(&webstack, "webstack", webstack, "print stack for GET requests")
}
type netrcLine struct {
machine string
login string
......@@ -95,16 +77,6 @@ func parseNetrc(data string) []netrcLine {
return nrc
}
func havePassword(machine string) bool {
netrcOnce.Do(readNetrc)
for _, line := range netrc {
if line.machine == machine {
return true
}
}
return false
}
func netrcPath() (string, error) {
if env := os.Getenv("NETRC"); env != "" {
return env, nil
......@@ -137,209 +109,3 @@ func readNetrc() {
netrc = parseNetrc(string(data))
}
type getState struct {
req *http.Request
resp *http.Response
body io.ReadCloser
non200ok bool
}
type Option interface {
option(*getState) error
}
func Non200OK() Option {
return optionFunc(func(g *getState) error {
g.non200ok = true
return nil
})
}
type optionFunc func(*getState) error
func (f optionFunc) option(g *getState) error {
return f(g)
}
func DecodeJSON(dst interface{}) Option {
return optionFunc(func(g *getState) error {
if g.resp != nil {
return json.NewDecoder(g.body).Decode(dst)
}
return nil
})
}
func ReadAllBody(body *[]byte) Option {
return optionFunc(func(g *getState) error {
if g.resp != nil {
var err error
*body, err = ioutil.ReadAll(g.body)
return err
}
return nil
})
}
func Body(body *io.ReadCloser) Option {
return optionFunc(func(g *getState) error {
if g.resp != nil {
*body = g.body
g.body = nil
}
return nil
})
}
func Header(hdr *http.Header) Option {
return optionFunc(func(g *getState) error {
if g.resp != nil {
*hdr = CopyHeader(g.resp.Header)
}
return nil
})
}
func CopyHeader(hdr http.Header) http.Header {
if hdr == nil {
return nil
}
h2 := make(http.Header)
for k, v := range hdr {
v2 := make([]string, len(v))
copy(v2, v)
h2[k] = v2
}
return h2
}
var cache struct {
mu sync.Mutex
byURL map[string]*cacheEntry
}
type cacheEntry struct {
mu sync.Mutex
resp *http.Response
body []byte
}
var httpDo = http.DefaultClient.Do
func SetHTTPDoForTesting(do func(*http.Request) (*http.Response, error)) {
if do == nil {
do = http.DefaultClient.Do
}
httpDo = do
}
func Get(url string, options ...Option) error {
if TraceGET || webstack || cfg.BuildV {
log.Printf("Fetching %s", url)
if webstack {
log.Println(string(debug.Stack()))
}
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return err
}
netrcOnce.Do(readNetrc)
for _, l := range netrc {
if l.machine == req.URL.Host {
req.SetBasicAuth(l.login, l.password)
break
}
}
g := &getState{req: req}
for _, o := range options {
if err := o.option(g); err != nil {
return err
}
}
cache.mu.Lock()
e := cache.byURL[url]
if e == nil {
e = new(cacheEntry)
if !strings.HasPrefix(url, "file:") {
if cache.byURL == nil {
cache.byURL = make(map[string]*cacheEntry)
}
cache.byURL[url] = e
}
}
cache.mu.Unlock()
e.mu.Lock()
if strings.HasPrefix(url, "file:") {
body, err := ioutil.ReadFile(req.URL.Path)
if err != nil {
e.mu.Unlock()
return err
}
e.body = body
e.resp = &http.Response{
StatusCode: 200,
}
} else if e.resp == nil {
resp, err := httpDo(req)
if err != nil {
e.mu.Unlock()
return err
}
e.resp = resp
// TODO: Spool to temp file.
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
resp.Body = nil
if err != nil {
e.mu.Unlock()
return err
}
e.body = body
}
g.resp = e.resp
g.body = ioutil.NopCloser(bytes.NewReader(e.body))
e.mu.Unlock()
defer func() {
if g.body != nil {
g.body.Close()
}
}()
if g.resp.StatusCode == 403 && req.URL.Host == "api.github.com" && !havePassword("api.github.com") {
base.Errorf("%s", githubMessage)
}
if !g.non200ok && g.resp.StatusCode != 200 {
return fmt.Errorf("unexpected status (%s): %v", url, g.resp.Status)
}
for _, o := range options {
if err := o.option(g); err != nil {
return err
}
}
return err
}
var githubMessage = `go: 403 response from api.github.com
GitHub applies fairly small rate limits to unauthenticated users, and
you appear to be hitting them. To authenticate, please visit
https://github.com/settings/tokens and click "Generate New Token" to
create a Personal Access Token. The token only needs "public_repo"
scope, but you can add "repo" if you want to access private
repositories too.
Add the token to your $HOME/.netrc (%USERPROFILE%\_netrc on Windows):
machine api.github.com login YOU password TOKEN
Sorry for the interruption.
`
src/cmd/go/internal/web2/web_test.go → src/cmd/go/internal/auth/netrc_test.go
View file @ 58de7c6d
......@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package web2
package auth
import (
"reflect"
......@@ -43,7 +43,7 @@ login oops
password too-late-in-file
`
func TestReadNetrc(t *testing.T) {
func TestParseNetrc(t *testing.T) {
lines := parseNetrc(testNetrc)
want := []netrcLine{
{"api.github.com", "user", "pwd"},
......
src/cmd/go/internal/bug/bug.go
View file @ 58de7c6d
......@@ -10,6 +10,7 @@ import (
"fmt"
"io"
"io/ioutil"
urlpkg "net/url"
"os"
"os/exec"
"path/filepath"
......@@ -62,7 +63,7 @@ func runBug(cmd *base.Command, args []string) {
fmt.Fprintln(&buf, "```")
body := buf.String()
url := "https://github.com/golang/go/issues/new?body=" + web.QueryEscape(body)
url := "https://github.com/golang/go/issues/new?body=" + urlpkg.QueryEscape(body)
if !web.OpenBrowser(url) {
fmt.Print("Please file a new issue at golang.org/issue/new using this template:\n\n")
fmt.Print(body)
......@@ -130,7 +131,12 @@ func printCDetails(w io.Writer) {
}
func inspectGoVersion(w io.Writer) {
data, err := web.Get("https://golang.org/VERSION?m=text")
data, err := web.GetBytes(&urlpkg.URL{
Scheme: "https",
Host: "golang.org",
Path: "/VERSION",
RawQuery: "?m=text",
})
if err != nil {
if cfg.BuildV {
fmt.Printf("failed to read from golang.org/VERSION: %v\n", err)
......
src/cmd/go/internal/get/get.go
View file @ 58de7c6d
......@@ -392,7 +392,7 @@ func downloadPackage(p *load.Package) error {
blindRepo bool // set if the repo has unusual configuration
)
security := web.Secure
security := web.SecureOnly
if Insecure {
security = web.Insecure
}
......
src/cmd/go/internal/get/vcs.go
View file @ 58de7c6d
......@@ -11,7 +11,7 @@ import (
"internal/lazyregexp"
"internal/singleflight"
"log"
"net/url"
urlpkg "net/url"
"os"
"os/exec"
"path/filepath"
......@@ -54,7 +54,7 @@ var defaultSecureScheme = map[string]bool{
}
func (v *vcsCmd) isSecure(repo string) bool {
u, err := url.Parse(repo)
u, err := urlpkg.Parse(repo)
if err != nil {
// If repo is not a URL, it's not secure.
return false
......@@ -188,19 +188,19 @@ func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error
}
out := strings.TrimSpace(string(outb))
var repoURL *url.URL
var repoURL *urlpkg.URL
if m := scpSyntaxRe.FindStringSubmatch(out); m != nil {
// Match SCP-like syntax and convert it to a URL.
// Eg, "git@github.com:user/repo" becomes
// "ssh://git@github.com/user/repo".
repoURL = &url.URL{
repoURL = &urlpkg.URL{
Scheme: "ssh",
User: url.User(m[1]),
User: urlpkg.User(m[1]),
Host: m[2],
Path: m[3],
}
} else {
repoURL, err = url.Parse(out)
repoURL, err = urlpkg.Parse(out)
if err != nil {
return "", err
}
......@@ -730,7 +730,7 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode,
match["repo"] = scheme + "://" + match["repo"]
} else {
for _, scheme := range vcs.scheme {
if security == web.Secure && !vcs.isSecureScheme(scheme) {
if security == web.SecureOnly && !vcs.isSecureScheme(scheme) {
continue
}
if vcs.pingCmd != "" && vcs.ping(scheme, match["repo"]) == nil {
......@@ -754,20 +754,35 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode,
return nil, errUnknownSite
}
// repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not
// statically known by repoRootForImportPathStatic.
// urlForImportPath returns a partially-populated URL for the given Go import path.
//
// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld".
func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
// The URL leaves the Scheme field blank so that web.Get will try any scheme
// allowed by the selected security mode.
func urlForImportPath(importPath string) (*urlpkg.URL, error) {
slash := strings.Index(importPath, "/")
if slash < 0 {
slash = len(importPath)
}
host := importPath[:slash]
host, path := importPath[:slash], importPath[slash:]
if !strings.Contains(host, ".") {
return nil, errors.New("import path does not begin with hostname")
}
urlStr, body, err := web.GetMaybeInsecure(importPath, security)
if len(path) == 0 {
path = "/"
}
return &urlpkg.URL{Host: host, Path: path, RawQuery: "go-get=1"}, nil
}
// repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not
// statically known by repoRootForImportPathStatic.
//
// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld".
func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
url, err := urlForImportPath(importPath)
if err != nil {
return nil, err
}
url, resp, err := web.Get(security, url)
if err != nil {
msg := "https fetch: %v"
if security == web.Insecure {
......@@ -775,6 +790,7 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se
}
return nil, fmt.Errorf(msg, err)
}
body := resp.Body
defer body.Close()
imports, err := parseMetaGoImports(body, mod)
if err != nil {
......@@ -784,12 +800,12 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se
mmi, err := matchGoImport(imports, importPath)
if err != nil {
if _, ok := err.(ImportMismatchError); !ok {
return nil, fmt.Errorf("parse %s: %v", urlStr, err)
return nil, fmt.Errorf("parse %s: %v", url, err)
}
return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", urlStr, err)
return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", url, err)
}
if cfg.BuildV {
log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, urlStr)
log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, url)
}
// If the import was "uni.edu/bob/project", which said the
// prefix was "uni.edu" and the RepoRoot was "evilroot.com",
......@@ -801,24 +817,24 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se
if cfg.BuildV {
log.Printf("get %q: verifying non-authoritative meta tag", importPath)
}
urlStr0 := urlStr
url0 := *url
var imports []metaImport
urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security)
url, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security)
if err != nil {
return nil, err
}
metaImport2, err := matchGoImport(imports, importPath)
if err != nil || mmi != metaImport2 {
return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, mmi.Prefix)
return nil, fmt.Errorf("%s and %s disagree about go-import for %s", &url0, url, mmi.Prefix)
}
}
if err := validateRepoRoot(mmi.RepoRoot); err != nil {
return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, mmi.RepoRoot, err)
return nil, fmt.Errorf("%s: invalid repo root %q: %v", url, mmi.RepoRoot, err)
}
vcs := vcsByCmd(mmi.VCS)
if vcs == nil && mmi.VCS != "mod" {
return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS)
return nil, fmt.Errorf("%s: unknown vcs %q", url, mmi.VCS)
}
rr := &RepoRoot{
......@@ -834,7 +850,7 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se
// validateRepoRoot returns an error if repoRoot does not seem to be
// a valid URL with scheme.
func validateRepoRoot(repoRoot string) error {
url, err := url.Parse(repoRoot)
url, err := urlpkg.Parse(repoRoot)
if err != nil {
return err
}
......@@ -856,9 +872,9 @@ var (
//
// The importPath is of the form "golang.org/x/tools".
// It is an error if no imports are found.
// urlStr will still be valid if err != nil.
// The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1"
func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (urlStr string, imports []metaImport, err error) {
// url will still be valid if err != nil.
// The returned url will be of the form "https://golang.org/x/tools?go-get=1"
func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (*urlpkg.URL, []metaImport, error) {
setCache := func(res fetchResult) (fetchResult, error) {
fetchCacheMu.Lock()
defer fetchCacheMu.Unlock()
......@@ -874,25 +890,31 @@ func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.Secu
}
fetchCacheMu.Unlock()
urlStr, body, err := web.GetMaybeInsecure(importPrefix, security)
url, err := urlForImportPath(importPrefix)
if err != nil {
return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)})
return setCache(fetchResult{err: err})
}
url, resp, err := web.Get(security, url)
if err != nil {
return setCache(fetchResult{url: url, err: fmt.Errorf("fetch %s: %v", url, err)})
}
body := resp.Body
defer body.Close()
imports, err := parseMetaGoImports(body, mod)
if err != nil {
return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)})
return setCache(fetchResult{url: url, err: fmt.Errorf("parsing %s: %v", url, err)})
}
if len(imports) == 0 {
err = fmt.Errorf("fetch %s: no go-import meta tag", urlStr)
err = fmt.Errorf("fetch %s: no go-import meta tag", url)
}
return setCache(fetchResult{urlStr: urlStr, imports: imports, err: err})
return setCache(fetchResult{url: url, imports: imports, err: err})
})
res := resi.(fetchResult)
return res.urlStr, res.imports, res.err
return res.url, res.imports, res.err
}
type fetchResult struct {
urlStr string // e.g. "https://foo.com/x/bar?go-get=1"
url *urlpkg.URL
imports []metaImport
err error
}
......@@ -1074,8 +1096,13 @@ func bitbucketVCS(match map[string]string) error {
var resp struct {
SCM string `json:"scm"`
}
url := expand(match, "https://api.bitbucket.org/2.0/repositories/{bitname}?fields=scm")
data, err := web.Get(url)
url := &urlpkg.URL{
Scheme: "https",
Host: "api.bitbucket.org",
Path: expand(match, "/2.0/repositories/{bitname}"),
RawQuery: "fields=scm",
}
data, err := web.GetBytes(url)
if err != nil {
if httpErr, ok := err.(*web.HTTPError); ok && httpErr.StatusCode == 403 {
// this may be a private repository. If so, attempt to determine which
......@@ -1117,7 +1144,12 @@ func launchpadVCS(match map[string]string) error {
if match["project"] == "" || match["series"] == "" {
return nil
}
_, err := web.Get(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format"))
url := &urlpkg.URL{
Scheme: "https",
Host: "code.launchpad.net",
Path: expand(match, "/{project}{series}/.bzr/branch-format"),
}
_, err := web.GetBytes(url)
if err != nil {
match["root"] = expand(match, "launchpad.net/{project}")
match["repo"] = expand(match, "https://{root}")
......
src/cmd/go/internal/get/vcs_test.go
View file @ 58de7c6d
......@@ -181,7 +181,7 @@ func TestRepoRootForImportPath(t *testing.T) {
}
for _, test := range tests {
got, err := RepoRootForImportPath(test.path, IgnoreMod, web.Secure)
got, err := RepoRootForImportPath(test.path, IgnoreMod, web.SecureOnly)
want := test.want
if want == nil {
......
src/cmd/go/internal/modfetch/noweb.go deleted 100644 → 0
View file @ 807761f3
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build cmd_go_bootstrap
package modfetch
import (
"fmt"
"io"
)
func webGetGoGet(url string, body *io.ReadCloser) error {
return fmt.Errorf("no network in go_bootstrap")
}
func webGetBytes(url string, body *[]byte) error {
return fmt.Errorf("no network in go_bootstrap")
}
func webGetBody(url string, body *io.ReadCloser) error {
return fmt.Errorf("no network in go_bootstrap")
}
src/cmd/go/internal/modfetch/proxy.go
View file @ 58de7c6d
......@@ -8,7 +8,11 @@ import (
"encoding/json"
"fmt"
"io"
"net/url"
"io/ioutil"
urlpkg "net/url"
"os"
pathpkg "path"
"path/filepath"
"strings"
"time"
......@@ -17,6 +21,7 @@ import (
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/module"
"cmd/go/internal/semver"
"cmd/go/internal/web"
)
var HelpGoproxy = &base.Command{
......@@ -99,34 +104,85 @@ func lookupProxy(path string) (Repo, error) {
if strings.Contains(proxyURL, ",") {
return nil, fmt.Errorf("invalid $GOPROXY setting: cannot have comma")
}
u, err := url.Parse(proxyURL)
if err != nil || u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "file" {
// Don't echo $GOPROXY back in case it has user:password in it (sigh).
return nil, fmt.Errorf("invalid $GOPROXY setting: malformed URL or invalid scheme (must be http, https, file)")
r, err := newProxyRepo(proxyURL, path)
if err != nil {
return nil, err
}
return newProxyRepo(u.String(), path)
return r, nil
}
type proxyRepo struct {
url string
url *urlpkg.URL
path string
}
func newProxyRepo(baseURL, path string) (Repo, error) {
url, err := urlpkg.Parse(baseURL)
if err != nil {
return nil, err
}
switch url.Scheme {
case "file":
if *url != (urlpkg.URL{Scheme: url.Scheme, Path: url.Path, RawPath: url.RawPath}) {
return nil, fmt.Errorf("proxy URL %q uses file scheme with non-path elements", web.PasswordRedacted(url))
}
case "http", "https":
case "":
return nil, fmt.Errorf("proxy URL %q missing scheme", web.PasswordRedacted(url))
default:
return nil, fmt.Errorf("unsupported proxy scheme %q", url.Scheme)
}
enc, err := module.EncodePath(path)
if err != nil {
return nil, err
}
return &proxyRepo{strings.TrimSuffix(baseURL, "/") + "/" + pathEscape(enc), path}, nil
url.Path = strings.TrimSuffix(url.Path, "/") + "/" + enc
url.RawPath = strings.TrimSuffix(url.RawPath, "/") + "/" + pathEscape(enc)
return &proxyRepo{url, path}, nil
}
func (p *proxyRepo) ModulePath() string {
return p.path
}
func (p *proxyRepo) getBytes(path string) ([]byte, error) {
body, err := p.getBody(path)
if err != nil {
return nil, err
}
defer body.Close()
return ioutil.ReadAll(body)
}
func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) {
fullPath := pathpkg.Join(p.url.Path, path)
if p.url.Scheme == "file" {
rawPath, err := urlpkg.PathUnescape(fullPath)
if err != nil {
return nil, err
}
return os.Open(filepath.FromSlash(rawPath))
}
url := new(urlpkg.URL)
*url = *p.url
url.Path = fullPath
url.RawPath = pathpkg.Join(url.RawPath, pathEscape(path))
_, resp, err := web.Get(web.DefaultSecurity, url)
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("unexpected status (%s): %v", web.PasswordRedacted(url), resp.Status)
}
return resp.Body, nil
}
func (p *proxyRepo) Versions(prefix string) ([]string, error) {
var data []byte
err := webGetBytes(p.url+"/@v/list", &data)
data, err := p.getBytes("@v/list")
if err != nil {
return nil, err
}
......@@ -142,8 +198,7 @@ func (p *proxyRepo) Versions(prefix string) ([]string, error) {
}
func (p *proxyRepo) latest() (*RevInfo, error) {
var data []byte
err := webGetBytes(p.url+"/@v/list", &data)
data, err := p.getBytes("@v/list")
if err != nil {
return nil, err
}
......@@ -172,12 +227,11 @@ func (p *proxyRepo) latest() (*RevInfo, error) {
}
func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
var data []byte
encRev, err := module.EncodeVersion(rev)
if err != nil {
return nil, err
}
err = webGetBytes(p.url+"/@v/"+pathEscape(encRev)+".info", &data)
data, err := p.getBytes("@v/" + encRev + ".info")
if err != nil {
return nil, err
}
......@@ -189,9 +243,7 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
}
func (p *proxyRepo) Latest() (*RevInfo, error) {
var data []byte
u := p.url + "/@latest"
err := webGetBytes(u, &data)
data, err := p.getBytes("@latest")
if err != nil {
// TODO return err if not 404
return p.latest()
......@@ -204,12 +256,11 @@ func (p *proxyRepo) Latest() (*RevInfo, error) {
}
func (p *proxyRepo) GoMod(version string) ([]byte, error) {
var data []byte
encVer, err := module.EncodeVersion(version)
if err != nil {
return nil, err
}
err = webGetBytes(p.url+"/@v/"+pathEscape(encVer)+".mod", &data)
data, err := p.getBytes("@v/" + encVer + ".mod")
if err != nil {
return nil, err
}
......@@ -217,12 +268,11 @@ func (p *proxyRepo) GoMod(version string) ([]byte, error) {
}
func (p *proxyRepo) Zip(dst io.Writer, version string) error {
var body io.ReadCloser
encVer, err := module.EncodeVersion(version)
if err != nil {
return err
}
err = webGetBody(p.url+"/@v/"+pathEscape(encVer)+".zip", &body)
body, err := p.getBody("@v/" + encVer + ".zip")
if err != nil {
return err
}
......@@ -242,5 +292,5 @@ func (p *proxyRepo) Zip(dst io.Writer, version string) error {
// That is, it escapes things like ? and # (which really shouldn't appear anyway).
// It does not escape / to %2F: our REST API is designed so that / can be left as is.
func pathEscape(s string) string {
return strings.ReplaceAll(url.PathEscape(s), "%2F", "/")
return strings.ReplaceAll(urlpkg.PathEscape(s), "%2F", "/")
}
src/cmd/go/internal/modfetch/repo.go
View file @ 58de7c6d
......@@ -209,7 +209,7 @@ func lookup(path string) (r Repo, err error) {
return lookupProxy(path)
}
security := web.Secure
security := web.SecureOnly
if get.Insecure {
security = web.Insecure
}
......@@ -254,7 +254,7 @@ func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) {
// Note: Because we are converting a code reference from a legacy
// version control system, we ignore meta tags about modules
// and use only direct source control entries (get.IgnoreMod).
security := web.Secure
security := web.SecureOnly
if get.Insecure {
security = web.Insecure
}
......
src/cmd/go/internal/modfetch/web.go deleted 100644 → 0
View file @ 807761f3
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !cmd_go_bootstrap
package modfetch
import (
"io"
web "cmd/go/internal/web2"
)
// webGetGoGet fetches a go-get=1 URL and returns the body in *body.
// It allows non-200 responses, as usual for these URLs.
func webGetGoGet(url string, body *io.ReadCloser) error {
return web.Get(url, web.Non200OK(), web.Body(body))
}
// webGetBytes returns the body returned by an HTTP GET, as a []byte.
// It insists on a 200 response.
func webGetBytes(url string, body *[]byte) error {
return web.Get(url, web.ReadAllBody(body))
}
// webGetBody returns the body returned by an HTTP GET, as a io.ReadCloser.
// It insists on a 200 response.
func webGetBody(url string, body *io.ReadCloser) error {
return web.Get(url, web.Body(body))
}
src/cmd/go/internal/web/api.go 0 → 100644
View file @ 58de7c6d
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package web defines minimal helper routines for accessing HTTP/HTTPS
// resources without requiring external dependenicies on the net package.
//
// If the cmd_go_bootstrap build tag is present, web avoids the use of the net
// package and returns errors for all network operations.
package web
import (
"fmt"
"io"
"io/ioutil"
urlpkg "net/url"
)
// SecurityMode specifies whether a function should make network
// calls using insecure transports (eg, plain text HTTP).
// The zero value is "secure".
type SecurityMode int
const (
SecureOnly SecurityMode = iota // Reject plain HTTP; validate HTTPS.
DefaultSecurity // Allow plain HTTP if explicit; validate HTTPS.
Insecure // Allow plain HTTP if not explicitly HTTPS; skip HTTPS validation.
)
type HTTPError struct {
status string
StatusCode int
url *urlpkg.URL
}
func (e *HTTPError) Error() string {
return fmt.Sprintf("%s: %s", e.url, e.status)
}
// GetBytes returns the body of the requested resource, or an error if the
// response status was not http.StatusOk.
//
// GetBytes is a convenience wrapper around Get.
func GetBytes(url *urlpkg.URL) ([]byte, error) {
url, resp, err := Get(DefaultSecurity, url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
err := &HTTPError{status: resp.Status, StatusCode: resp.StatusCode, url: url}
return nil, err
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("%s: %v", url, err)
}
return b, nil
}
type Response struct {
Status string
StatusCode int
Header map[string][]string
Body io.ReadCloser
}
// Get returns the body of the HTTP or HTTPS resource specified at the given URL.
//
// If the URL does not include an explicit scheme, Get first tries "https".
// If the server does not respond under that scheme and the security mode is
// Insecure, Get then tries "http".
// The returned URL indicates which scheme was actually used.
//
// For the "https" scheme only, credentials are attached using the
// cmd/go/internal/auth package. If the URL itself includes a username and
// password, it will not be attempted under the "http" scheme unless the
// security mode is Insecure.
//
// Get returns a non-nil error only if the request did not receive a response
// under any applicable scheme. (A non-2xx response does not cause an error.)
func Get(security SecurityMode, url *urlpkg.URL) (*urlpkg.URL, *Response, error) {
return get(security, url)
}
// PasswordRedacted returns url directly if it does not encode a password,
// or else a copy of url with the password redacted.
func PasswordRedacted(url *urlpkg.URL) *urlpkg.URL {
if url.User != nil {
if _, ok := url.User.Password(); ok {
redacted := *url
redacted.User = urlpkg.UserPassword(url.User.Username(), "[redacted]")
return &redacted
}
}
return url
}
// OpenBrowser attempts to open the requested URL in a web browser.
func OpenBrowser(url string) (opened bool) {
return openBrowser(url)
}
src/cmd/go/internal/web/bootstrap.go
View file @ 58de7c6d
......@@ -6,32 +6,18 @@
// This code is compiled only into the bootstrap 'go' binary.
// These stubs avoid importing packages with large dependency
// trees, like the use of "net/http" in vcs.go.
// trees that potentially require C linking,
// like the use of "net/http" in vcs.go.
package web
import (
"errors"
"io"
urlpkg "net/url"
)
var errHTTP = errors.New("no http in bootstrap go command")
type HTTPError struct {
StatusCode int
}
func (e *HTTPError) Error() string {
panic("unreachable")
}
func Get(url string) ([]byte, error) {
return nil, errHTTP
}
func GetMaybeInsecure(importPath string, security SecurityMode) (string, io.ReadCloser, error) {
return "", nil, errHTTP
func get(security SecurityMode, url *urlpkg.URL) (*urlpkg.URL, *Response, error) {
return nil, nil, errors.New("no http in bootstrap go command")
}
func QueryEscape(s string) string { panic("unreachable") }
func OpenBrowser(url string) bool { panic("unreachable") }
func openBrowser(url string) bool { return false }
src/cmd/go/internal/web/http.go
View file @ 58de7c6d
......@@ -14,13 +14,12 @@ package web
import (
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
urlpkg "net/url"
"time"
"cmd/go/internal/auth"
"cmd/go/internal/cfg"
"cmd/internal/browser"
)
......@@ -50,81 +49,92 @@ var securityPreservingHTTPClient = &http.Client{
},
}
type HTTPError struct {
status string
StatusCode int
url string
}
func (e *HTTPError) Error() string {
return fmt.Sprintf("%s: %s", e.url, e.status)
}
// Get returns the data from an HTTP GET request for the given URL.
func Get(url string) ([]byte, error) {
resp, err := securityPreservingHTTPClient.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
err := &HTTPError{status: resp.Status, StatusCode: resp.StatusCode, url: url}
return nil, err
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("%s: %v", url, err)
}
return b, nil
}
func get(security SecurityMode, url *urlpkg.URL) (*urlpkg.URL, *Response, error) {
fetch := func(url *urlpkg.URL) (*urlpkg.URL, *http.Response, error) {
if cfg.BuildV {
log.Printf("Fetching %s", url)
}
// GetMaybeInsecure returns the body of either the importPath's
// https resource or, if unavailable and permitted by the security mode, the http resource.
func GetMaybeInsecure(importPath string, security SecurityMode) (urlStr string, body io.ReadCloser, err error) {
fetch := func(scheme string) (urlStr string, res *http.Response, err error) {
u, err := url.Parse(scheme + "://" + importPath)
req, err := http.NewRequest("GET", url.String(), nil)
if err != nil {
return "", nil, err
return nil, nil, err
}
u.RawQuery = "go-get=1"
urlStr = u.String()
if cfg.BuildV {
log.Printf("Fetching %s", urlStr)
if url.Scheme == "https" {
auth.AddCredentials(req)
}
if security == Insecure && scheme == "https" { // fail earlier
res, err = impatientInsecureHTTPClient.Get(urlStr)
var res *http.Response
if security == Insecure && url.Scheme == "https" { // fail earlier
res, err = impatientInsecureHTTPClient.Do(req)
} else {
res, err = securityPreservingHTTPClient.Get(urlStr)
res, err = securityPreservingHTTPClient.Do(req)
}
return
return url, res, err
}
closeBody := func(res *http.Response) {
if res != nil {
res.Body.Close()
var (
fetched *urlpkg.URL
res *http.Response
err error
)
if url.Scheme == "" || url.Scheme == "https" {
secure := new(urlpkg.URL)
*secure = *url
secure.Scheme = "https"
fetched, res, err = fetch(secure)
if err != nil {
if cfg.BuildV {
log.Printf("https fetch failed: %v", err)
}
if security != Insecure || url.Scheme == "https" {
// HTTPS failed, and we can't fall back to plain HTTP.
// Report the error from the HTTPS attempt.
return nil, nil, err
}
}
}
urlStr, res, err := fetch("https")
if err != nil {
if cfg.BuildV {
log.Printf("https fetch failed: %v", err)
if res == nil {
switch url.Scheme {
case "http":
if security == SecureOnly {
return nil, nil, fmt.Errorf("URL %q is not secure", PasswordRedacted(url))
}
case "":
if security != Insecure {
panic("should have returned after HTTPS failure")
}
default:
return nil, nil, fmt.Errorf("unsupported scheme %s", url.Scheme)
}
if security == Insecure {
closeBody(res)
urlStr, res, err = fetch("http")
insecure := new(urlpkg.URL)
*insecure = *url
insecure.Scheme = "http"
if insecure.User != nil && security != Insecure {
return nil, nil, fmt.Errorf("refusing to pass credentials to insecure URL %q", PasswordRedacted(insecure))
}
fetched, res, err = fetch(insecure)
if err != nil {
// HTTP failed, and we already tried HTTPS if applicable.
// Report the error from the HTTP attempt.
return nil, nil, err
}
}
if err != nil {
closeBody(res)
return "", nil, err
}
// Note: accepting a non-200 OK here, so people can serve a
// meta import in their http 404 page.
if cfg.BuildV {
log.Printf("Parsing meta tags from %s (status code %d)", urlStr, res.StatusCode)
log.Printf("Parsing meta tags from %s (status code %d)", PasswordRedacted(fetched), res.StatusCode)
}
return urlStr, res.Body, nil
return fetched, &Response{
Status: res.Status,
StatusCode: res.StatusCode,
Header: map[string][]string(res.Header),
Body: res.Body,
}, nil
}
func QueryEscape(s string) string { return url.QueryEscape(s) }
func OpenBrowser(url string) bool { return browser.Open(url) }
func openBrowser(url string) bool { return browser.Open(url) }
src/cmd/go/internal/web/security.go deleted 100644 → 0
View file @ 807761f3
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package web defines helper routines for accessing HTTP/HTTPS resources.
package web
// SecurityMode specifies whether a function should make network
// calls using insecure transports (eg, plain text HTTP).
// The zero value is "secure".
type SecurityMode int
const (
Secure SecurityMode = iota
Insecure
)
src/cmd/go/internal/webtest/test.go deleted 100644 → 0
View file @ 807761f3
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package webtest
import (
"bufio"
"bytes"
"encoding/hex"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"sort"
"strconv"
"strings"
"sync"
"unicode/utf8"
web "cmd/go/internal/web2"
)
var mode = flag.String("webtest", "replay", "set webtest `mode` - record, replay, bypass")
func Hook() {
if *mode == "bypass" {
return
}
web.SetHTTPDoForTesting(Do)
}
func Unhook() {
web.SetHTTPDoForTesting(nil)
}
func Print() {
web.SetHTTPDoForTesting(DoPrint)
}
var responses struct {
mu sync.Mutex
byURL map[string]*respEntry
}
type respEntry struct {
status string
code int
hdr http.Header
body []byte
}
func Serve(url string, status string, hdr http.Header, body []byte) {
if status == "" {
status = "200 OK"
}
code, err := strconv.Atoi(strings.Fields(status)[0])
if err != nil {
panic("bad Serve status - " + status + " - " + err.Error())
}
responses.mu.Lock()
defer responses.mu.Unlock()
if responses.byURL == nil {
responses.byURL = make(map[string]*respEntry)
}
responses.byURL[url] = &respEntry{status: status, code: code, hdr: web.CopyHeader(hdr), body: body}
}
func Do(req *http.Request) (*http.Response, error) {
if req.Method != "GET" {
return nil, fmt.Errorf("bad method - must be GET")
}
responses.mu.Lock()
e := responses.byURL[req.URL.String()]
responses.mu.Unlock()
if e == nil {
if *mode == "record" {
loaded.mu.Lock()
if len(loaded.did) != 1 {
loaded.mu.Unlock()
return nil, fmt.Errorf("cannot use -webtest=record with multiple loaded response files")
}
var file string
for file = range loaded.did {
break
}
loaded.mu.Unlock()
return doSave(file, req)
}
e = &respEntry{code: 599, status: "599 unexpected request (no canned response)"}
}
resp := &http.Response{
Status: e.status,
StatusCode: e.code,
Header: web.CopyHeader(e.hdr),
Body: ioutil.NopCloser(bytes.NewReader(e.body)),
}
return resp, nil
}
func DoPrint(req *http.Request) (*http.Response, error) {
return doSave("", req)
}
func doSave(file string, req *http.Request) (*http.Response, error) {
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
data, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return nil, err
}
resp.Body = ioutil.NopCloser(bytes.NewReader(data))
var f *os.File
if file == "" {
f = os.Stderr
} else {
f, err = os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Fatal(err)
}
defer f.Close()
}
fmt.Fprintf(f, "GET %s\n", req.URL.String())
fmt.Fprintf(f, "%s\n", resp.Status)
var keys []string
for k := range resp.Header {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if k == "Set-Cookie" {
continue
}
for _, v := range resp.Header[k] {
fmt.Fprintf(f, "%s: %s\n", k, v)
}
}
fmt.Fprintf(f, "\n")
if utf8.Valid(data) && !bytes.Contains(data, []byte("\nGET")) && !isHexDump(data) {
fmt.Fprintf(f, "%s\n\n", data)
} else {
fmt.Fprintf(f, "%s\n", hex.Dump(data))
}
return resp, err
}
var loaded struct {
mu sync.Mutex
did map[string]bool
}
func LoadOnce(file string) {
loaded.mu.Lock()
if loaded.did[file] {
loaded.mu.Unlock()
return
}
if loaded.did == nil {
loaded.did = make(map[string]bool)
}
loaded.did[file] = true
loaded.mu.Unlock()
f, err := os.Open(file)
if err != nil {
log.Fatal(err)
}
defer f.Close()
b := bufio.NewReader(f)
var ungetLine string
nextLine := func() string {
if ungetLine != "" {
l := ungetLine
ungetLine = ""
return l
}
line, err := b.ReadString('\n')
if err != nil {
if err == io.EOF {
return ""
}
log.Fatalf("%s: unexpected read error: %v", file, err)
}
return line
}
for {
line := nextLine()
if line == "" { // EOF
break
}
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "#") || line == "" {
continue
}
if !strings.HasPrefix(line, "GET ") {
log.Fatalf("%s: malformed GET line: %s", file, line)
}
url := line[len("GET "):]
status := nextLine()
if _, err := strconv.Atoi(strings.Fields(status)[0]); err != nil {
log.Fatalf("%s: malformed status line (after GET %s): %s", file, url, status)
}
hdr := make(http.Header)
for {
kv := strings.TrimSpace(nextLine())
if kv == "" {
break
}
i := strings.Index(kv, ":")
if i < 0 {
log.Fatalf("%s: malformed header line (after GET %s): %s", file, url, kv)
}
k, v := kv[:i], strings.TrimSpace(kv[i+1:])
hdr[k] = append(hdr[k], v)
}
var body []byte
Body:
for n := 0; ; n++ {
line := nextLine()
if n == 0 && isHexDump([]byte(line)) {
ungetLine = line
b, err := parseHexDump(nextLine)
if err != nil {
log.Fatalf("%s: malformed hex dump (after GET %s): %v", file, url, err)
}
body = b
break
}
if line == "" { // EOF
for i := 0; i < 2; i++ {
if len(body) > 0 && body[len(body)-1] == '\n' {
body = body[:len(body)-1]
}
}
break
}
body = append(body, line...)
for line == "\n" {
line = nextLine()
if strings.HasPrefix(line, "GET ") {
ungetLine = line
body = body[:len(body)-1]
if len(body) > 0 {
body = body[:len(body)-1]
}
break Body
}
body = append(body, line...)
}
}
Serve(url, status, hdr, body)
}
}
func isHexDump(data []byte) bool {
return bytes.HasPrefix(data, []byte("00000000 ")) || bytes.HasPrefix(data, []byte("0000000 "))
}
// parseHexDump parses the hex dump in text, which should be the
// output of "hexdump -C" or Plan 9's "xd -b" or Go's hex.Dump
// and returns the original data used to produce the dump.
// It is meant to enable storing golden binary files as text, so that
// changes to the golden files can be seen during code reviews.
func parseHexDump(nextLine func() string) ([]byte, error) {
var out []byte
for {
line := nextLine()
if line == "" || line == "\n" {
break
}
if i := strings.Index(line, "|"); i >= 0 { // remove text dump
line = line[:i]
}
f := strings.Fields(line)
if len(f) > 1+16 {
return nil, fmt.Errorf("parsing hex dump: too many fields on line %q", line)
}
if len(f) == 0 || len(f) == 1 && f[0] == "*" { // all zeros block omitted
continue
}
addr64, err := strconv.ParseUint(f[0], 16, 0)
if err != nil {
return nil, fmt.Errorf("parsing hex dump: invalid address %q", f[0])
}
addr := int(addr64)
if len(out) < addr {
out = append(out, make([]byte, addr-len(out))...)
}
for _, x := range f[1:] {
val, err := strconv.ParseUint(x, 16, 8)
if err != nil {
return nil, fmt.Errorf("parsing hexdump: invalid hex byte %q", x)
}
out = append(out, byte(val))
}
}
return out, nil
}
src/cmd/go/testdata/script/get_404_meta.txt 0 → 100644
View file @ 58de7c6d
# golang.org/issue/13037: 'go get' was not parsing <meta> tags in 404 served over HTTPS.
[!net] skip
env GO111MODULE=off
go get -d -insecure bazil.org/fuse/fs/fstestutil
env GO111MODULE=on
env GOPROXY=direct
go get -d -insecure bazil.org/fuse/fs/fstestutil
src/cmd/go/testdata/script/get_insecure_redirect.txt
View file @ 58de7c6d
# golang.org/issue/13037: 'go get' was not parsing <meta> tags in 404 served over HTTPS.
# golang.org/issue/29591: 'go get' was following plain-HTTP redirects even without -insecure.
[!net] skip
......
src/cmd/go/testdata/script/mod_auth.txt 0 → 100644
View file @ 58de7c6d
[!net] skip
env GO111MODULE=on
env GOPROXY=direct
# Without credentials, downloading a module from a path that requires HTTPS
# basic auth should fail.
env NETRC=$WORK/empty
! go list all
# With credentials from a netrc file, it should succeed.
env NETRC=$WORK/netrc
go mod tidy
go list all
stdout vcs-test.golang.org/auth/or401
stdout vcs-test.golang.org/auth/or404
-- go.mod --
module private.example.com
-- main.go --
package useprivate
import (
_ "vcs-test.golang.org/auth/or401"
_ "vcs-test.golang.org/auth/or404"
)
-- $WORK/empty --
-- $WORK/netrc --
machine vcs-test.golang.org
login aladdin
password opensesame
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7