Commit f7248f05 authored by Russ Cox's avatar Russ Cox

cmd/go: merge module support from x/vgo repo

This CL corresponds to CL 123361, the final manual CL in that repo,
making this the final manual sync.

All future commits will happen in this repo (the main Go repo),
and we'll update x/vgo automatically with a fixed patch+script.

Change-Id: I572243309c1809727604fd704705a23c30e85d1a
Reviewed-on: https://go-review.googlesource.com/123576
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: default avatarBryan C. Mills <bcmills@google.com>
parent f22dd66b
...@@ -8,6 +8,7 @@ import ( ...@@ -8,6 +8,7 @@ import (
"bytes" "bytes"
"debug/elf" "debug/elf"
"debug/macho" "debug/macho"
"flag"
"fmt" "fmt"
"go/format" "go/format"
"internal/race" "internal/race"
...@@ -112,6 +113,12 @@ func TestMain(m *testing.M) { ...@@ -112,6 +113,12 @@ func TestMain(m *testing.M) {
} }
os.Unsetenv("GOROOT_FINAL") os.Unsetenv("GOROOT_FINAL")
flag.Parse()
if *proxyAddr != "" {
StartProxy()
select {}
}
if canRun { if canRun {
args := []string{"build", "-tags", "testgo", "-o", "testgo" + exeSuffix} args := []string{"build", "-tags", "testgo", "-o", "testgo" + exeSuffix}
if race.Enabled { if race.Enabled {
...@@ -417,7 +424,8 @@ func (tg *testgoData) doRun(args []string) error { ...@@ -417,7 +424,8 @@ func (tg *testgoData) doRun(args []string) error {
func (tg *testgoData) run(args ...string) { func (tg *testgoData) run(args ...string) {
tg.t.Helper() tg.t.Helper()
if status := tg.doRun(args); status != nil { if status := tg.doRun(args); status != nil {
tg.t.Logf("go %v failed unexpectedly: %v", args, status) wd, _ := os.Getwd()
tg.t.Logf("go %v failed unexpectedly in %s: %v", args, wd, status)
tg.t.FailNow() tg.t.FailNow()
} }
} }
...@@ -760,24 +768,51 @@ func (tg *testgoData) wantNotStale(pkg, reason, msg string) { ...@@ -760,24 +768,51 @@ func (tg *testgoData) wantNotStale(pkg, reason, msg string) {
} }
} }
// If -testwork is specified, the test prints the name of the temp directory
// and does not remove it when done, so that a programmer can
// poke at the test file tree afterward.
var testWork = flag.Bool("testwork", false, "")
// cleanup cleans up a test that runs testgo. // cleanup cleans up a test that runs testgo.
func (tg *testgoData) cleanup() { func (tg *testgoData) cleanup() {
tg.t.Helper() tg.t.Helper()
if tg.wd != "" { if tg.wd != "" {
wd, _ := os.Getwd()
tg.t.Logf("ended in %s", wd)
if err := os.Chdir(tg.wd); err != nil { if err := os.Chdir(tg.wd); err != nil {
// We are unlikely to be able to continue. // We are unlikely to be able to continue.
fmt.Fprintln(os.Stderr, "could not restore working directory, crashing:", err) fmt.Fprintln(os.Stderr, "could not restore working directory, crashing:", err)
os.Exit(2) os.Exit(2)
} }
} }
if *testWork {
tg.t.Logf("TESTWORK=%s\n", tg.path("."))
return
}
for _, path := range tg.temps { for _, path := range tg.temps {
tg.check(os.RemoveAll(path)) tg.check(removeAll(path))
} }
if tg.tempdir != "" { if tg.tempdir != "" {
tg.check(os.RemoveAll(tg.tempdir)) tg.check(removeAll(tg.tempdir))
} }
} }
func removeAll(dir string) error {
// module cache has 0444 directories;
// make them writable in order to remove content.
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil // ignore errors walking in file system
}
if info.IsDir() {
os.Chmod(path, 0777)
}
return nil
})
return os.RemoveAll(dir)
}
// failSSH puts an ssh executable in the PATH that always fails. // failSSH puts an ssh executable in the PATH that always fails.
// This is to stub out uses of ssh by go get. // This is to stub out uses of ssh by go get.
func (tg *testgoData) failSSH() { func (tg *testgoData) failSSH() {
...@@ -1745,8 +1780,8 @@ func TestGoGetTestOnlyPkg(t *testing.T) { ...@@ -1745,8 +1780,8 @@ func TestGoGetTestOnlyPkg(t *testing.T) {
defer tg.cleanup() defer tg.cleanup()
tg.tempDir("gopath") tg.tempDir("gopath")
tg.setenv("GOPATH", tg.path("gopath")) tg.setenv("GOPATH", tg.path("gopath"))
tg.run("get", "golang.org/x/tour/content") tg.run("get", "golang.org/x/tour/content...")
tg.run("get", "-t", "golang.org/x/tour/content") tg.run("get", "-t", "golang.org/x/tour/content...")
} }
func TestInstalls(t *testing.T) { func TestInstalls(t *testing.T) {
...@@ -3165,11 +3200,25 @@ func TestBuildDashIInstallsDependencies(t *testing.T) { ...@@ -3165,11 +3200,25 @@ func TestBuildDashIInstallsDependencies(t *testing.T) {
checkbar("cmd") checkbar("cmd")
} }
func TestGoBuildInTestOnlyDirectoryFailsWithAGoodError(t *testing.T) { func TestGoBuildTestOnly(t *testing.T) {
tg := testgo(t) tg := testgo(t)
defer tg.cleanup() defer tg.cleanup()
tg.runFail("build", "./testdata/testonly") tg.makeTempdir()
tg.grepStderr("no non-test Go files in", "go build ./testdata/testonly produced unexpected error") tg.setenv("GOPATH", tg.path("."))
tg.tempFile("src/testonly/t_test.go", `package testonly`)
tg.tempFile("src/testonly2/t.go", `package testonly2`)
tg.cd(tg.path("src"))
// Named explicitly, test-only packages should be reported as unbuildable/uninstallable,
// even if there is a wildcard also matching.
tg.runFail("build", "testonly", "testonly...")
tg.grepStderr("no non-test Go files in", "go build ./xtestonly produced unexpected error")
tg.runFail("install", "./testonly")
tg.grepStderr("no non-test Go files in", "go install ./testonly produced unexpected error")
// Named through a wildcards, the test-only packages should be silently ignored.
tg.run("build", "testonly...")
tg.run("install", "./testonly...")
} }
func TestGoTestDetectsTestOnlyImportCycles(t *testing.T) { func TestGoTestDetectsTestOnlyImportCycles(t *testing.T) {
...@@ -6115,12 +6164,12 @@ func TestBadCommandLines(t *testing.T) { ...@@ -6115,12 +6164,12 @@ func TestBadCommandLines(t *testing.T) {
tg.tempFile("src/@x/x.go", "package x\n") tg.tempFile("src/@x/x.go", "package x\n")
tg.setenv("GOPATH", tg.path(".")) tg.setenv("GOPATH", tg.path("."))
tg.runFail("build", "@x") tg.runFail("build", "@x")
tg.grepStderr("invalid input directory name \"@x\"", "did not reject @x directory") tg.grepStderr("invalid input directory name \"@x\"|cannot use path@version syntax", "did not reject @x directory")
tg.tempFile("src/@x/y/y.go", "package y\n") tg.tempFile("src/@x/y/y.go", "package y\n")
tg.setenv("GOPATH", tg.path(".")) tg.setenv("GOPATH", tg.path("."))
tg.runFail("build", "@x/y") tg.runFail("build", "@x/y")
tg.grepStderr("invalid import path \"@x/y\"", "did not reject @x/y import path") tg.grepStderr("invalid import path \"@x/y\"|cannot use path@version syntax", "did not reject @x/y import path")
tg.tempFile("src/-x/x.go", "package x\n") tg.tempFile("src/-x/x.go", "package x\n")
tg.setenv("GOPATH", tg.path(".")) tg.setenv("GOPATH", tg.path("."))
......
...@@ -21,6 +21,7 @@ var ( ...@@ -21,6 +21,7 @@ var (
BuildA bool // -a flag BuildA bool // -a flag
BuildBuildmode string // -buildmode flag BuildBuildmode string // -buildmode flag
BuildContext = build.Default BuildContext = build.Default
BuildGetmode string // -getmode flag
BuildI bool // -i flag BuildI bool // -i flag
BuildLinkshared bool // -linkshared flag BuildLinkshared bool // -linkshared flag
BuildMSan bool // -msan flag BuildMSan bool // -msan flag
...@@ -67,6 +68,11 @@ var ( ...@@ -67,6 +68,11 @@ var (
Goos = BuildContext.GOOS Goos = BuildContext.GOOS
ExeSuffix string ExeSuffix string
Gopath = filepath.SplitList(BuildContext.GOPATH) Gopath = filepath.SplitList(BuildContext.GOPATH)
// ModulesEnabled specifies whether the go command is running
// in module-aware mode (as opposed to GOPATH mode).
// It is equal to modload.Enabled, but not all packages can import modload.
ModulesEnabled bool
) )
func init() { func init() {
......
...@@ -9,6 +9,7 @@ import ( ...@@ -9,6 +9,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
"path/filepath"
"runtime" "runtime"
"strings" "strings"
...@@ -16,7 +17,7 @@ import ( ...@@ -16,7 +17,7 @@ import (
"cmd/go/internal/cache" "cmd/go/internal/cache"
"cmd/go/internal/cfg" "cmd/go/internal/cfg"
"cmd/go/internal/load" "cmd/go/internal/load"
"cmd/go/internal/vgo" "cmd/go/internal/modload"
"cmd/go/internal/work" "cmd/go/internal/work"
) )
...@@ -112,6 +113,18 @@ func findEnv(env []cfg.EnvVar, name string) string { ...@@ -112,6 +113,18 @@ func findEnv(env []cfg.EnvVar, name string) string {
// ExtraEnvVars returns environment variables that should not leak into child processes. // ExtraEnvVars returns environment variables that should not leak into child processes.
func ExtraEnvVars() []cfg.EnvVar { func ExtraEnvVars() []cfg.EnvVar {
gomod := ""
if modload.Init(); modload.ModRoot != "" {
gomod = filepath.Join(modload.ModRoot, "go.mod")
}
return []cfg.EnvVar{
{Name: "GOMOD", Value: gomod},
}
}
// ExtraEnvVarsCostly returns environment variables that should not leak into child processes
// but are costly to evaluate.
func ExtraEnvVarsCostly() []cfg.EnvVar {
var b work.Builder var b work.Builder
b.Init() b.Init()
cppflags, cflags, cxxflags, fflags, ldflags, err := b.CFlags(&load.Package{}) cppflags, cflags, cxxflags, fflags, ldflags, err := b.CFlags(&load.Package{})
...@@ -121,6 +134,7 @@ func ExtraEnvVars() []cfg.EnvVar { ...@@ -121,6 +134,7 @@ func ExtraEnvVars() []cfg.EnvVar {
return nil return nil
} }
cmd := b.GccCmd(".", "") cmd := b.GccCmd(".", "")
return []cfg.EnvVar{ return []cfg.EnvVar{
// Note: Update the switch in runEnv below when adding to this list. // Note: Update the switch in runEnv below when adding to this list.
{Name: "CGO_CFLAGS", Value: strings.Join(cflags, " ")}, {Name: "CGO_CFLAGS", Value: strings.Join(cflags, " ")},
...@@ -130,19 +144,19 @@ func ExtraEnvVars() []cfg.EnvVar { ...@@ -130,19 +144,19 @@ func ExtraEnvVars() []cfg.EnvVar {
{Name: "CGO_LDFLAGS", Value: strings.Join(ldflags, " ")}, {Name: "CGO_LDFLAGS", Value: strings.Join(ldflags, " ")},
{Name: "PKG_CONFIG", Value: b.PkgconfigCmd()}, {Name: "PKG_CONFIG", Value: b.PkgconfigCmd()},
{Name: "GOGCCFLAGS", Value: strings.Join(cmd[3:], " ")}, {Name: "GOGCCFLAGS", Value: strings.Join(cmd[3:], " ")},
{Name: "VGOMODROOT", Value: vgo.ModRoot},
} }
} }
func runEnv(cmd *base.Command, args []string) { func runEnv(cmd *base.Command, args []string) {
env := cfg.CmdEnv env := cfg.CmdEnv
env = append(env, ExtraEnvVars()...)
// Do we need to call ExtraEnvVars, which is a bit expensive? // Do we need to call ExtraEnvVarsCostly, which is a bit expensive?
// Only if we're listing all environment variables ("go env") // Only if we're listing all environment variables ("go env")
// or the variables being requested are in the extra list. // or the variables being requested are in the extra list.
needExtra := true needCostly := true
if len(args) > 0 { if len(args) > 0 {
needExtra = false needCostly = false
for _, arg := range args { for _, arg := range args {
switch arg { switch arg {
case "CGO_CFLAGS", case "CGO_CFLAGS",
...@@ -152,12 +166,12 @@ func runEnv(cmd *base.Command, args []string) { ...@@ -152,12 +166,12 @@ func runEnv(cmd *base.Command, args []string) {
"CGO_LDFLAGS", "CGO_LDFLAGS",
"PKG_CONFIG", "PKG_CONFIG",
"GOGCCFLAGS": "GOGCCFLAGS":
needExtra = true needCostly = true
} }
} }
} }
if needExtra { if needCostly {
env = append(env, ExtraEnvVars()...) env = append(env, ExtraEnvVarsCostly()...)
} }
if len(args) > 0 { if len(args) > 0 {
......
...@@ -9,8 +9,8 @@ import ( ...@@ -9,8 +9,8 @@ import (
"cmd/go/internal/base" "cmd/go/internal/base"
"cmd/go/internal/cfg" "cmd/go/internal/cfg"
"cmd/go/internal/load" "cmd/go/internal/load"
"cmd/go/internal/modload"
"cmd/go/internal/str" "cmd/go/internal/str"
"cmd/go/internal/vgo"
"fmt" "fmt"
"os" "os"
) )
...@@ -34,9 +34,9 @@ See also: go fmt, go vet. ...@@ -34,9 +34,9 @@ See also: go fmt, go vet.
func runFix(cmd *base.Command, args []string) { func runFix(cmd *base.Command, args []string) {
printed := false printed := false
for _, pkg := range load.Packages(args) { for _, pkg := range load.Packages(args) {
if vgo.Enabled() && !pkg.Module.Top { if modload.Enabled() && !pkg.Module.Main {
if !printed { if !printed {
fmt.Fprintf(os.Stderr, "vgo: not fixing packages in dependency modules\n") fmt.Fprintf(os.Stderr, "go: not fixing packages in dependency modules\n")
printed = true printed = true
} }
continue continue
......
...@@ -16,8 +16,8 @@ import ( ...@@ -16,8 +16,8 @@ import (
"cmd/go/internal/base" "cmd/go/internal/base"
"cmd/go/internal/cfg" "cmd/go/internal/cfg"
"cmd/go/internal/load" "cmd/go/internal/load"
"cmd/go/internal/modload"
"cmd/go/internal/str" "cmd/go/internal/str"
"cmd/go/internal/vgo"
) )
func init() { func init() {
...@@ -60,9 +60,9 @@ func runFmt(cmd *base.Command, args []string) { ...@@ -60,9 +60,9 @@ func runFmt(cmd *base.Command, args []string) {
}() }()
} }
for _, pkg := range load.PackagesAndErrors(args) { for _, pkg := range load.PackagesAndErrors(args) {
if vgo.Enabled() && !pkg.Module.Top { if modload.Enabled() && !pkg.Module.Main {
if !printed { if !printed {
fmt.Fprintf(os.Stderr, "vgo: not formatting packages in dependency modules\n") fmt.Fprintf(os.Stderr, "go: not formatting packages in dependency modules\n")
printed = true printed = true
} }
continue continue
......
...@@ -21,7 +21,7 @@ import ( ...@@ -21,7 +21,7 @@ import (
"cmd/go/internal/base" "cmd/go/internal/base"
"cmd/go/internal/cfg" "cmd/go/internal/cfg"
"cmd/go/internal/load" "cmd/go/internal/load"
"cmd/go/internal/vgo" "cmd/go/internal/modload"
"cmd/go/internal/work" "cmd/go/internal/work"
) )
...@@ -161,9 +161,9 @@ func runGenerate(cmd *base.Command, args []string) { ...@@ -161,9 +161,9 @@ func runGenerate(cmd *base.Command, args []string) {
// Even if the arguments are .go files, this loop suffices. // Even if the arguments are .go files, this loop suffices.
printed := false printed := false
for _, pkg := range load.Packages(args) { for _, pkg := range load.Packages(args) {
if vgo.Enabled() && !pkg.Module.Top { if modload.Enabled() && !pkg.Module.Main {
if !printed { if !printed {
fmt.Fprintf(os.Stderr, "vgo: not generating in packages in dependency modules\n") fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
printed = true printed = true
} }
continue continue
......
...@@ -28,7 +28,7 @@ func charsetReader(charset string, input io.Reader) (io.Reader, error) { ...@@ -28,7 +28,7 @@ func charsetReader(charset string, input io.Reader) (io.Reader, error) {
// parseMetaGoImports returns meta imports from the HTML in r. // parseMetaGoImports returns meta imports from the HTML in r.
// Parsing ends at the end of the <head> section or the beginning of the <body>. // Parsing ends at the end of the <head> section or the beginning of the <body>.
func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) { func parseMetaGoImports(r io.Reader, mod ModuleMode) (imports []metaImport, err error) {
d := xml.NewDecoder(r) d := xml.NewDecoder(r)
d.CharsetReader = charsetReader d.CharsetReader = charsetReader
d.Strict = false d.Strict = false
...@@ -39,13 +39,13 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) { ...@@ -39,13 +39,13 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
if err == io.EOF || len(imports) > 0 { if err == io.EOF || len(imports) > 0 {
err = nil err = nil
} }
return break
} }
if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
return break
} }
if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
return break
} }
e, ok := t.(xml.StartElement) e, ok := t.(xml.StartElement)
if !ok || !strings.EqualFold(e.Name.Local, "meta") { if !ok || !strings.EqualFold(e.Name.Local, "meta") {
...@@ -55,13 +55,6 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) { ...@@ -55,13 +55,6 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
continue continue
} }
if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 { if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
// Ignore VCS type "mod", which is new Go modules.
// This code is for old go get and must ignore the new mod lines.
// Otherwise matchGoImport will complain about two
// different metaImport lines for the same Prefix.
if f[1] == "mod" {
continue
}
imports = append(imports, metaImport{ imports = append(imports, metaImport{
Prefix: f[0], Prefix: f[0],
VCS: f[1], VCS: f[1],
...@@ -69,6 +62,27 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) { ...@@ -69,6 +62,27 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
}) })
} }
} }
// Extract mod entries if we are paying attention to them.
var list []metaImport
var have map[string]bool
if mod == PreferMod {
have = make(map[string]bool)
for _, m := range imports {
if m.VCS == "mod" {
have[m.Prefix] = true
list = append(list, m)
}
}
}
// Append non-mod entries, ignoring those superseded by a mod entry.
for _, m := range imports {
if m.VCS != "mod" && !have[m.Prefix] {
list = append(list, m)
}
}
return list, nil
} }
// attrValue returns the attribute value for the case-insensitive key // attrValue returns the attribute value for the case-insensitive key
......
...@@ -18,13 +18,12 @@ import ( ...@@ -18,13 +18,12 @@ import (
"cmd/go/internal/load" "cmd/go/internal/load"
"cmd/go/internal/search" "cmd/go/internal/search"
"cmd/go/internal/str" "cmd/go/internal/str"
"cmd/go/internal/vgo"
"cmd/go/internal/web" "cmd/go/internal/web"
"cmd/go/internal/work" "cmd/go/internal/work"
) )
var CmdGet = &base.Command{ var CmdGet = &base.Command{
UsageLine: "get [-d] [-f] [-fix] [-insecure] [-t] [-u] [-v] [build flags] [packages]", UsageLine: "get [-d] [-f] [-t] [-u] [-v] [-fix] [-insecure] [build flags] [packages]",
Short: "download and install packages and dependencies", Short: "download and install packages and dependencies",
Long: ` Long: `
Get downloads the packages named by the import paths, along with their Get downloads the packages named by the import paths, along with their
...@@ -75,25 +74,49 @@ For more about specifying packages, see 'go help packages'. ...@@ -75,25 +74,49 @@ For more about specifying packages, see 'go help packages'.
For more about how 'go get' finds source code to For more about how 'go get' finds source code to
download, see 'go help importpath'. download, see 'go help importpath'.
This text describes the behavior of get when using GOPATH
to manage source code and dependencies.
If instead the go command is running in module-aware mode,
the details of get's flags and effects change, as does 'go help get'.
See 'go help modules' and 'go help module-get'.
See also: go build, go install, go clean. See also: go build, go install, go clean.
`, `,
} }
var getD = CmdGet.Flag.Bool("d", false, "") var HelpGopathGet = &base.Command{
var getF = CmdGet.Flag.Bool("f", false, "") UsageLine: "gopath-get",
var getT = CmdGet.Flag.Bool("t", false, "") Short: "legacy GOPATH go get",
var getU = CmdGet.Flag.Bool("u", false, "") Long: `
var getFix = CmdGet.Flag.Bool("fix", false, "") The 'go get' command changes behavior depending on whether the
var getInsecure = CmdGet.Flag.Bool("insecure", false, "") go command is running in module-aware mode or legacy GOPATH mode.
This help text, accessible as 'go help gopath-get' even in module-aware mode,
describes 'go get' as it operates in legacy GOPATH mode.
Usage: ` + CmdGet.UsageLine + `
` + CmdGet.Long,
}
var (
getD = CmdGet.Flag.Bool("d", false, "")
getF = CmdGet.Flag.Bool("f", false, "")
getT = CmdGet.Flag.Bool("t", false, "")
getU = CmdGet.Flag.Bool("u", false, "")
getFix = CmdGet.Flag.Bool("fix", false, "")
Insecure bool
)
func init() { func init() {
work.AddBuildFlags(CmdGet) work.AddBuildFlags(CmdGet)
CmdGet.Run = runGet // break init loop CmdGet.Run = runGet // break init loop
CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "")
} }
func runGet(cmd *base.Command, args []string) { func runGet(cmd *base.Command, args []string) {
if vgo.Enabled() { if cfg.ModulesEnabled {
base.Fatalf("go get: vgo not implemented") // Should not happen: main.go should install the separate module-enabled get code.
base.Fatalf("go get: modules not implemented")
} }
work.BuildInit() work.BuildInit()
...@@ -167,7 +190,7 @@ func runGet(cmd *base.Command, args []string) { ...@@ -167,7 +190,7 @@ func runGet(cmd *base.Command, args []string) {
return return
} }
work.InstallPackages(args, true) work.InstallPackages(args)
} }
// downloadPaths prepares the list of paths to pass to download. // downloadPaths prepares the list of paths to pass to download.
...@@ -176,6 +199,12 @@ func runGet(cmd *base.Command, args []string) { ...@@ -176,6 +199,12 @@ func runGet(cmd *base.Command, args []string) {
// in the hope that we can figure out the repository from the // in the hope that we can figure out the repository from the
// initial ...-free prefix. // initial ...-free prefix.
func downloadPaths(args []string) []string { func downloadPaths(args []string) []string {
for _, arg := range args {
if strings.Contains(arg, "@") {
base.Fatalf("go: cannot use path@version syntax in GOPATH mode")
}
}
args = load.ImportPathsForGoGet(args) args = load.ImportPathsForGoGet(args)
var out []string var out []string
for _, a := range args { for _, a := range args {
...@@ -379,7 +408,7 @@ func downloadPackage(p *load.Package) error { ...@@ -379,7 +408,7 @@ func downloadPackage(p *load.Package) error {
) )
security := web.Secure security := web.Secure
if *getInsecure { if Insecure {
security = web.Insecure security = web.Insecure
} }
...@@ -402,16 +431,16 @@ func downloadPackage(p *load.Package) error { ...@@ -402,16 +431,16 @@ func downloadPackage(p *load.Package) error {
} }
repo = remote repo = remote
if !*getF && err == nil { if !*getF && err == nil {
if rr, err := repoRootForImportPath(p.ImportPath, security); err == nil { if rr, err := RepoRootForImportPath(p.ImportPath, IgnoreMod, security); err == nil {
repo := rr.repo repo := rr.Repo
if rr.vcs.resolveRepo != nil { if rr.vcs.resolveRepo != nil {
resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo) resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo)
if err == nil { if err == nil {
repo = resolved repo = resolved
} }
} }
if remote != repo && rr.isCustom { if remote != repo && rr.IsCustom {
return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.root, repo, dir, remote) return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.Root, repo, dir, remote)
} }
} }
} }
...@@ -419,13 +448,13 @@ func downloadPackage(p *load.Package) error { ...@@ -419,13 +448,13 @@ func downloadPackage(p *load.Package) error {
} else { } else {
// Analyze the import path to determine the version control system, // Analyze the import path to determine the version control system,
// repository, and the import path for the root of the repository. // repository, and the import path for the root of the repository.
rr, err := repoRootForImportPath(p.ImportPath, security) rr, err := RepoRootForImportPath(p.ImportPath, IgnoreMod, security)
if err != nil { if err != nil {
return err return err
} }
vcs, repo, rootPath = rr.vcs, rr.repo, rr.root vcs, repo, rootPath = rr.vcs, rr.Repo, rr.Root
} }
if !blindRepo && !vcs.isSecure(repo) && !*getInsecure { if !blindRepo && !vcs.isSecure(repo) && !Insecure {
return fmt.Errorf("cannot download, %v uses insecure protocol", repo) return fmt.Errorf("cannot download, %v uses insecure protocol", repo)
} }
......
...@@ -33,15 +33,18 @@ func TestFoldDup(t *testing.T) { ...@@ -33,15 +33,18 @@ func TestFoldDup(t *testing.T) {
var parseMetaGoImportsTests = []struct { var parseMetaGoImportsTests = []struct {
in string in string
mod ModuleMode
out []metaImport out []metaImport
}{ }{
{ {
`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`, `<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
IgnoreMod,
[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
}, },
{ {
`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar"> `<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
<meta name="go-import" content="baz/quux git http://github.com/rsc/baz/quux">`, <meta name="go-import" content="baz/quux git http://github.com/rsc/baz/quux">`,
IgnoreMod,
[]metaImport{ []metaImport{
{"foo/bar", "git", "https://github.com/rsc/foo/bar"}, {"foo/bar", "git", "https://github.com/rsc/foo/bar"},
{"baz/quux", "git", "http://github.com/rsc/baz/quux"}, {"baz/quux", "git", "http://github.com/rsc/baz/quux"},
...@@ -50,6 +53,7 @@ var parseMetaGoImportsTests = []struct { ...@@ -50,6 +53,7 @@ var parseMetaGoImportsTests = []struct {
{ {
`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar"> `<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">`, <meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">`,
IgnoreMod,
[]metaImport{ []metaImport{
{"foo/bar", "git", "https://github.com/rsc/foo/bar"}, {"foo/bar", "git", "https://github.com/rsc/foo/bar"},
}, },
...@@ -57,35 +61,48 @@ var parseMetaGoImportsTests = []struct { ...@@ -57,35 +61,48 @@ var parseMetaGoImportsTests = []struct {
{ {
`<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux"> `<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">
<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`, <meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
IgnoreMod,
[]metaImport{ []metaImport{
{"foo/bar", "git", "https://github.com/rsc/foo/bar"}, {"foo/bar", "git", "https://github.com/rsc/foo/bar"},
}, },
}, },
{
`<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">
<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
PreferMod,
[]metaImport{
{"foo/bar", "mod", "http://github.com/rsc/baz/quux"},
},
},
{ {
`<head> `<head>
<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar"> <meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
</head>`, </head>`,
IgnoreMod,
[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
}, },
{ {
`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar"> `<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
<body>`, <body>`,
IgnoreMod,
[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
}, },
{ {
`<!doctype html><meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`, `<!doctype html><meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
IgnoreMod,
[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}}, []metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
}, },
{ {
// XML doesn't like <div style=position:relative>. // XML doesn't like <div style=position:relative>.
`<!doctype html><title>Page Not Found</title><meta name=go-import content="chitin.io/chitin git https://github.com/chitin-io/chitin"><div style=position:relative>DRAFT</div>`, `<!doctype html><title>Page Not Found</title><meta name=go-import content="chitin.io/chitin git https://github.com/chitin-io/chitin"><div style=position:relative>DRAFT</div>`,
IgnoreMod,
[]metaImport{{"chitin.io/chitin", "git", "https://github.com/chitin-io/chitin"}}, []metaImport{{"chitin.io/chitin", "git", "https://github.com/chitin-io/chitin"}},
}, },
} }
func TestParseMetaGoImports(t *testing.T) { func TestParseMetaGoImports(t *testing.T) {
for i, tt := range parseMetaGoImportsTests { for i, tt := range parseMetaGoImportsTests {
out, err := parseMetaGoImports(strings.NewReader(tt.in)) out, err := parseMetaGoImports(strings.NewReader(tt.in), tt.mod)
if err != nil { if err != nil {
t.Errorf("test#%d: %v", i, err) t.Errorf("test#%d: %v", i, err)
continue continue
......
...@@ -624,27 +624,29 @@ func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error { ...@@ -624,27 +624,29 @@ func checkNestedVCS(vcs *vcsCmd, dir, srcRoot string) error {
return nil return nil
} }
// repoRoot represents a version control system, a repo, and a root of // RepoRoot describes the repository root for a tree of source code.
// where to put it on disk. type RepoRoot struct {
type repoRoot struct { Repo string // repository URL, including scheme
vcs *vcsCmd Root string // import path corresponding to root of repo
IsCustom bool // defined by served <meta> tags (as opposed to hard-coded pattern)
// repo is the repository URL, including scheme VCS string // vcs type ("mod", "git", ...)
repo string
vcs *vcsCmd // internal: vcs command access
// root is the import path corresponding to the root of the
// repository
root string
// isCustom is true for custom import paths (those defined by HTML meta tags)
isCustom bool
} }
var httpPrefixRE = regexp.MustCompile(`^https?:`) var httpPrefixRE = regexp.MustCompile(`^https?:`)
// repoRootForImportPath analyzes importPath to determine the // ModuleMode specifies whether to prefer modules when looking up code sources.
type ModuleMode int
const (
IgnoreMod ModuleMode = iota
PreferMod
)
// RepoRootForImportPath analyzes importPath to determine the
// version control system, and code repository to use. // version control system, and code repository to use.
func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoRoot, error) { func RepoRootForImportPath(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths) rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths)
if err == errUnknownSite { if err == errUnknownSite {
// If there are wildcards, look up the thing before the wildcard, // If there are wildcards, look up the thing before the wildcard,
...@@ -654,7 +656,7 @@ func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoR ...@@ -654,7 +656,7 @@ func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoR
if i := strings.Index(lookup, "/.../"); i >= 0 { if i := strings.Index(lookup, "/.../"); i >= 0 {
lookup = lookup[:i] lookup = lookup[:i]
} }
rr, err = repoRootForImportDynamic(lookup, security) rr, err = repoRootForImportDynamic(lookup, mod, security)
if err != nil { if err != nil {
err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err) err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err)
} }
...@@ -667,7 +669,7 @@ func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoR ...@@ -667,7 +669,7 @@ func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoR
} }
} }
if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.root, "...") { if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") {
// Do not allow wildcards in the repo root. // Do not allow wildcards in the repo root.
rr = nil rr = nil
err = fmt.Errorf("cannot expand ... in %q", importPath) err = fmt.Errorf("cannot expand ... in %q", importPath)
...@@ -680,7 +682,7 @@ var errUnknownSite = errors.New("dynamic lookup required to find mapping") ...@@ -680,7 +682,7 @@ var errUnknownSite = errors.New("dynamic lookup required to find mapping")
// repoRootFromVCSPaths attempts to map importPath to a repoRoot // repoRootFromVCSPaths attempts to map importPath to a repoRoot
// using the mappings defined in vcsPaths. // using the mappings defined in vcsPaths.
// If scheme is non-empty, that scheme is forced. // If scheme is non-empty, that scheme is forced.
func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, vcsPaths []*vcsPath) (*repoRoot, error) { func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) {
// A common error is to use https://packagepath because that's what // A common error is to use https://packagepath because that's what
// hg and git require. Diagnose this helpfully. // hg and git require. Diagnose this helpfully.
if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil { if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil {
...@@ -733,28 +735,32 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, ...@@ -733,28 +735,32 @@ func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode,
if security == web.Secure && !vcs.isSecureScheme(scheme) { if security == web.Secure && !vcs.isSecureScheme(scheme) {
continue continue
} }
if vcs.ping(scheme, match["repo"]) == nil { if vcs.pingCmd != "" && vcs.ping(scheme, match["repo"]) == nil {
match["repo"] = scheme + "://" + match["repo"] match["repo"] = scheme + "://" + match["repo"]
break goto Found
} }
} }
// No scheme found. Fall back to the first one.
match["repo"] = vcs.scheme[0] + "://" + match["repo"]
Found:
} }
} }
rr := &repoRoot{ rr := &RepoRoot{
Repo: match["repo"],
Root: match["root"],
VCS: vcs.cmd,
vcs: vcs, vcs: vcs,
repo: match["repo"],
root: match["root"],
} }
return rr, nil return rr, nil
} }
return nil, errUnknownSite return nil, errUnknownSite
} }
// repoRootForImportDynamic finds a *repoRoot for a custom domain that's not // repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not
// statically known by repoRootForImportPathStatic. // statically known by repoRootForImportPathStatic.
// //
// This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". // This handles custom import paths like "name.tld/pkg/foo" or just "name.tld".
func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*repoRoot, error) { func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
slash := strings.Index(importPath, "/") slash := strings.Index(importPath, "/")
if slash < 0 { if slash < 0 {
slash = len(importPath) slash = len(importPath)
...@@ -772,7 +778,7 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re ...@@ -772,7 +778,7 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re
return nil, fmt.Errorf(msg, err) return nil, fmt.Errorf(msg, err)
} }
defer body.Close() defer body.Close()
imports, err := parseMetaGoImports(body) imports, err := parseMetaGoImports(body, mod)
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing %s: %v", importPath, err) return nil, fmt.Errorf("parsing %s: %v", importPath, err)
} }
...@@ -799,7 +805,7 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re ...@@ -799,7 +805,7 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re
} }
urlStr0 := urlStr urlStr0 := urlStr
var imports []metaImport var imports []metaImport
urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, security) urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -812,15 +818,18 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re ...@@ -812,15 +818,18 @@ func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*re
if err := validateRepoRoot(mmi.RepoRoot); err != nil { if err := validateRepoRoot(mmi.RepoRoot); err != nil {
return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, mmi.RepoRoot, err) return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, mmi.RepoRoot, err)
} }
rr := &repoRoot{ vcs := vcsByCmd(mmi.VCS)
vcs: vcsByCmd(mmi.VCS), if vcs == nil && mmi.VCS != "mod" {
repo: mmi.RepoRoot,
root: mmi.Prefix,
isCustom: true,
}
if rr.vcs == nil {
return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS) return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS)
} }
rr := &RepoRoot{
Repo: mmi.RepoRoot,
Root: mmi.Prefix,
IsCustom: true,
VCS: mmi.VCS,
vcs: vcs,
}
return rr, nil return rr, nil
} }
...@@ -851,7 +860,7 @@ var ( ...@@ -851,7 +860,7 @@ var (
// It is an error if no imports are found. // It is an error if no imports are found.
// urlStr will still be valid if err != nil. // urlStr will still be valid if err != nil.
// The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1" // The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1"
func metaImportsForPrefix(importPrefix string, security web.SecurityMode) (urlStr string, imports []metaImport, err error) { func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (urlStr string, imports []metaImport, err error) {
setCache := func(res fetchResult) (fetchResult, error) { setCache := func(res fetchResult) (fetchResult, error) {
fetchCacheMu.Lock() fetchCacheMu.Lock()
defer fetchCacheMu.Unlock() defer fetchCacheMu.Unlock()
...@@ -871,7 +880,7 @@ func metaImportsForPrefix(importPrefix string, security web.SecurityMode) (urlSt ...@@ -871,7 +880,7 @@ func metaImportsForPrefix(importPrefix string, security web.SecurityMode) (urlSt
if err != nil { if err != nil {
return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)}) return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)})
} }
imports, err := parseMetaGoImports(body) imports, err := parseMetaGoImports(body, mod)
if err != nil { if err != nil {
return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)}) return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)})
} }
......
...@@ -16,43 +16,43 @@ import ( ...@@ -16,43 +16,43 @@ import (
"cmd/go/internal/web" "cmd/go/internal/web"
) )
// Test that RepoRootForImportPath creates the correct RepoRoot for a given importPath. // Test that RepoRootForImportPath determines the correct RepoRoot for a given importPath.
// TODO(cmang): Add tests for SVN and BZR. // TODO(cmang): Add tests for SVN and BZR.
func TestRepoRootForImportPath(t *testing.T) { func TestRepoRootForImportPath(t *testing.T) {
testenv.MustHaveExternalNetwork(t) testenv.MustHaveExternalNetwork(t)
tests := []struct { tests := []struct {
path string path string
want *repoRoot want *RepoRoot
}{ }{
{ {
"github.com/golang/groupcache", "github.com/golang/groupcache",
&repoRoot{ &RepoRoot{
vcs: vcsGit, vcs: vcsGit,
repo: "https://github.com/golang/groupcache", Repo: "https://github.com/golang/groupcache",
}, },
}, },
// Unicode letters in directories (issue 18660). // Unicode letters in directories (issue 18660).
{ {
"github.com/user/unicode/испытание", "github.com/user/unicode/испытание",
&repoRoot{ &RepoRoot{
vcs: vcsGit, vcs: vcsGit,
repo: "https://github.com/user/unicode", Repo: "https://github.com/user/unicode",
}, },
}, },
// IBM DevOps Services tests // IBM DevOps Services tests
{ {
"hub.jazz.net/git/user1/pkgname", "hub.jazz.net/git/user1/pkgname",
&repoRoot{ &RepoRoot{
vcs: vcsGit, vcs: vcsGit,
repo: "https://hub.jazz.net/git/user1/pkgname", Repo: "https://hub.jazz.net/git/user1/pkgname",
}, },
}, },
{ {
"hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule", "hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule",
&repoRoot{ &RepoRoot{
vcs: vcsGit, vcs: vcsGit,
repo: "https://hub.jazz.net/git/user1/pkgname", Repo: "https://hub.jazz.net/git/user1/pkgname",
}, },
}, },
{ {
...@@ -91,9 +91,9 @@ func TestRepoRootForImportPath(t *testing.T) { ...@@ -91,9 +91,9 @@ func TestRepoRootForImportPath(t *testing.T) {
}, },
{ {
"hub.jazz.net/git/user/pkg.name", "hub.jazz.net/git/user/pkg.name",
&repoRoot{ &RepoRoot{
vcs: vcsGit, vcs: vcsGit,
repo: "https://hub.jazz.net/git/user/pkg.name", Repo: "https://hub.jazz.net/git/user/pkg.name",
}, },
}, },
// User names cannot have uppercase letters // User names cannot have uppercase letters
...@@ -104,9 +104,9 @@ func TestRepoRootForImportPath(t *testing.T) { ...@@ -104,9 +104,9 @@ func TestRepoRootForImportPath(t *testing.T) {
// OpenStack tests // OpenStack tests
{ {
"git.openstack.org/openstack/swift", "git.openstack.org/openstack/swift",
&repoRoot{ &RepoRoot{
vcs: vcsGit, vcs: vcsGit,
repo: "https://git.openstack.org/openstack/swift", Repo: "https://git.openstack.org/openstack/swift",
}, },
}, },
// Trailing .git is less preferred but included for // Trailing .git is less preferred but included for
...@@ -114,16 +114,16 @@ func TestRepoRootForImportPath(t *testing.T) { ...@@ -114,16 +114,16 @@ func TestRepoRootForImportPath(t *testing.T) {
// be compilable on both old and new go // be compilable on both old and new go
{ {
"git.openstack.org/openstack/swift.git", "git.openstack.org/openstack/swift.git",
&repoRoot{ &RepoRoot{
vcs: vcsGit, vcs: vcsGit,
repo: "https://git.openstack.org/openstack/swift.git", Repo: "https://git.openstack.org/openstack/swift.git",
}, },
}, },
{ {
"git.openstack.org/openstack/swift/go/hummingbird", "git.openstack.org/openstack/swift/go/hummingbird",
&repoRoot{ &RepoRoot{
vcs: vcsGit, vcs: vcsGit,
repo: "https://git.openstack.org/openstack/swift", Repo: "https://git.openstack.org/openstack/swift",
}, },
}, },
{ {
...@@ -150,23 +150,23 @@ func TestRepoRootForImportPath(t *testing.T) { ...@@ -150,23 +150,23 @@ func TestRepoRootForImportPath(t *testing.T) {
}, },
{ {
"git.apache.org/package-name.git", "git.apache.org/package-name.git",
&repoRoot{ &RepoRoot{
vcs: vcsGit, vcs: vcsGit,
repo: "https://git.apache.org/package-name.git", Repo: "https://git.apache.org/package-name.git",
}, },
}, },
{ {
"git.apache.org/package-name_2.x.git/path/to/lib", "git.apache.org/package-name_2.x.git/path/to/lib",
&repoRoot{ &RepoRoot{
vcs: vcsGit, vcs: vcsGit,
repo: "https://git.apache.org/package-name_2.x.git", Repo: "https://git.apache.org/package-name_2.x.git",
}, },
}, },
{ {
"chiselapp.com/user/kyle/repository/fossilgg", "chiselapp.com/user/kyle/repository/fossilgg",
&repoRoot{ &RepoRoot{
vcs: vcsFossil, vcs: vcsFossil,
repo: "https://chiselapp.com/user/kyle/repository/fossilgg", Repo: "https://chiselapp.com/user/kyle/repository/fossilgg",
}, },
}, },
{ {
...@@ -181,21 +181,21 @@ func TestRepoRootForImportPath(t *testing.T) { ...@@ -181,21 +181,21 @@ func TestRepoRootForImportPath(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
got, err := repoRootForImportPath(test.path, web.Secure) got, err := RepoRootForImportPath(test.path, IgnoreMod, web.Secure)
want := test.want want := test.want
if want == nil { if want == nil {
if err == nil { if err == nil {
t.Errorf("repoRootForImportPath(%q): Error expected but not received", test.path) t.Errorf("RepoRootForImportPath(%q): Error expected but not received", test.path)
} }
continue continue
} }
if err != nil { if err != nil {
t.Errorf("repoRootForImportPath(%q): %v", test.path, err) t.Errorf("RepoRootForImportPath(%q): %v", test.path, err)
continue continue
} }
if got.vcs.name != want.vcs.name || got.repo != want.repo { if got.vcs.name != want.vcs.name || got.Repo != want.Repo {
t.Errorf("repoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.repo, want.vcs, want.repo) t.Errorf("RepoRootForImportPath(%q) = VCS(%s) Repo(%s), want VCS(%s) Repo(%s)", test.path, got.vcs, got.Repo, want.vcs, want.Repo)
} }
} }
} }
...@@ -227,18 +227,18 @@ func TestFromDir(t *testing.T) { ...@@ -227,18 +227,18 @@ func TestFromDir(t *testing.T) {
f.Close() f.Close()
} }
want := repoRoot{ want := RepoRoot{
vcs: vcs, vcs: vcs,
root: path.Join("example.com", vcs.name), Root: path.Join("example.com", vcs.name),
} }
var got repoRoot var got RepoRoot
got.vcs, got.root, err = vcsFromDir(dir, tempDir) got.vcs, got.Root, err = vcsFromDir(dir, tempDir)
if err != nil { if err != nil {
t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err) t.Errorf("FromDir(%q, %q): %v", dir, tempDir, err)
continue continue
} }
if got.vcs.name != want.vcs.name || got.root != want.root { if got.vcs.name != want.vcs.name || got.Root != want.Root {
t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.root, want.vcs, want.root) t.Errorf("FromDir(%q, %q) = VCS(%s) Root(%s), want VCS(%s) Root(%s)", dir, tempDir, got.vcs, got.Root, want.vcs, want.Root)
} }
} }
} }
......
...@@ -45,7 +45,15 @@ func Help(args []string) { ...@@ -45,7 +45,15 @@ func Help(args []string) {
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
PrintUsage(buf) PrintUsage(buf)
usage := &base.Command{Long: buf.String()} usage := &base.Command{Long: buf.String()}
tmpl(&commentWriter{W: os.Stdout}, documentationTemplate, append([]*base.Command{usage}, base.Commands...)) cmds := []*base.Command{usage}
for _, cmd := range base.Commands {
if cmd.UsageLine == "gopath-get" {
// Avoid duplication of the "get" documentation.
continue
}
cmds = append(cmds, cmd)
}
tmpl(&commentWriter{W: os.Stdout}, documentationTemplate, cmds)
fmt.Println("package main") fmt.Println("package main")
return return
} }
......
...@@ -30,7 +30,7 @@ the C or C++ compiler, respectively, to use. ...@@ -30,7 +30,7 @@ the C or C++ compiler, respectively, to use.
var HelpPackages = &base.Command{ var HelpPackages = &base.Command{
UsageLine: "packages", UsageLine: "packages",
Short: "package lists", Short: "package lists and patterns",
Long: ` Long: `
Many commands apply to a set of packages: Many commands apply to a set of packages:
...@@ -54,9 +54,11 @@ for packages to be built with the go tool: ...@@ -54,9 +54,11 @@ for packages to be built with the go tool:
- "main" denotes the top-level package in a stand-alone executable. - "main" denotes the top-level package in a stand-alone executable.
- "all" expands to all package directories found in all the GOPATH - "all" expands to all packages found in all the GOPATH
trees. For example, 'go list all' lists all the packages on the local trees. For example, 'go list all' lists all the packages on the local
system. system. When using modules, "all" expands to all packages in
the main module and their dependencies, including dependencies
needed by tests of any of those.
- "std" is like all but expands to just the packages in the standard - "std" is like all but expands to just the packages in the standard
Go library. Go library.
......
This diff is collapsed.
...@@ -6,6 +6,7 @@ package load ...@@ -6,6 +6,7 @@ package load
import ( import (
"cmd/go/internal/base" "cmd/go/internal/base"
"cmd/go/internal/search"
"cmd/go/internal/str" "cmd/go/internal/str"
"fmt" "fmt"
"strings" "strings"
...@@ -92,7 +93,10 @@ func (f *PerPackageFlag) For(p *Package) []string { ...@@ -92,7 +93,10 @@ func (f *PerPackageFlag) For(p *Package) []string {
return flags return flags
} }
var cmdlineMatchers []func(*Package) bool var (
cmdlineMatchers []func(*Package) bool
cmdlineMatcherLiterals []func(*Package) bool
)
// SetCmdlinePatterns records the set of patterns given on the command line, // SetCmdlinePatterns records the set of patterns given on the command line,
// for use by the PerPackageFlags. // for use by the PerPackageFlags.
...@@ -105,9 +109,15 @@ func setCmdlinePatterns(args []string, cwd string) { ...@@ -105,9 +109,15 @@ func setCmdlinePatterns(args []string, cwd string) {
args = []string{"."} args = []string{"."}
} }
cmdlineMatchers = nil // allow reset for testing cmdlineMatchers = nil // allow reset for testing
cmdlineMatcherLiterals = nil
for _, arg := range args { for _, arg := range args {
cmdlineMatchers = append(cmdlineMatchers, MatchPackage(arg, cwd)) cmdlineMatchers = append(cmdlineMatchers, MatchPackage(arg, cwd))
} }
for _, arg := range args {
if !strings.Contains(arg, "...") && !search.IsMetaPackage(arg) {
cmdlineMatcherLiterals = append(cmdlineMatcherLiterals, MatchPackage(arg, cwd))
}
}
} }
// isCmdlinePkg reports whether p is a package listed on the command line. // isCmdlinePkg reports whether p is a package listed on the command line.
...@@ -119,3 +129,15 @@ func isCmdlinePkg(p *Package) bool { ...@@ -119,3 +129,15 @@ func isCmdlinePkg(p *Package) bool {
} }
return false return false
} }
// isCmdlinePkgLiteral reports whether p is a package listed as
// a literal package argument on the command line
// (as opposed to being the result of expanding a wildcard).
func isCmdlinePkgLiteral(p *Package) bool {
for _, m := range cmdlineMatcherLiterals {
if m(p) {
return true
}
}
return false
}
This diff is collapsed.
...@@ -15,7 +15,7 @@ import ( ...@@ -15,7 +15,7 @@ import (
// MatchPackage(pattern, cwd)(p) reports whether package p matches pattern in the working directory cwd. // MatchPackage(pattern, cwd)(p) reports whether package p matches pattern in the working directory cwd.
func MatchPackage(pattern, cwd string) func(*Package) bool { func MatchPackage(pattern, cwd string) func(*Package) bool {
switch { switch {
case strings.HasPrefix(pattern, "./") || strings.HasPrefix(pattern, "../") || pattern == "." || pattern == "..": case search.IsRelativePath(pattern):
// Split pattern into leading pattern-free directory path // Split pattern into leading pattern-free directory path
// (including all . and .. elements) and the final pattern. // (including all . and .. elements) and the final pattern.
var dir string var dir string
......
This diff is collapsed.
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
package vgo package modcmd
import ( import (
"bytes" "bytes"
...@@ -14,143 +14,165 @@ import ( ...@@ -14,143 +14,165 @@ import (
"strings" "strings"
"cmd/go/internal/base" "cmd/go/internal/base"
"cmd/go/internal/modload"
"cmd/go/internal/module" "cmd/go/internal/module"
) )
var CmdVendor = &base.Command{ func runVendor() {
UsageLine: "vendor [-v]", pkgs := modload.LoadVendor()
Short: "vendor dependencies of current module",
Long: `
Vendor resets the module's vendor directory to include all
packages needed to build and test all packages in the module
and their dependencies.
The -v flag causes vendor to print to standard error the
module paths of the modules processed and the import paths
of the packages copied.
`,
}
var vendorV = CmdVendor.Flag.Bool("v", false, "")
func init() { vdir := filepath.Join(modload.ModRoot, "vendor")
CmdVendor.Run = runVendor // break init cycle
}
func runVendor(cmd *base.Command, args []string) {
if Init(); !Enabled() {
base.Fatalf("vgo vendor: cannot use -m outside module")
}
if len(args) != 0 {
base.Fatalf("vgo vendor: vendor takes no arguments")
}
InitMod()
pkgs := ImportPaths([]string{"ALL"})
vdir := filepath.Join(ModRoot, "vendor")
if err := os.RemoveAll(vdir); err != nil { if err := os.RemoveAll(vdir); err != nil {
base.Fatalf("vgo vendor: %v", err) base.Fatalf("go vendor: %v", err)
} }
modpkgs := make(map[module.Version][]string) modpkgs := make(map[module.Version][]string)
for _, pkg := range pkgs { for _, pkg := range pkgs {
m := pkgmod[pkg] m := modload.PackageModule(pkg)
if m == Target { if m == modload.Target {
continue continue
} }
modpkgs[m] = append(modpkgs[m], pkg) modpkgs[m] = append(modpkgs[m], pkg)
} }
var buf bytes.Buffer var buf bytes.Buffer
for _, m := range buildList[1:] { for _, m := range modload.BuildList()[1:] {
if pkgs := modpkgs[m]; len(pkgs) > 0 { if pkgs := modpkgs[m]; len(pkgs) > 0 {
repl := "" repl := ""
if r := replaced(m); r != nil { if r := modload.Replacement(m); r.Path != "" {
repl = " => " + r.New.Path repl = " => " + r.Path
if r.New.Version != "" { if r.Version != "" {
repl += " " + r.New.Version repl += " " + r.Version
} }
} }
fmt.Fprintf(&buf, "# %s %s%s\n", m.Path, m.Version, repl) fmt.Fprintf(&buf, "# %s %s%s\n", m.Path, m.Version, repl)
if *vendorV { if *modV {
fmt.Fprintf(os.Stderr, "# %s %s%s\n", m.Path, m.Version, repl) fmt.Fprintf(os.Stderr, "# %s %s%s\n", m.Path, m.Version, repl)
} }
for _, pkg := range pkgs { for _, pkg := range pkgs {
fmt.Fprintf(&buf, "%s\n", pkg) fmt.Fprintf(&buf, "%s\n", pkg)
if *vendorV { if *modV {
fmt.Fprintf(os.Stderr, "%s\n", pkg) fmt.Fprintf(os.Stderr, "%s\n", pkg)
} }
vendorPkg(vdir, pkg) vendorPkg(vdir, pkg)
} }
} }
} }
if err := ioutil.WriteFile(filepath.Join(vdir, "vgo.list"), buf.Bytes(), 0666); err != nil { if buf.Len() == 0 {
base.Fatalf("vgo vendor: %v", err) fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
return
}
if err := ioutil.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil {
base.Fatalf("go vendor: %v", err)
} }
} }
func vendorPkg(vdir, pkg string) { func vendorPkg(vdir, pkg string) {
realPath := importmap[pkg] realPath := modload.ImportMap(pkg)
if realPath != pkg && importmap[realPath] != "" { if realPath != pkg && modload.ImportMap(realPath) != "" {
fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg) fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
} }
dst := filepath.Join(vdir, pkg) dst := filepath.Join(vdir, pkg)
src := pkgdir[realPath] src := modload.PackageDir(realPath)
if src == "" { if src == "" {
fmt.Fprintf(os.Stderr, "internal error: no pkg for %s -> %s\n", pkg, realPath) fmt.Fprintf(os.Stderr, "internal error: no pkg for %s -> %s\n", pkg, realPath)
} }
copyDir(dst, src, false) copyDir(dst, src, matchNonTest)
if m := modload.PackageModule(realPath); m.Path != "" {
copyMetadata(m.Path, realPath, dst, src)
}
} }
func copyDir(dst, src string, recursive bool) { type metakey struct {
modPath string
dst string
}
var copiedMetadata = make(map[metakey]bool)
// copyMetadata copies metadata files from parents of src to parents of dst,
// stopping after processing the src parent for modPath.
func copyMetadata(modPath, pkg, dst, src string) {
for parent := 0; ; parent++ {
if copiedMetadata[metakey{modPath, dst}] {
break
}
copiedMetadata[metakey{modPath, dst}] = true
if parent > 0 {
copyDir(dst, src, matchMetadata)
}
if modPath == pkg {
break
}
pkg = filepath.Dir(pkg)
dst = filepath.Dir(dst)
src = filepath.Dir(src)
}
}
// metaPrefixes is the list of metadata file prefixes.
// Vendoring copies metadata files from parents of copied directories.
// Note that this list could be arbitrarily extended, and it is longer
// in other tools (such as godep or dep). By using this limited set of
// prefixes and also insisting on capitalized file names, we are trying
// to nudge people toward more agreement on the naming
// and also trying to avoid false positives.
var metaPrefixes = []string{
"AUTHORS",
"CONTRIBUTORS",
"COPYLEFT",
"COPYING",
"COPYRIGHT",
"LEGAL",
"LICENSE",
"NOTICE",
"PATENTS",
}
// matchMetadata reports whether info is a metadata file.
func matchMetadata(info os.FileInfo) bool {
name := info.Name()
for _, p := range metaPrefixes {
if strings.HasPrefix(name, p) {
return true
}
}
return false
}
// matchNonTest reports whether info is any non-test file (including non-Go files).
func matchNonTest(info os.FileInfo) bool {
return !strings.HasSuffix(info.Name(), "_test.go")
}
// copyDir copies all regular files satisfying match(info) from src to dst.
func copyDir(dst, src string, match func(os.FileInfo) bool) {
files, err := ioutil.ReadDir(src) files, err := ioutil.ReadDir(src)
if err != nil { if err != nil {
base.Fatalf("vgo vendor: %v", err) base.Fatalf("go vendor: %v", err)
} }
if err := os.MkdirAll(dst, 0777); err != nil { if err := os.MkdirAll(dst, 0777); err != nil {
base.Fatalf("vgo vendor: %v", err) base.Fatalf("go vendor: %v", err)
} }
for _, file := range files { for _, file := range files {
if file.IsDir() { if file.IsDir() || !file.Mode().IsRegular() || !match(file) {
if recursive || file.Name() == "testdata" {
copyDir(filepath.Join(dst, file.Name()), filepath.Join(src, file.Name()), true)
}
continue
}
if !file.Mode().IsRegular() {
continue continue
} }
r, err := os.Open(filepath.Join(src, file.Name())) r, err := os.Open(filepath.Join(src, file.Name()))
if err != nil { if err != nil {
base.Fatalf("vgo vendor: %v", err) base.Fatalf("go vendor: %v", err)
} }
w, err := os.Create(filepath.Join(dst, file.Name())) w, err := os.Create(filepath.Join(dst, file.Name()))
if err != nil { if err != nil {
base.Fatalf("vgo vendor: %v", err) base.Fatalf("go vendor: %v", err)
} }
if _, err := io.Copy(w, r); err != nil { if _, err := io.Copy(w, r); err != nil {
base.Fatalf("vgo vendor: %v", err) base.Fatalf("go vendor: %v", err)
} }
r.Close() r.Close()
if err := w.Close(); err != nil { if err := w.Close(); err != nil {
base.Fatalf("vgo vendor: %v", err) base.Fatalf("go vendor: %v", err)
}
}
}
// hasPathPrefix reports whether the path s begins with the
// elements in prefix.
func hasPathPrefix(s, prefix string) bool {
switch {
default:
return false
case len(s) == len(prefix):
return s == prefix
case len(s) > len(prefix):
if prefix != "" && prefix[len(prefix)-1] == '/' {
return strings.HasPrefix(s, prefix)
} }
return s[len(prefix)] == '/' && s[:len(prefix)] == prefix
} }
} }
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
package vgo package modcmd
import ( import (
"bytes" "bytes"
...@@ -13,43 +13,14 @@ import ( ...@@ -13,43 +13,14 @@ import (
"cmd/go/internal/base" "cmd/go/internal/base"
"cmd/go/internal/dirhash" "cmd/go/internal/dirhash"
"cmd/go/internal/modfetch"
"cmd/go/internal/modload"
"cmd/go/internal/module" "cmd/go/internal/module"
) )
var CmdVerify = &base.Command{ func runVerify() {
UsageLine: "verify",
Run: runVerify,
Short: "verify downloaded modules against expected hashes",
Long: `
Verify checks that the dependencies of the current module,
which are stored in a local downloaded source cache,
have not been modified since being downloaded.
If all the modules are unmodified, verify prints
all modules verified
and exits successfully (status 0). Otherwise, verify reports
which modules have been changed and exits with a non-zero status.
`,
}
func runVerify(cmd *base.Command, args []string) {
if Init(); !Enabled() {
base.Fatalf("vgo verify: cannot use outside module")
}
if len(args) != 0 {
// TODO: take arguments
base.Fatalf("vgo verify: verify takes no arguments")
}
// Make go.mod consistent but don't load any packages.
InitMod()
iterate(func(*loader) {})
writeGoMod()
ok := true ok := true
for _, mod := range buildList[1:] { for _, mod := range modload.LoadBuildList()[1:] {
ok = verifyMod(mod) && ok ok = verifyMod(mod) && ok
} }
if ok { if ok {
...@@ -59,9 +30,9 @@ func runVerify(cmd *base.Command, args []string) { ...@@ -59,9 +30,9 @@ func runVerify(cmd *base.Command, args []string) {
func verifyMod(mod module.Version) bool { func verifyMod(mod module.Version) bool {
ok := true ok := true
zip := filepath.Join(srcV, "cache", mod.Path, "/@v/", mod.Version+".zip") zip := filepath.Join(modfetch.SrcMod, "cache/download", mod.Path, "/@v/", mod.Version+".zip")
_, zipErr := os.Stat(zip) _, zipErr := os.Stat(zip)
dir := filepath.Join(srcV, mod.Path+"@"+mod.Version) dir := filepath.Join(modfetch.SrcMod, mod.Path+"@"+mod.Version)
_, dirErr := os.Stat(dir) _, dirErr := os.Stat(dir)
data, err := ioutil.ReadFile(zip + "hash") data, err := ioutil.ReadFile(zip + "hash")
if err != nil { if err != nil {
......
...@@ -2,16 +2,20 @@ ...@@ -2,16 +2,20 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
package modfetch package modconv
import ( import (
"fmt" "fmt"
"os" "os"
"sort" "sort"
"strings" "strings"
"sync"
"cmd/go/internal/modconv" "cmd/go/internal/base"
"cmd/go/internal/modfetch"
"cmd/go/internal/modfile" "cmd/go/internal/modfile"
"cmd/go/internal/module"
"cmd/go/internal/par"
"cmd/go/internal/semver" "cmd/go/internal/semver"
) )
...@@ -23,47 +27,45 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { ...@@ -23,47 +27,45 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
if i >= 0 { if i >= 0 {
j = strings.LastIndex(file[:i], "/") j = strings.LastIndex(file[:i], "/")
} }
convert := modconv.Converters[file[i+1:]] convert := Converters[file[i+1:]]
if convert == nil && j != -2 { if convert == nil && j != -2 {
convert = modconv.Converters[file[j+1:]] convert = Converters[file[j+1:]]
} }
if convert == nil { if convert == nil {
return fmt.Errorf("unknown legacy config file %s", file) return fmt.Errorf("unknown legacy config file %s", file)
} }
require, err := convert(file, data) mf, err := convert(file, data)
if err != nil { if err != nil {
return fmt.Errorf("parsing %s: %v", file, err) return fmt.Errorf("parsing %s: %v", file, err)
} }
// Convert requirements block, which may use raw SHA1 hashes as versions, // Convert requirements block, which may use raw SHA1 hashes as versions,
// to valid semver requirement list, respecting major versions. // to valid semver requirement list, respecting major versions.
need := make(map[string]string) var work par.Work
for _, r := range require { for _, r := range mf.Require {
if r.Path == "" { m := r.Mod
if m.Path == "" {
continue continue
} }
work.Add(r.Mod)
}
// TODO: Something better here. var (
if strings.HasPrefix(r.Path, "github.com/") || strings.HasPrefix(r.Path, "golang.org/x/") { mu sync.Mutex
f := strings.Split(r.Path, "/") need = make(map[string]string)
if len(f) > 3 { )
r.Path = strings.Join(f[:3], "/") work.Do(10, func(item interface{}) {
} r := item.(module.Version)
} repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version)
repo, err := Lookup(r.Path)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "vgo: lookup %s: %v\n", r.Path, err) fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err)
continue return
}
info, err := repo.Stat(r.Version)
if err != nil {
fmt.Fprintf(os.Stderr, "vgo: stat %s@%s: %v\n", r.Path, r.Version, err)
continue
} }
mu.Lock()
path := repo.ModulePath() path := repo.ModulePath()
need[path] = semver.Max(need[path], info.Version) need[path] = semver.Max(need[path], info.Version)
} mu.Unlock()
})
var paths []string var paths []string
for path := range need { for path := range need {
...@@ -71,8 +73,15 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { ...@@ -71,8 +73,15 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
} }
sort.Strings(paths) sort.Strings(paths)
for _, path := range paths { for _, path := range paths {
f.AddRequire(path, need[path]) f.AddNewRequire(path, need[path], false)
} }
for _, r := range mf.Replace {
err := f.AddReplace(r.Old.Path, r.Old.Version, r.New.Path, r.New.Version)
if err != nil {
return fmt.Errorf("add replace: %v", err)
}
}
f.Cleanup()
return nil return nil
} }
...@@ -2,19 +2,49 @@ ...@@ -2,19 +2,49 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
package modfetch package modconv
import ( import (
"bytes" "bytes"
"fmt"
"internal/testenv" "internal/testenv"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strings" "strings"
"testing" "testing"
"cmd/go/internal/cfg" "cmd/go/internal/cfg"
"cmd/go/internal/modconv" "cmd/go/internal/modfetch"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/modfile" "cmd/go/internal/modfile"
"cmd/go/internal/module"
) )
func TestMain(m *testing.M) {
os.Exit(testMain(m))
}
func testMain(m *testing.M) int {
if _, err := exec.LookPath("git"); err != nil {
fmt.Fprintln(os.Stderr, "skipping because git binary not found")
fmt.Println("PASS")
return 0
}
dir, err := ioutil.TempDir("", "modconv-test-")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(dir)
modfetch.SrcMod = filepath.Join(dir, "src/mod")
codehost.WorkRoot = filepath.Join(dir, "codework")
return m.Run()
}
func TestConvertLegacyConfig(t *testing.T) { func TestConvertLegacyConfig(t *testing.T) {
testenv.MustHaveExternalNetwork(t) testenv.MustHaveExternalNetwork(t)
...@@ -65,7 +95,7 @@ func TestConvertLegacyConfig(t *testing.T) { ...@@ -65,7 +95,7 @@ func TestConvertLegacyConfig(t *testing.T) {
require ( require (
github.com/AdRoll/goamz v0.0.0-20150130162828-d3664b76d905 github.com/AdRoll/goamz v0.0.0-20150130162828-d3664b76d905
github.com/MSOpenTech/azure-sdk-for-go v0.0.0-20150323223030-d90753bcad2e github.com/MSOpenTech/azure-sdk-for-go v0.0.0-20150323223030-d90753bcad2e
github.com/Sirupsen/logrus v0.0.0-20150409230825-55eb11d21d2a github.com/Sirupsen/logrus v0.7.3
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b
github.com/bugsnag/panicwrap v0.0.0-20141110184334-e5f9854865b9 github.com/bugsnag/panicwrap v0.0.0-20141110184334-e5f9854865b9
...@@ -117,23 +147,32 @@ func TestConvertLegacyConfig(t *testing.T) { ...@@ -117,23 +147,32 @@ func TestConvertLegacyConfig(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
repo, err := Lookup(tt.path)
if err != nil {
t.Fatal(err)
}
out, err := repo.GoMod(tt.vers) dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
prefix := modconv.Prefix + "\n"
if !bytes.HasPrefix(out, []byte(prefix)) { for name := range Converters {
t.Fatalf("go.mod missing prefix %q:\n%s", prefix, out) file := filepath.Join(dir, name)
} data, err := ioutil.ReadFile(file)
out = out[len(prefix):] if err == nil {
if !bytes.Equal(out, want) { f := new(modfile.File)
t.Fatalf("final go.mod:\n%s\n\nwant:\n%s", out, want) f.AddModuleStmt(tt.path)
if err := ConvertLegacyConfig(f, filepath.ToSlash(file), data); err != nil {
t.Fatal(err)
}
out, err := f.Format()
if err != nil {
t.Fatalf("format after conversion: %v", err)
}
if !bytes.Equal(out, want) {
t.Fatalf("final go.mod:\n%s\n\nwant:\n%s", out, want)
}
return
}
} }
t.Fatalf("no converter found for %s@%s", tt.path, tt.vers)
}) })
} }
} }
...@@ -9,11 +9,13 @@ import ( ...@@ -9,11 +9,13 @@ import (
"strconv" "strconv"
"strings" "strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module" "cmd/go/internal/module"
"cmd/go/internal/semver" "cmd/go/internal/semver"
) )
func ParseGopkgLock(file string, data []byte) ([]module.Version, error) { func ParseGopkgLock(file string, data []byte) (*modfile.File, error) {
mf := new(modfile.File)
var list []module.Version var list []module.Version
var r *module.Version var r *module.Version
for lineno, line := range strings.Split(string(data), "\n") { for lineno, line := range strings.Split(string(data), "\n") {
...@@ -66,6 +68,7 @@ func ParseGopkgLock(file string, data []byte) ([]module.Version, error) { ...@@ -66,6 +68,7 @@ func ParseGopkgLock(file string, data []byte) ([]module.Version, error) {
if r.Path == "" || r.Version == "" { if r.Path == "" || r.Version == "" {
return nil, fmt.Errorf("%s: empty [[projects]] stanza (%s)", file, r.Path) return nil, fmt.Errorf("%s: empty [[projects]] stanza (%s)", file, r.Path)
} }
mf.Require = append(mf.Require, &modfile.Require{Mod: r})
} }
return list, nil return mf, nil
} }
...@@ -5,12 +5,14 @@ ...@@ -5,12 +5,14 @@
package modconv package modconv
import ( import (
"cmd/go/internal/module"
"strings" "strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
) )
func ParseGlideLock(file string, data []byte) ([]module.Version, error) { func ParseGlideLock(file string, data []byte) (*modfile.File, error) {
var list []module.Version mf := new(modfile.File)
imports := false imports := false
name := "" name := ""
for lineno, line := range strings.Split(string(data), "\n") { for lineno, line := range strings.Split(string(data), "\n") {
...@@ -32,9 +34,9 @@ func ParseGlideLock(file string, data []byte) ([]module.Version, error) { ...@@ -32,9 +34,9 @@ func ParseGlideLock(file string, data []byte) ([]module.Version, error) {
if strings.HasPrefix(line, " version:") { if strings.HasPrefix(line, " version:") {
version := strings.TrimSpace(line[len(" version:"):]) version := strings.TrimSpace(line[len(" version:"):])
if name != "" && version != "" { if name != "" && version != "" {
list = append(list, module.Version{Path: name, Version: version}) mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: name, Version: version}})
} }
} }
} }
return list, nil return mf, nil
} }
...@@ -7,17 +7,18 @@ package modconv ...@@ -7,17 +7,18 @@ package modconv
import ( import (
"strings" "strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module" "cmd/go/internal/module"
) )
func ParseGLOCKFILE(file string, data []byte) ([]module.Version, error) { func ParseGLOCKFILE(file string, data []byte) (*modfile.File, error) {
var list []module.Version mf := new(modfile.File)
for lineno, line := range strings.Split(string(data), "\n") { for lineno, line := range strings.Split(string(data), "\n") {
lineno++ lineno++
f := strings.Fields(line) f := strings.Fields(line)
if len(f) >= 2 && f[0] != "cmd" { if len(f) >= 2 && f[0] != "cmd" {
list = append(list, module.Version{Path: f[0], Version: f[1]}) mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[1]}})
} }
} }
return list, nil return mf, nil
} }
...@@ -7,10 +7,11 @@ package modconv ...@@ -7,10 +7,11 @@ package modconv
import ( import (
"encoding/json" "encoding/json"
"cmd/go/internal/modfile"
"cmd/go/internal/module" "cmd/go/internal/module"
) )
func ParseGodepsJSON(file string, data []byte) ([]module.Version, error) { func ParseGodepsJSON(file string, data []byte) (*modfile.File, error) {
var cfg struct { var cfg struct {
ImportPath string ImportPath string
Deps []struct { Deps []struct {
...@@ -21,9 +22,9 @@ func ParseGodepsJSON(file string, data []byte) ([]module.Version, error) { ...@@ -21,9 +22,9 @@ func ParseGodepsJSON(file string, data []byte) ([]module.Version, error) {
if err := json.Unmarshal(data, &cfg); err != nil { if err := json.Unmarshal(data, &cfg); err != nil {
return nil, err return nil, err
} }
var list []module.Version mf := new(modfile.File)
for _, d := range cfg.Deps { for _, d := range cfg.Deps {
list = append(list, module.Version{Path: d.ImportPath, Version: d.Rev}) mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: d.ImportPath, Version: d.Rev}})
} }
return list, nil return mf, nil
} }
...@@ -4,9 +4,9 @@ ...@@ -4,9 +4,9 @@
package modconv package modconv
import "cmd/go/internal/module" import "cmd/go/internal/modfile"
var Converters = map[string]func(string, []byte) ([]module.Version, error){ var Converters = map[string]func(string, []byte) (*modfile.File, error){
"GLOCKFILE": ParseGLOCKFILE, "GLOCKFILE": ParseGLOCKFILE,
"Godeps/Godeps.json": ParseGodepsJSON, "Godeps/Godeps.json": ParseGodepsJSON,
"Gopkg.lock": ParseGopkgLock, "Gopkg.lock": ParseGopkgLock,
...@@ -17,9 +17,3 @@ var Converters = map[string]func(string, []byte) ([]module.Version, error){ ...@@ -17,9 +17,3 @@ var Converters = map[string]func(string, []byte) ([]module.Version, error){
"vendor/manifest": ParseVendorManifest, "vendor/manifest": ParseVendorManifest,
"vendor/vendor.json": ParseVendorJSON, "vendor/vendor.json": ParseVendorJSON,
} }
// Prefix is a line we write at the top of auto-converted go.mod files
// for dependencies before caching them.
// In case of bugs in the converter, if we bump this version number,
// then all the cached copies will be ignored.
const Prefix = "//vgo 0.0.4\n"
...@@ -7,7 +7,6 @@ package modconv ...@@ -7,7 +7,6 @@ package modconv
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"internal/testenv"
"io/ioutil" "io/ioutil"
"path/filepath" "path/filepath"
"testing" "testing"
...@@ -26,8 +25,6 @@ var extMap = map[string]string{ ...@@ -26,8 +25,6 @@ var extMap = map[string]string{
} }
func Test(t *testing.T) { func Test(t *testing.T) {
testenv.MustHaveExternalNetwork(t)
tests, _ := filepath.Glob("testdata/*") tests, _ := filepath.Glob("testdata/*")
if len(tests) == 0 { if len(tests) == 0 {
t.Fatalf("no tests found") t.Fatalf("no tests found")
...@@ -58,8 +55,8 @@ func Test(t *testing.T) { ...@@ -58,8 +55,8 @@ func Test(t *testing.T) {
t.Error(err) t.Error(err)
} }
var buf bytes.Buffer var buf bytes.Buffer
for _, r := range out { for _, r := range out.Require {
fmt.Fprintf(&buf, "%s %s\n", r.Path, r.Version) fmt.Fprintf(&buf, "%s %s\n", r.Mod.Path, r.Mod.Version)
} }
if !bytes.Equal(buf.Bytes(), want) { if !bytes.Equal(buf.Bytes(), want) {
t.Errorf("have:\n%s\nwant:\n%s", buf.Bytes(), want) t.Errorf("have:\n%s\nwant:\n%s", buf.Bytes(), want)
......
...@@ -7,17 +7,18 @@ package modconv ...@@ -7,17 +7,18 @@ package modconv
import ( import (
"strings" "strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module" "cmd/go/internal/module"
) )
func ParseDependenciesTSV(file string, data []byte) ([]module.Version, error) { func ParseDependenciesTSV(file string, data []byte) (*modfile.File, error) {
var list []module.Version mf := new(modfile.File)
for lineno, line := range strings.Split(string(data), "\n") { for lineno, line := range strings.Split(string(data), "\n") {
lineno++ lineno++
f := strings.Split(line, "\t") f := strings.Split(line, "\t")
if len(f) >= 3 { if len(f) >= 3 {
list = append(list, module.Version{Path: f[0], Version: f[2]}) mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[2]}})
} }
} }
return list, nil return mf, nil
} }
...@@ -7,11 +7,12 @@ package modconv ...@@ -7,11 +7,12 @@ package modconv
import ( import (
"strings" "strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module" "cmd/go/internal/module"
) )
func ParseVendorConf(file string, data []byte) ([]module.Version, error) { func ParseVendorConf(file string, data []byte) (*modfile.File, error) {
var list []module.Version mf := new(modfile.File)
for lineno, line := range strings.Split(string(data), "\n") { for lineno, line := range strings.Split(string(data), "\n") {
lineno++ lineno++
if i := strings.Index(line, "#"); i >= 0 { if i := strings.Index(line, "#"); i >= 0 {
...@@ -19,8 +20,8 @@ func ParseVendorConf(file string, data []byte) ([]module.Version, error) { ...@@ -19,8 +20,8 @@ func ParseVendorConf(file string, data []byte) ([]module.Version, error) {
} }
f := strings.Fields(line) f := strings.Fields(line)
if len(f) >= 2 { if len(f) >= 2 {
list = append(list, module.Version{Path: f[0], Version: f[1]}) mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: f[0], Version: f[1]}})
} }
} }
return list, nil return mf, nil
} }
...@@ -7,10 +7,11 @@ package modconv ...@@ -7,10 +7,11 @@ package modconv
import ( import (
"encoding/json" "encoding/json"
"cmd/go/internal/modfile"
"cmd/go/internal/module" "cmd/go/internal/module"
) )
func ParseVendorJSON(file string, data []byte) ([]module.Version, error) { func ParseVendorJSON(file string, data []byte) (*modfile.File, error) {
var cfg struct { var cfg struct {
Package []struct { Package []struct {
Path string Path string
...@@ -20,9 +21,9 @@ func ParseVendorJSON(file string, data []byte) ([]module.Version, error) { ...@@ -20,9 +21,9 @@ func ParseVendorJSON(file string, data []byte) ([]module.Version, error) {
if err := json.Unmarshal(data, &cfg); err != nil { if err := json.Unmarshal(data, &cfg); err != nil {
return nil, err return nil, err
} }
var list []module.Version mf := new(modfile.File)
for _, d := range cfg.Package { for _, d := range cfg.Package {
list = append(list, module.Version{Path: d.Path, Version: d.Revision}) mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: d.Path, Version: d.Revision}})
} }
return list, nil return mf, nil
} }
...@@ -7,10 +7,11 @@ package modconv ...@@ -7,10 +7,11 @@ package modconv
import ( import (
"encoding/json" "encoding/json"
"cmd/go/internal/modfile"
"cmd/go/internal/module" "cmd/go/internal/module"
) )
func ParseVendorManifest(file string, data []byte) ([]module.Version, error) { func ParseVendorManifest(file string, data []byte) (*modfile.File, error) {
var cfg struct { var cfg struct {
Dependencies []struct { Dependencies []struct {
ImportPath string ImportPath string
...@@ -20,9 +21,9 @@ func ParseVendorManifest(file string, data []byte) ([]module.Version, error) { ...@@ -20,9 +21,9 @@ func ParseVendorManifest(file string, data []byte) ([]module.Version, error) {
if err := json.Unmarshal(data, &cfg); err != nil { if err := json.Unmarshal(data, &cfg); err != nil {
return nil, err return nil, err
} }
var list []module.Version mf := new(modfile.File)
for _, d := range cfg.Dependencies { for _, d := range cfg.Dependencies {
list = append(list, module.Version{Path: d.ImportPath, Version: d.Revision}) mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: d.ImportPath, Version: d.Revision}})
} }
return list, nil return mf, nil
} }
...@@ -5,12 +5,14 @@ ...@@ -5,12 +5,14 @@
package modconv package modconv
import ( import (
"cmd/go/internal/module"
"strings" "strings"
"cmd/go/internal/modfile"
"cmd/go/internal/module"
) )
func ParseVendorYML(file string, data []byte) ([]module.Version, error) { func ParseVendorYML(file string, data []byte) (*modfile.File, error) {
var list []module.Version mf := new(modfile.File)
vendors := false vendors := false
path := "" path := ""
for lineno, line := range strings.Split(string(data), "\n") { for lineno, line := range strings.Split(string(data), "\n") {
...@@ -32,9 +34,9 @@ func ParseVendorYML(file string, data []byte) ([]module.Version, error) { ...@@ -32,9 +34,9 @@ func ParseVendorYML(file string, data []byte) ([]module.Version, error) {
if strings.HasPrefix(line, " rev:") { if strings.HasPrefix(line, " rev:") {
rev := strings.TrimSpace(line[len(" rev:"):]) rev := strings.TrimSpace(line[len(" rev:"):])
if path != "" && rev != "" { if path != "" && rev != "" {
list = append(list, module.Version{Path: path, Version: rev}) mf.Require = append(mf.Require, &modfile.Require{Mod: module.Version{Path: path, Version: rev}})
} }
} }
} }
return list, nil return mf, nil
} }
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bitbucket
import (
"fmt"
"strings"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/modfetch/gitrepo"
)
func Lookup(path string) (codehost.Repo, error) {
f := strings.Split(path, "/")
if len(f) < 3 || f[0] != "bitbucket.org" {
return nil, fmt.Errorf("bitbucket repo must be bitbucket.org/org/project")
}
path = f[0] + "/" + f[1] + "/" + f[2]
return gitrepo.Repo("https://"+path, path)
}
This diff is collapsed.
...@@ -2,21 +2,24 @@ ...@@ -2,21 +2,24 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// TODO: Figure out what gopkg.in should do.
package modfetch package modfetch
import ( import (
"cmd/go/internal/modfetch/codehost" "io/ioutil"
"cmd/go/internal/modfetch/gitrepo" "os"
"cmd/go/internal/modfile" "path/filepath"
"fmt" "testing"
) )
func gopkginLookup(path string) (codehost.Repo, error) { func TestWriteDiskCache(t *testing.T) {
root, _, _, _, ok := modfile.ParseGopkgIn(path) tmpdir, err := ioutil.TempDir("", "go-writeCache-test-")
if !ok { if err != nil {
return nil, fmt.Errorf("invalid gopkg.in/ path: %q", path) t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
err = writeDiskCache(filepath.Join(tmpdir, "file"), []byte("data"))
if err != nil {
t.Fatal(err)
} }
return gitrepo.Repo("https://"+root, root)
} }
...@@ -16,6 +16,7 @@ import ( ...@@ -16,6 +16,7 @@ import (
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"time" "time"
"cmd/go/internal/cfg" "cmd/go/internal/cfg"
...@@ -32,10 +33,8 @@ const ( ...@@ -32,10 +33,8 @@ const (
// A Repo represents a code hosting source. // A Repo represents a code hosting source.
// Typical implementations include local version control repositories, // Typical implementations include local version control repositories,
// remote version control servers, and code hosting sites. // remote version control servers, and code hosting sites.
// A Repo must be safe for simultaneous use by multiple goroutines.
type Repo interface { type Repo interface {
// Root returns the import path of the root directory of the repository.
Root() string
// List lists all tags with the given prefix. // List lists all tags with the given prefix.
Tags(prefix string) (tags []string, err error) Tags(prefix string) (tags []string, err error)
...@@ -50,6 +49,9 @@ type Repo interface { ...@@ -50,6 +49,9 @@ type Repo interface {
// ReadFile reads the given file in the file tree corresponding to revision rev. // ReadFile reads the given file in the file tree corresponding to revision rev.
// It should refuse to read more than maxSize bytes. // It should refuse to read more than maxSize bytes.
//
// If the requested file does not exist it should return an error for which
// os.IsNotExist(err) returns true.
ReadFile(rev, file string, maxSize int64) (data []byte, err error) ReadFile(rev, file string, maxSize int64) (data []byte, err error)
// ReadZip downloads a zip file for the subdir subdirectory // ReadZip downloads a zip file for the subdir subdirectory
...@@ -66,8 +68,9 @@ type Repo interface { ...@@ -66,8 +68,9 @@ type Repo interface {
type RevInfo struct { type RevInfo struct {
Name string // complete ID in underlying repository Name string // complete ID in underlying repository
Short string // shortened ID, for use in pseudo-version Short string // shortened ID, for use in pseudo-version
Version string // TODO what is this? Version string // version used in lookup
Time time.Time // commit time Time time.Time // commit time
Tags []string // known tags for commit
} }
// AllHex reports whether the revision rev is entirely lower-case hexadecimal digits. // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
...@@ -92,7 +95,7 @@ func ShortenSHA1(rev string) string { ...@@ -92,7 +95,7 @@ func ShortenSHA1(rev string) string {
} }
// WorkRoot is the root of the cached work directory. // WorkRoot is the root of the cached work directory.
// It is set by cmd/go/internal/vgo.InitMod. // It is set by cmd/go/internal/modload.InitMod.
var WorkRoot string var WorkRoot string
// WorkDir returns the name of the cached work directory to use for the // WorkDir returns the name of the cached work directory to use for the
...@@ -113,21 +116,20 @@ func WorkDir(typ, name string) (string, error) { ...@@ -113,21 +116,20 @@ func WorkDir(typ, name string) (string, error) {
key := typ + ":" + name key := typ + ":" + name
dir := filepath.Join(WorkRoot, fmt.Sprintf("%x", sha256.Sum256([]byte(key)))) dir := filepath.Join(WorkRoot, fmt.Sprintf("%x", sha256.Sum256([]byte(key))))
data, err := ioutil.ReadFile(dir + ".info") data, err := ioutil.ReadFile(dir + ".info")
if err == nil { info, err2 := os.Stat(dir)
if err == nil && err2 == nil && info.IsDir() {
// Info file and directory both already exist: reuse.
have := strings.TrimSuffix(string(data), "\n") have := strings.TrimSuffix(string(data), "\n")
if have != key { if have != key {
return "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key) return "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key)
} }
_, err := os.Stat(dir)
if err != nil {
return "", fmt.Errorf("%s exists but %s does not", dir+".info", dir)
}
if cfg.BuildX { if cfg.BuildX {
fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name) fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name)
} }
return dir, nil return dir, nil
} }
// Info file or directory missing. Start from scratch.
if cfg.BuildX { if cfg.BuildX {
fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name) fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name)
} }
...@@ -157,19 +159,36 @@ func (e *RunError) Error() string { ...@@ -157,19 +159,36 @@ func (e *RunError) Error() string {
return text return text
} }
var dirLock sync.Map
// Run runs the command line in the given directory // Run runs the command line in the given directory
// (an empty dir means the current directory). // (an empty dir means the current directory).
// It returns the standard output and, for a non-zero exit, // It returns the standard output and, for a non-zero exit,
// a *RunError indicating the command, exit status, and standard error. // a *RunError indicating the command, exit status, and standard error.
// Standard error is unavailable for commands that exit successfully. // Standard error is unavailable for commands that exit successfully.
func Run(dir string, cmdline ...interface{}) ([]byte, error) { func Run(dir string, cmdline ...interface{}) ([]byte, error) {
if dir != "" {
muIface, ok := dirLock.Load(dir)
if !ok {
muIface, _ = dirLock.LoadOrStore(dir, new(sync.Mutex))
}
mu := muIface.(*sync.Mutex)
mu.Lock()
defer mu.Unlock()
}
cmd := str.StringList(cmdline...) cmd := str.StringList(cmdline...)
if cfg.BuildX { if cfg.BuildX {
var cd string var text string
if dir != "" { if dir != "" {
cd = "cd " + dir + "; " text = "cd " + dir + "; "
} }
fmt.Fprintf(os.Stderr, "%s%s\n", cd, strings.Join(cmd, " ")) text += strings.Join(cmd, " ")
fmt.Fprintf(os.Stderr, "%s\n", text)
start := time.Now()
defer func() {
fmt.Fprintf(os.Stderr, "%.3fs # %s\n", time.Since(start).Seconds(), text)
}()
} }
// TODO: Impose limits on command output size. // TODO: Impose limits on command output size.
// TODO: Set environment to get English error messages. // TODO: Set environment to get English error messages.
......
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
// Interactive debugging shell for codehost.Repo implementations.
package main
import (
"archive/zip"
"bufio"
"bytes"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"time"
"cmd/go/internal/modfetch/codehost"
)
func usage() {
fmt.Fprintf(os.Stderr, "usage: go run shell.go vcs remote\n")
os.Exit(2)
}
func main() {
codehost.WorkRoot = "/tmp/vcswork"
log.SetFlags(0)
log.SetPrefix("shell: ")
flag.Usage = usage
flag.Parse()
if flag.NArg() != 2 {
usage()
}
repo, err := codehost.NewRepo(flag.Arg(0), flag.Arg(1))
if err != nil {
log.Fatal(err)
}
b := bufio.NewReader(os.Stdin)
for {
fmt.Fprintf(os.Stderr, ">>> ")
line, err := b.ReadString('\n')
if err != nil {
log.Fatal(err)
}
f := strings.Fields(line)
if len(f) == 0 {
continue
}
switch f[0] {
default:
fmt.Fprintf(os.Stderr, "?unknown command\n")
continue
case "tags":
prefix := ""
if len(f) == 2 {
prefix = f[1]
}
if len(f) > 2 {
fmt.Fprintf(os.Stderr, "?usage: tags [prefix]\n")
continue
}
tags, err := repo.Tags(prefix)
if err != nil {
fmt.Fprintf(os.Stderr, "?%s\n", err)
continue
}
for _, tag := range tags {
fmt.Printf("%s\n", tag)
}
case "stat":
if len(f) != 2 {
fmt.Fprintf(os.Stderr, "?usage: stat rev\n")
continue
}
info, err := repo.Stat(f[1])
if err != nil {
fmt.Fprintf(os.Stderr, "?%s\n", err)
continue
}
fmt.Printf("name=%s short=%s version=%s time=%s\n", info.Name, info.Short, info.Version, info.Time.UTC().Format(time.RFC3339))
case "read":
if len(f) != 3 {
fmt.Fprintf(os.Stderr, "?usage: read rev file\n")
continue
}
data, err := repo.ReadFile(f[1], f[2], 10<<20)
if err != nil {
fmt.Fprintf(os.Stderr, "?%s\n", err)
continue
}
os.Stdout.Write(data)
case "zip":
if len(f) != 4 {
fmt.Fprintf(os.Stderr, "?usage: zip rev subdir output\n")
continue
}
subdir := f[2]
if subdir == "-" {
subdir = ""
}
rc, _, err := repo.ReadZip(f[1], subdir, 10<<20)
if err != nil {
fmt.Fprintf(os.Stderr, "?%s\n", err)
continue
}
data, err := ioutil.ReadAll(rc)
rc.Close()
if err != nil {
fmt.Fprintf(os.Stderr, "?%s\n", err)
continue
}
if f[3] != "-" {
if err := ioutil.WriteFile(f[3], data, 0666); err != nil {
fmt.Fprintf(os.Stderr, "?%s\n", err)
continue
}
}
z, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
fmt.Fprintf(os.Stderr, "?%s\n", err)
continue
}
for _, f := range z.File {
fmt.Printf("%s %d\n", f.Name, f.UncompressedSize64)
}
}
}
}
This diff is collapsed.
This diff is collapsed.
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Support for custom domains.
package modfetch
import (
"encoding/xml"
"fmt"
"io"
"net/url"
"os"
"strings"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/modfetch/gitrepo"
)
// metaImport represents the parsed <meta name="go-import"
// content="prefix vcs reporoot" /> tags from HTML files.
type metaImport struct {
Prefix, VCS, RepoRoot string
}
func lookupCustomDomain(path string) (Repo, error) {
dom := path
if i := strings.Index(dom, "/"); i >= 0 {
dom = dom[:i]
}
if !strings.Contains(dom, ".") {
return nil, fmt.Errorf("unknown module %s: not a domain name", path)
}
var body io.ReadCloser
err := webGetGoGet("https://"+path+"?go-get=1", &body)
if body != nil {
defer body.Close()
}
if err != nil {
fmt.Fprintf(os.Stderr, "FindRepo: %v\n", err)
return nil, err
}
// Note: accepting a non-200 OK here, so people can serve a
// meta import in their http 404 page.
imports, err := parseMetaGoImports(body)
if err != nil {
fmt.Fprintf(os.Stderr, "findRepo: %v\n", err)
return nil, err
}
if len(imports) == 0 {
return nil, fmt.Errorf("unknown module %s: no go-import tags", path)
}
// First look for new module definition.
for _, imp := range imports {
if path == imp.Prefix || strings.HasPrefix(path, imp.Prefix+"/") {
if imp.VCS == "mod" {
u, err := url.Parse(imp.RepoRoot)
if err != nil {
return nil, fmt.Errorf("invalid module URL %q", imp.RepoRoot)
} else if u.Scheme != "https" {
// TODO: Allow -insecure flag as a build flag?
return nil, fmt.Errorf("invalid module URL %q: must be HTTPS", imp.RepoRoot)
}
return newProxyRepo(imp.RepoRoot, imp.Prefix), nil
}
}
}
// Fall back to redirections to known version control systems.
for _, imp := range imports {
if path == imp.Prefix {
if !strings.HasPrefix(imp.RepoRoot, "https://") {
// TODO: Allow -insecure flag as a build flag?
return nil, fmt.Errorf("invalid server URL %q: must be HTTPS", imp.RepoRoot)
}
if imp.VCS == "git" {
code, err := gitrepo.Repo(imp.RepoRoot, imp.Prefix)
if err != nil {
return nil, err
}
return newCodeRepo(code, path)
}
return nil, fmt.Errorf("unknown VCS, Repo: %s, %s", imp.VCS, imp.RepoRoot)
}
}
// Check for redirect to repo root.
for _, imp := range imports {
if strings.HasPrefix(path, imp.Prefix+"/") {
return nil, &ModuleSubdirError{imp.Prefix}
}
}
return nil, fmt.Errorf("unknown module %s: no matching go-import tags", path)
}
type ModuleSubdirError struct {
ModulePath string
}
func (e *ModuleSubdirError) Error() string {
return fmt.Sprintf("module root is %q", e.ModulePath)
}
type customPrefix struct {
codehost.Repo
root string
}
func (c *customPrefix) Root() string {
return c.root
}
// parseMetaGoImports returns meta imports from the HTML in r.
// Parsing ends at the end of the <head> section or the beginning of the <body>.
func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
d := xml.NewDecoder(r)
d.CharsetReader = charsetReader
d.Strict = false
var t xml.Token
for {
t, err = d.RawToken()
if err != nil {
if err == io.EOF || len(imports) > 0 {
err = nil
}
return
}
if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
return
}
if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
return
}
e, ok := t.(xml.StartElement)
if !ok || !strings.EqualFold(e.Name.Local, "meta") {
continue
}
if attrValue(e.Attr, "name") != "go-import" {
continue
}
if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
imports = append(imports, metaImport{
Prefix: f[0],
VCS: f[1],
RepoRoot: f[2],
})
}
}
}
// attrValue returns the attribute value for the case-insensitive key
// `name', or the empty string if nothing is found.
func attrValue(attrs []xml.Attr, name string) string {
for _, a := range attrs {
if strings.EqualFold(a.Name.Local, name) {
return a.Value
}
}
return ""
}
// charsetReader returns a reader for the given charset. Currently
// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful
// error which is printed by go get, so the user can find why the package
// wasn't downloaded if the encoding is not supported. Note that, in
// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters
// greater than 0x7f are not rejected).
func charsetReader(charset string, input io.Reader) (io.Reader, error) {
switch strings.ToLower(charset) {
case "ascii":
return input, nil
default:
return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
}
}
This diff is collapsed.
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package github
import (
"fmt"
"strings"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/modfetch/gitrepo"
)
// Lookup returns the code repository enclosing the given module path,
// which must begin with github.com/.
func Lookup(path string) (codehost.Repo, error) {
f := strings.Split(path, "/")
if len(f) < 3 || f[0] != "github.com" {
return nil, fmt.Errorf("github repo must be github.com/org/project")
}
path = f[0] + "/" + f[1] + "/" + f[2]
return gitrepo.Repo("https://"+path, path)
}
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package googlesource
import (
"fmt"
"strings"
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/modfetch/gitrepo"
)
func Lookup(path string) (codehost.Repo, error) {
i := strings.Index(path, "/")
if i+1 == len(path) || !strings.HasSuffix(path[:i+1], ".googlesource.com/") {
return nil, fmt.Errorf("not *.googlesource.com/*")
}
j := strings.Index(path[i+1:], "/")
if j >= 0 {
path = path[:i+1+j]
}
return gitrepo.Repo("https://"+path, path)
}
...@@ -55,6 +55,7 @@ func (p *proxyRepo) Versions(prefix string) ([]string, error) { ...@@ -55,6 +55,7 @@ func (p *proxyRepo) Versions(prefix string) ([]string, error) {
list = append(list, f[0]) list = append(list, f[0])
} }
} }
SortVersions(list)
return list, nil return list, nil
} }
...@@ -134,7 +135,7 @@ func (p *proxyRepo) Zip(version string, tmpdir string) (tmpfile string, err erro ...@@ -134,7 +135,7 @@ func (p *proxyRepo) Zip(version string, tmpdir string) (tmpfile string, err erro
defer body.Close() defer body.Close()
// Spool to local file. // Spool to local file.
f, err := ioutil.TempFile(tmpdir, "vgo-proxy-download-") f, err := ioutil.TempFile(tmpdir, "go-proxy-download-")
if err != nil { if err != nil {
return "", err return "", err
} }
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
...@@ -5,4 +5,6 @@ replace ( ...@@ -5,4 +5,6 @@ replace (
xyz v1.3.4 => my/xyz v1.3.4-me xyz v1.3.4 => my/xyz v1.3.4-me
xyz v1.4.5 => "/tmp/my dir" xyz v1.4.5 => "/tmp/my dir"
xyz v1.5.6 => my/xyz v1.5.6 xyz v1.5.6 => my/xyz v1.5.6
xyz => my/other/xyz v1.5.4
) )
...@@ -5,4 +5,6 @@ replace ( ...@@ -5,4 +5,6 @@ replace (
"xyz" v1.3.4 => "my/xyz" "v1.3.4-me" "xyz" v1.3.4 => "my/xyz" "v1.3.4-me"
xyz "v1.4.5" => "/tmp/my dir" xyz "v1.4.5" => "/tmp/my dir"
xyz v1.5.6 => my/xyz v1.5.6 xyz v1.5.6 => my/xyz v1.5.6
xyz => my/other/xyz v1.5.4
) )
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
...@@ -280,7 +280,7 @@ func Get(url string, options ...Option) error { ...@@ -280,7 +280,7 @@ func Get(url string, options ...Option) error {
return err return err
} }
var githubMessage = `vgo: 403 response from api.github.com var githubMessage = `go: 403 response from api.github.com
GitHub applies fairly small rate limits to unauthenticated users, and GitHub applies fairly small rate limits to unauthenticated users, and
you appear to be hitting them. To authenticate, please visit you appear to be hitting them. To authenticate, please visit
......
...@@ -327,8 +327,8 @@ func (b *Builder) AutoAction(mode, depMode BuildMode, p *load.Package) *Action { ...@@ -327,8 +327,8 @@ func (b *Builder) AutoAction(mode, depMode BuildMode, p *load.Package) *Action {
// depMode is the action (build or install) to use when building dependencies. // depMode is the action (build or install) to use when building dependencies.
// To turn package main into an executable, call b.Link instead. // To turn package main into an executable, call b.Link instead.
func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Action { func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Action {
if mode != ModeBuild && p.Internal.Local && p.Target == "" { if mode != ModeBuild && (p.Internal.Local || p.Module != nil) && p.Target == "" {
// Imported via local path. No permanent target. // Imported via local path or using modules. No permanent target.
mode = ModeBuild mode = ModeBuild
} }
if mode != ModeBuild && p.Name == "main" { if mode != ModeBuild && p.Name == "main" {
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Written by hand.
Test case for module at root of domain.
-- .mod --
module example.com
-- .info --
{"Version": "v1.0.0"}
-- x.go --
package x
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment