cmd/dist: read dependencies from source files

I do not remember why we require deps.go to have a hard-coded
copy of the dependency information for cmd/go, when we can
read it from the source files instead. The answer probably involves
cmd/dist once being a C program.

In any event, stop doing that, which will eliminate the builder-only
failures in the builder-only deps test.

Change-Id: I0abd384c47401940ca08427b5be544812edcbe7f
Reviewed-on: https://go-review.googlesource.com/76021
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: David Crawshaw <crawshaw@golang.org>

src/cmd/dist/build.go

@@ -533,15 +533,35 @@ var gentab = []struct {
 // installed maps from a dir name (as given to install) to a chan
 // closed when the dir's package is installed.
 var installed = make(map[string]chan struct{})
+var installedMu sync.Mutex
 
-// install installs the library, package, or binary associated with dir,
-// which is relative to $GOROOT/src.
 func install(dir string) {
-	if ch, ok := installed[dir]; ok {
-		defer close(ch)
+	<-startInstall(dir)
+}
+
+func startInstall(dir string) chan struct{} {
+	installedMu.Lock()
+	ch := installed[dir]
+	if ch == nil {
+		ch = make(chan struct{})
+		installed[dir] = ch
+		go runInstall(dir, ch)
+	}
+	installedMu.Unlock()
+	return ch
+}
+
+// runInstall installs the library, package, or binary associated with dir,
+// which is relative to $GOROOT/src.
+func runInstall(dir string, ch chan struct{}) {
+	if dir == "net" || dir == "os/user" || dir == "crypto/x509" {
+		fatalf("go_bootstrap cannot depend on cgo package %s", dir)
 	}
-	for _, dep := range builddeps[dir] {
-		<-installed[dep]
+
+	defer close(ch)
+
+	if dir == "unsafe" {
+		return
 	}
 
 	if vflag > 0 {
@@ -660,7 +680,7 @@ func install(dir string) {
 	}
 
 	// For package runtime, copy some files into the work space.
-	if dir == "runtime" || strings.HasPrefix(dir, "runtime/internal/") {
+	if dir == "runtime" {
 		xmkdirall(pathf("%s/pkg/include", goroot))
 		// For use by assembly and C files.
 		copyfile(pathf("%s/pkg/include/textflag.h", goroot),
@@ -700,6 +720,18 @@ func install(dir string) {
 	built:
 	}
 
+	// Make sure dependencies are installed.
+	var deps []string
+	for _, p := range gofiles {
+		deps = append(deps, readimports(p)...)
+	}
+	for _, dir1 := range deps {
+		startInstall(dir1)
+	}
+	for _, dir1 := range deps {
+		install(dir1)
+	}
+
 	if goos != gohostos || goarch != gohostarch {
 		// We've generated the right files; the go command can do the build.
 		if vflag > 1 {
@@ -906,28 +938,21 @@ func dopack(dst, src string, extra []string) {
 	writefile(bdst.String(), dst, 0)
 }
 
-// builddeps records the build dependencies for the 'go bootstrap' command.
-// It is a map[string][]string and generated by mkdeps.bash into deps.go.
-
-// buildlist is the list of directories being built, sorted by name.
-var buildlist = makeBuildlist()
-
-func makeBuildlist() []string {
-	var all []string
-	for dir := range builddeps {
-		all = append(all, dir)
-	}
-	sort.Strings(all)
-	return all
-}
-
 var runtimegen = []string{
 	"zaexperiment.h",
 	"zversion.go",
 }
 
+// cleanlist is a list of packages with generated files and commands.
+var cleanlist = []string{
+	"runtime/internal/sys",
+	"cmd/cgo",
+	"cmd/go/internal/cfg",
+	"go/build",
+}
+
 func clean() {
-	for _, name := range buildlist {
+	for _, name := range cleanlist {
 		path := pathf("%s/src/%s", goroot, name)
 		// Remove generated files.
 		for _, elem := range xreaddir(path) {
@@ -1110,13 +1135,8 @@ func cmdbootstrap() {
 
 	timelog("build", "go_bootstrap")
 	xprintf("Building Go bootstrap cmd/go (go_bootstrap) using Go toolchain1.\n")
-	for _, dir := range buildlist {
-		installed[dir] = make(chan struct{})
-	}
-	for _, dir := range buildlist {
-		go install(dir)
-	}
-	<-installed["cmd/go"]
+	install("runtime") // dependency not visible in sources; also sets up textflag.h
+	install("cmd/go")
 	if vflag > 0 {
 		xprintf("\n")
 	}

src/cmd/dist/deps_test.go

-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main_test
-
-import (
-	"bytes"
-	"internal/testenv"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"strings"
-	"testing"
-)
-
-func TestDeps(t *testing.T) {
-	if testing.Short() && testenv.Builder() == "" {
-		t.Skip("skipping in short mode")
-	}
-
-	current, err := ioutil.ReadFile("deps.go")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	bash, err := exec.LookPath("bash")
-	if err != nil {
-		t.Skipf("skipping because bash not found: %v", err)
-	}
-
-	outf, err := ioutil.TempFile("", "dist-deps-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	outf.Close()
-	outname := outf.Name()
-	defer os.Remove(outname)
-
-	out, err := exec.Command(bash, "mkdeps.bash", outname).CombinedOutput()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Logf("%s", out)
-
-	updated, err := ioutil.ReadFile(outname)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !bytes.Equal(current, updated) {
-		// Very simple minded diff.
-		t.Log("-current +generated")
-		clines := strings.Split(string(current), "\n")
-		for i, line := range clines {
-			clines[i] = strings.Join(strings.Fields(line), " ")
-		}
-		ulines := strings.Split(string(updated), "\n")
-		for i, line := range ulines {
-			ulines[i] = strings.Join(strings.Fields(line), " ")
-		}
-		for len(clines) > 0 {
-			cl := clines[0]
-			switch {
-			case len(ulines) == 0:
-				t.Logf("-%s", cl)
-				clines = clines[1:]
-			case cl == ulines[0]:
-				clines = clines[1:]
-				ulines = ulines[1:]
-			case pkg(cl) == pkg(ulines[0]):
-				t.Logf("-%s", cl)
-				t.Logf("+%s", ulines[0])
-				clines = clines[1:]
-				ulines = ulines[1:]
-			case pkg(cl) < pkg(ulines[0]):
-				t.Logf("-%s", cl)
-				clines = clines[1:]
-			default:
-				cp := pkg(cl)
-				for len(ulines) > 0 && pkg(ulines[0]) < cp {
-					t.Logf("+%s", ulines[0])
-					ulines = ulines[1:]
-				}
-			}
-		}
-
-		t.Error("cmd/dist/deps.go is out of date; run cmd/dist/mkdeps.bash")
-	}
-}
-
-// pkg returns the package of a line in deps.go.
-func pkg(line string) string {
-	i := strings.Index(line, `"`)
-	if i < 0 {
-		return ""
-	}
-	line = line[i+1:]
-	i = strings.Index(line, `"`)
-	if i < 0 {
-		return ""
-	}
-	return line[:i]
-}

src/cmd/dist/imports.go

+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file is forked from go/build/read.go.
+// (cmd/dist must not import go/build because we do not want it to be
+// sensitive to the specific version of go/build present in $GOROOT_BOOTSTRAP.)
+
+package main
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+)
+
+type importReader struct {
+	b    *bufio.Reader
+	buf  []byte
+	peek byte
+	err  error
+	eof  bool
+	nerr int
+}
+
+func isIdent(c byte) bool {
+	return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '_' || c >= utf8.RuneSelf
+}
+
+var (
+	errSyntax = errors.New("syntax error")
+	errNUL    = errors.New("unexpected NUL in input")
+)
+
+// syntaxError records a syntax error, but only if an I/O error has not already been recorded.
+func (r *importReader) syntaxError() {
+	if r.err == nil {
+		r.err = errSyntax
+	}
+}
+
+// readByte reads the next byte from the input, saves it in buf, and returns it.
+// If an error occurs, readByte records the error in r.err and returns 0.
+func (r *importReader) readByte() byte {
+	c, err := r.b.ReadByte()
+	if err == nil {
+		r.buf = append(r.buf, c)
+		if c == 0 {
+			err = errNUL
+		}
+	}
+	if err != nil {
+		if err == io.EOF {
+			r.eof = true
+		} else if r.err == nil {
+			r.err = err
+		}
+		c = 0
+	}
+	return c
+}
+
+// peekByte returns the next byte from the input reader but does not advance beyond it.
+// If skipSpace is set, peekByte skips leading spaces and comments.
+func (r *importReader) peekByte(skipSpace bool) byte {
+	if r.err != nil {
+		if r.nerr++; r.nerr > 10000 {
+			panic("go/build: import reader looping")
+		}
+		return 0
+	}
+
+	// Use r.peek as first input byte.
+	// Don't just return r.peek here: it might have been left by peekByte(false)
+	// and this might be peekByte(true).
+	c := r.peek
+	if c == 0 {
+		c = r.readByte()
+	}
+	for r.err == nil && !r.eof {
+		if skipSpace {
+			// For the purposes of this reader, semicolons are never necessary to
+			// understand the input and are treated as spaces.
+			switch c {
+			case ' ', '\f', '\t', '\r', '\n', ';':
+				c = r.readByte()
+				continue
+
+			case '/':
+				c = r.readByte()
+				if c == '/' {
+					for c != '\n' && r.err == nil && !r.eof {
+						c = r.readByte()
+					}
+				} else if c == '*' {
+					var c1 byte
+					for (c != '*' || c1 != '/') && r.err == nil {
+						if r.eof {
+							r.syntaxError()
+						}
+						c, c1 = c1, r.readByte()
+					}
+				} else {
+					r.syntaxError()
+				}
+				c = r.readByte()
+				continue
+			}
+		}
+		break
+	}
+	r.peek = c
+	return r.peek
+}
+
+// nextByte is like peekByte but advances beyond the returned byte.
+func (r *importReader) nextByte(skipSpace bool) byte {
+	c := r.peekByte(skipSpace)
+	r.peek = 0
+	return c
+}
+
+// readKeyword reads the given keyword from the input.
+// If the keyword is not present, readKeyword records a syntax error.
+func (r *importReader) readKeyword(kw string) {
+	r.peekByte(true)
+	for i := 0; i < len(kw); i++ {
+		if r.nextByte(false) != kw[i] {
+			r.syntaxError()
+			return
+		}
+	}
+	if isIdent(r.peekByte(false)) {
+		r.syntaxError()
+	}
+}
+
+// readIdent reads an identifier from the input.
+// If an identifier is not present, readIdent records a syntax error.
+func (r *importReader) readIdent() {
+	c := r.peekByte(true)
+	if !isIdent(c) {
+		r.syntaxError()
+		return
+	}
+	for isIdent(r.peekByte(false)) {
+		r.peek = 0
+	}
+}
+
+// readString reads a quoted string literal from the input.
+// If an identifier is not present, readString records a syntax error.
+func (r *importReader) readString(save *[]string) {
+	switch r.nextByte(true) {
+	case '`':
+		start := len(r.buf) - 1
+		for r.err == nil {
+			if r.nextByte(false) == '`' {
+				if save != nil {
+					*save = append(*save, string(r.buf[start:]))
+				}
+				break
+			}
+			if r.eof {
+				r.syntaxError()
+			}
+		}
+	case '"':
+		start := len(r.buf) - 1
+		for r.err == nil {
+			c := r.nextByte(false)
+			if c == '"' {
+				if save != nil {
+					*save = append(*save, string(r.buf[start:]))
+				}
+				break
+			}
+			if r.eof || c == '\n' {
+				r.syntaxError()
+			}
+			if c == '\\' {
+				r.nextByte(false)
+			}
+		}
+	default:
+		r.syntaxError()
+	}
+}
+
+// readImport reads an import clause - optional identifier followed by quoted string -
+// from the input.
+func (r *importReader) readImport(imports *[]string) {
+	c := r.peekByte(true)
+	if c == '.' {
+		r.peek = 0
+	} else if isIdent(c) {
+		r.readIdent()
+	}
+	r.readString(imports)
+}
+
+// readComments is like ioutil.ReadAll, except that it only reads the leading
+// block of comments in the file.
+func readComments(f io.Reader) ([]byte, error) {
+	r := &importReader{b: bufio.NewReader(f)}
+	r.peekByte(true)
+	if r.err == nil && !r.eof {
+		// Didn't reach EOF, so must have found a non-space byte. Remove it.
+		r.buf = r.buf[:len(r.buf)-1]
+	}
+	return r.buf, r.err
+}
+
+// readimports returns the imports found in the named file.
+func readimports(file string) []string {
+	var imports []string
+	r := &importReader{b: bufio.NewReader(strings.NewReader(readfile(file)))}
+	r.readKeyword("package")
+	r.readIdent()
+	for r.peekByte(true) == 'i' {
+		r.readKeyword("import")
+		if r.peekByte(true) == '(' {
+			r.nextByte(false)
+			for r.peekByte(true) != ')' && r.err == nil {
+				r.readImport(&imports)
+			}
+			r.nextByte(false)
+		} else {
+			r.readImport(&imports)
+		}
+	}
+
+	for i := range imports {
+		unquoted, err := strconv.Unquote(imports[i])
+		if err != nil {
+			fatalf("reading imports from %s: %v", file, err)
+		}
+		imports[i] = unquoted
+	}
+
+	return imports
+}

src/cmd/dist/mkdeps.bash

-#!/bin/bash
-# Copyright 2015 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-# This script regenerates deps.go.
-# The script used to do all the work, but now a Go program does.
-# The script has been preserved so that people who learned to type
-# ./mkdeps.bash don't have to relearn a new method.
-# It's fine to run "go run mkdeps.go" directly instead.
-
-set -e
-go run mkdeps.go -- "$@"
-exit 0

src/cmd/dist/mkdeps.go

-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This program generates deps.go.
-// Run as "go run mkdeps.go" or, to redirect the output, "go run mkdeps.go x.txt".
-
-// +build ignore
-
-package main
-
-import (
-	"bytes"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"os/exec"
-	"sort"
-	"strings"
-)
-
-// We need to test enough GOOS/GOARCH combinations
-// to find all the package dependencies of cmd/go on all systems.
-var targetList = strings.Fields(`
-	linux/386
-	linux/amd64
-	windows/amd64
-`)
-
-func usage() {
-	fmt.Fprintf(os.Stderr, "usage: mkdeps [deps.go]\n")
-	os.Exit(2)
-}
-
-func main() {
-	log.SetPrefix("mkdeps: ")
-	log.SetFlags(0)
-	flag.Usage = usage
-	flag.Parse()
-	if flag.NArg() > 1 {
-		usage()
-	}
-
-	outfile := "deps.go"
-	if flag.NArg() == 1 {
-		outfile = flag.Arg(0)
-	}
-
-	_, deps := importsAndDepsOf("cmd/go")
-	all := deps["cmd/go"]
-	all = append(all, "cmd/go")
-	imports, deps := importsAndDepsOf(all...)
-
-	// Sort topologically, then by import path.
-	var topo []string
-	walked := make(map[string]bool)
-	var walk func(string)
-	walk = func(p string) {
-		if walked[p] {
-			return
-		}
-		walked[p] = true
-		sort.Strings(deps[p])
-		for _, d := range deps[p] {
-			walk(d)
-		}
-		topo = append(topo, p)
-	}
-	walk("cmd/go")
-
-	// We're only going to print imports, not deps,
-	// in hopes of making deps.go intelligible to people
-	// who need to debug it or attempt to resolve merge conflicts.
-	// For the most part, deps is just the transitive closure of imports,
-	// but sometimes there are implicit deps supplied by the go command
-	// that are not derivable from imports.
-	// Find those (if any) and copy them explicitly into imports.
-	for _, p := range topo {
-		for _, dp := range deps[p] {
-			found := false
-			for _, ip := range imports[p] {
-				if dp == ip || inList(deps[ip], dp) {
-					found = true
-					break
-				}
-			}
-			if !found {
-				imports[p] = append(imports[p], dp)
-			}
-		}
-		sort.Strings(imports[p])
-	}
-	sort.Strings(all)
-
-	// Print table.
-	var buf bytes.Buffer
-	fmt.Fprintf(&buf, "// Code generated by mkdeps.bash; DO NOT EDIT.\n\n")
-	fmt.Fprintf(&buf, "package main\n\n")
-	fmt.Fprintf(&buf, "var builddeps = map[string][]string{\n")
-	for _, p := range all {
-		if p == "unsafe" { // unsafe should not be built
-			continue
-		}
-		// We're printing a multiline format here to make the output more
-		// intelligible both to people and to merge tools.
-		// We put the name of the parent package as a comment on every line
-		// to keep a merge tool from applying the diff for one package
-		// to the dependency list for a different package.
-		// The extra blank line at the start stops any attempt by gofmt at
-		// lining up the slice literals from different packages,
-		// even if they are empty slices (on a single line with the key).
-		fmt.Fprintf(&buf, "\n\t%q: {\n", p)
-		for _, d := range imports[p] {
-			if d != "unsafe" {
-				fmt.Fprintf(&buf, "\t\t%q, // %s\n", d, p)
-			}
-		}
-		fmt.Fprintf(&buf, "\t},\n")
-	}
-	fmt.Fprintf(&buf, "\n}\n")
-
-	// Run the installed gofmt instead of using go/format,
-	// because, on the off chance they disagree,
-	// the installed gofmt binary is by definition the correct one.
-	cmd := exec.Command("gofmt")
-	cmd.Stdin = &buf
-	var out bytes.Buffer
-	cmd.Stdout = &out
-	if err := cmd.Run(); err != nil {
-		log.Fatalf("gofmt: %v", err)
-	}
-
-	if err := ioutil.WriteFile(outfile, out.Bytes(), 0666); err != nil {
-		log.Fatal(err)
-	}
-}
-
-func inList(xs []string, s string) bool {
-	for _, x := range xs {
-		if x == s {
-			return true
-		}
-	}
-	return false
-}
-
-// importsAndDepsOf returns two maps, one giving the imports for each package in pkgs,
-// and one giving the dependencies for each package in pkgs.
-// Both the keys and the entries in the value slices are full import paths.
-func importsAndDepsOf(pkgs ...string) (map[string][]string, map[string][]string) {
-	imports := make(map[string][]string)
-	deps := make(map[string][]string)
-	for _, target := range targetList {
-		args := []string{"list", "-tags", "cmd_go_bootstrap", "-f", "{{range .Imports}}import {{$.ImportPath}} {{.}}\n{{end}}{{range .Deps}}dep {{$.ImportPath}} {{.}}\n{{end}}"}
-		args = append(args, pkgs...)
-		cmd := exec.Command("go", args...)
-		t := strings.Split(target, "/")
-		cmd.Env = append(os.Environ(), "GOOS="+t[0], "GOARCH="+t[1])
-		var stderr bytes.Buffer
-		cmd.Stderr = &stderr
-		out, err := cmd.Output()
-		if err != nil && !strings.Contains(stderr.String(), "build constraints exclude all Go files") {
-			log.Fatalf("GOOS=%s GOARCH=%s go list: %v\n%s\n%s", t[0], t[1], err, stderr.Bytes(), out)
-		}
-		helped := false
-		for _, line := range strings.Split(string(out), "\n") {
-			f := strings.Fields(line)
-			if len(f) != 3 {
-				continue
-			}
-			if f[0] == "import" && !inList(imports[f[1]], f[2]) {
-				helped = true
-				imports[f[1]] = append(imports[f[1]], f[2])
-			}
-			if f[0] == "dep" && !inList(deps[f[1]], f[2]) {
-				helped = true
-				deps[f[1]] = append(deps[f[1]], f[2])
-			}
-		}
-		if !helped {
-			fmt.Fprintf(os.Stderr, "mkdeps: note: %s did not contribute any new dependencies\n", target)
-		}
-	}
-	return imports, deps
-}