cmd/go: add support for concurrent backend compilation

It is disabled by default.
It can be enabled by setting the environment variable
GO19CONCURRENTCOMPILATION=1.

Benchmarking results are presented in a grid.
Columns are different values of c (compiler backend concurrency);
rows are different values of p (process concurrency).

'go build -a std cmd', a 4 core raspberry pi 3:

            c=1        c=2        c=4
StdCmd/p=1  504s ± 2%  413s ± 4%  367s ± 3%
StdCmd/p=2  314s ± 3%  266s ± 4%  267s ± 4%
StdCmd/p=4  254s ± 5%  241s ± 5%  238s ± 6%

'go build -a std cmd', an 8 core darwin/amd64 laptop:

            c=1         c=2         c=4         c=6         c=8
StdCmd/p=1  40.4s ± 7%  31.0s ± 1%  27.3s ± 1%  27.8s ± 0%  27.7s ± 0%
StdCmd/p=2  21.9s ± 1%  17.9s ± 1%  16.9s ± 1%  17.0s ± 1%  17.2s ± 0%
StdCmd/p=4  17.4s ± 2%  14.5s ± 2%  13.3s ± 2%  13.5s ± 2%  13.6s ± 2%
StdCmd/p=6  16.9s ± 1%  14.2s ± 2%  13.1s ± 2%  13.2s ± 2%  13.3s ± 2%
StdCmd/p=8  16.7s ± 2%  14.2s ± 2%  13.2s ± 3%  13.2s ± 2%  13.4s ± 2%

'go build -a std cmd', a 96 core arm64 server:

             c=1         c=2         c=4         c=6         c=8         c=16        c=32        c=64        c=96
StdCmd/p=1    173s ± 1%   133s ± 1%   114s ± 1%   109s ± 1%   106s ± 0%   106s ± 1%   107s ± 1%   110s ± 1%   113s ± 1%
StdCmd/p=2   94.2s ± 2%  71.5s ± 1%  61.7s ± 1%  58.7s ± 1%  57.5s ± 2%  56.9s ± 1%  58.0s ± 1%  59.6s ± 1%  61.0s ± 1%
StdCmd/p=4   74.1s ± 2%  53.5s ± 1%  43.7s ± 2%  40.5s ± 1%  39.2s ± 2%  38.9s ± 2%  39.5s ± 3%  40.3s ± 2%  40.8s ± 1%
StdCmd/p=6   69.3s ± 1%  50.2s ± 2%  40.3s ± 2%  37.3s ± 3%  36.0s ± 3%  35.3s ± 2%  36.0s ± 2%  36.8s ± 2%  37.5s ± 2%
StdCmd/p=8   66.1s ± 2%  47.7s ± 2%  38.6s ± 2%  35.7s ± 2%  34.4s ± 1%  33.6s ± 2%  34.2s ± 2%  34.6s ± 1%  35.0s ± 1%
StdCmd/p=16  63.4s ± 2%  45.3s ± 2%  36.3s ± 2%  33.3s ± 2%  32.0s ± 3%  31.6s ± 2%  32.1s ± 2%  32.5s ± 2%  32.7s ± 2%
StdCmd/p=32  62.2s ± 1%  44.2s ± 2%  35.3s ± 2%  32.4s ± 2%  31.2s ± 2%  30.9s ± 2%  31.1s ± 2%  31.7s ± 2%  32.0s ± 2%
StdCmd/p=64  62.2s ± 1%  44.3s ± 2%  35.4s ± 2%  32.4s ± 2%  31.2s ± 2%  30.9s ± 2%  31.2s ± 2%  31.8s ± 3%  32.2s ± 3%
StdCmd/p=96  62.2s ± 2%  44.4s ± 2%  35.3s ± 2%  32.3s ± 2%  31.1s ± 2%  30.9s ± 3%  31.3s ± 2%  31.7s ± 1%  32.1s ± 2%

benchjuju, an 8 core darwin/amd64 laptop:

               c=1         c=2         c=4         c=6         c=8
BuildJuju/p=1  55.3s ± 0%  46.3s ± 0%  41.9s ± 0%  41.4s ± 1%  41.3s ± 0%
BuildJuju/p=2  33.7s ± 1%  28.4s ± 1%  26.7s ± 1%  26.6s ± 1%  26.8s ± 1%
BuildJuju/p=4  24.7s ± 1%  22.3s ± 1%  21.4s ± 1%  21.7s ± 1%  21.8s ± 1%
BuildJuju/p=6  20.6s ± 1%  19.3s ± 2%  19.4s ± 1%  19.7s ± 1%  19.9s ± 1%
BuildJuju/p=8  20.6s ± 2%  19.5s ± 2%  19.3s ± 2%  19.6s ± 1%  19.8s ± 2%

Updates #15756

Change-Id: I8a56e88953071a05eee764002024c54cd888a56c
Reviewed-on: https://go-review.googlesource.com/41819
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

src/cmd/go/internal/work/build.go

@@ -144,6 +144,8 @@ See also: go install, go get, go clean.
 	`,
 }
 
+const concurrentGCBackendCompilationEnabledByDefault = false
+
 func init() {
 	// break init cycle
 	CmdBuild.Run = runBuild
@@ -2252,6 +2254,12 @@ func (gcToolchain) gc(b *Builder, p *load.Package, archive, obj string, asmhdr b
 	if asmhdr {
 		args = append(args, "-asmhdr", obj+"go_asm.h")
 	}
+
+	// Add -c=N to use concurrent backend compilation, if possible.
+	if c := gcBackendConcurrency(gcflags); c > 1 {
+		args = append(args, fmt.Sprintf("-c=%d", c))
+	}
+
 	for _, f := range gofiles {
 		args = append(args, mkAbs(p.Dir, f))
 	}
@@ -2260,6 +2268,77 @@ func (gcToolchain) gc(b *Builder, p *load.Package, archive, obj string, asmhdr b
 	return ofile, output, err
 }
 
+// gcBackendConcurrency returns the backend compiler concurrency level for a package compilation.
+func gcBackendConcurrency(gcflags []string) int {
+	// First, check whether we can use -c at all for this compilation.
+	canDashC := concurrentGCBackendCompilationEnabledByDefault
+
+	switch e := os.Getenv("GO19CONCURRENTCOMPILATION"); e {
+	case "0":
+		canDashC = false
+	case "1":
+		canDashC = true
+	case "":
+		// Not set. Use default.
+	default:
+		log.Fatalf("GO19CONCURRENTCOMPILATION must be 0, 1, or unset, got %q", e)
+	}
+
+	if os.Getenv("GOEXPERIMENT") != "" {
+		// Concurrent compilation is presumed incompatible with GOEXPERIMENTs.
+		canDashC = false
+	}
+
+CheckFlags:
+	for _, flag := range gcflags {
+		// Concurrent compilation is presumed incompatible with any gcflags,
+		// except for a small whitelist of commonly used flags.
+		// If the user knows better, they can manually add their own -c to the gcflags.
+		switch flag {
+		case "-N", "-l", "-S", "-B", "-C", "-I":
+			// OK
+		default:
+			canDashC = false
+			break CheckFlags
+		}
+	}
+
+	if !canDashC {
+		return 1
+	}
+
+	// Decide how many concurrent backend compilations to allow.
+	//
+	// If we allow too many, in theory we might end up with p concurrent processes,
+	// each with c concurrent backend compiles, all fighting over the same resources.
+	// However, in practice, that seems not to happen too much.
+	// Most build graphs are surprisingly serial, so p==1 for much of the build.
+	// Furthermore, concurrent backend compilation is only enabled for a part
+	// of the overall compiler execution, so c==1 for much of the build.
+	// So don't worry too much about that interaction for now.
+	//
+	// However, in practice, setting c above 4 tends not to help very much.
+	// See the analysis in CL 41192.
+	//
+	// TODO(josharian): attempt to detect whether this particular compilation
+	// is likely to be a bottleneck, e.g. when:
+	//   - it has no successor packages to compile (usually package main)
+	//   - all paths through the build graph pass through it
+	//   - critical path scheduling says it is high priority
+	// and in such a case, set c to runtime.NumCPU.
+	// We do this now when p==1.
+	if cfg.BuildP == 1 {
+		// No process parallelism. Max out c.
+		return runtime.NumCPU()
+	}
+	// Some process parallelism. Set c to min(4, numcpu).
+	c := 4
+	if ncpu := runtime.NumCPU(); ncpu < c {
+		c = ncpu
+	}
+	return c
+}
+
 func (gcToolchain) asm(b *Builder, p *load.Package, obj string, sfiles []string) ([]string, error) {
 	// Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files.
 	inc := filepath.Join(cfg.GOROOT, "pkg", "include")