cmd/link: mmap object data

This resurrects CL 121198, except that this time we map read-only.

In case that we need to apply relocations to the symbol's
content that is backed by read-only memory, we do our own copy-
on-write. This can happen if we failed to mmap the output file,
or we build for Wasm.

Memory profile for building k8s.io/kubernetes/cmd/kube-apiserver
on Linux/AMD64:

Old (before this sequence of CLs):
inuse_space 1598.75MB total
669.87MB 41.90% 41.90%   669.87MB 41.90%  cmd/link/internal/objfile.(*objReader).readSlices

New:
inuse_space 1280.45MB total
441.18MB 34.46% 34.46%   441.18MB 34.46%  cmd/link/internal/objfile.(*objReader).readSlices

Change-Id: I6b4d29d6eee9828089ea3120eb38c212db21330b
Reviewed-on: https://go-review.googlesource.com/c/go/+/170741
Run-TryBot: Cherry Zhang <cherryyz@google.com>
Reviewed-by: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/cmd/internal/bio/buf.go

@@ -7,6 +7,7 @@ package bio
 
 import (
 	"bufio"
+	"io"
 	"log"
 	"os"
 )
@@ -105,3 +106,26 @@ func (r *Reader) File() *os.File {
 func (w *Writer) File() *os.File {
 	return w.f
 }
+
+// Slice reads the next length bytes of r into a slice.
+//
+// This slice may be backed by mmap'ed memory. Currently, this memory
+// will never be unmapped. The second result reports whether the
+// backing memory is read-only.
+func (r *Reader) Slice(length uint64) ([]byte, bool, error) {
+	if length == 0 {
+		return []byte{}, false, nil
+	}
+
+	data, ok := r.sliceOS(length)
+	if ok {
+		return data, true, nil
+	}
+
+	data = make([]byte, length)
+	_, err := io.ReadFull(r, data)
+	if err != nil {
+		return nil, false, err
+	}
+	return data, false, nil
+}

src/cmd/internal/bio/buf_mmap.go

+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd
+
+package bio
+
+import (
+	"runtime"
+	"sync/atomic"
+	"syscall"
+)
+
+// mmapLimit is the maximum number of mmaped regions to create before
+// falling back to reading into a heap-allocated slice. This exists
+// because some operating systems place a limit on the number of
+// distinct mapped regions per process. As of this writing:
+//
+//  Darwin    unlimited
+//  DragonFly   1000000 (vm.max_proc_mmap)
+//  FreeBSD   unlimited
+//  Linux         65530 (vm.max_map_count) // TODO: query /proc/sys/vm/max_map_count?
+//  NetBSD    unlimited
+//  OpenBSD   unlimited
+var mmapLimit int32 = 1<<31 - 1
+
+func init() {
+	// Linux is the only practically concerning OS.
+	if runtime.GOOS == "linux" {
+		mmapLimit = 30000
+	}
+}
+
+func (r *Reader) sliceOS(length uint64) ([]byte, bool) {
+	// For small slices, don't bother with the overhead of a
+	// mapping, especially since we have no way to unmap it.
+	const threshold = 16 << 10
+	if length < threshold {
+		return nil, false
+	}
+
+	// Have we reached the mmap limit?
+	if atomic.AddInt32(&mmapLimit, -1) < 0 {
+		atomic.AddInt32(&mmapLimit, 1)
+		return nil, false
+	}
+
+	// Page-align the offset.
+	off := r.Offset()
+	align := syscall.Getpagesize()
+	aoff := off &^ int64(align-1)
+
+	data, err := syscall.Mmap(int(r.f.Fd()), aoff, int(length+uint64(off-aoff)), syscall.PROT_READ, syscall.MAP_SHARED|syscall.MAP_FILE)
+	if err != nil {
+		return nil, false
+	}
+
+	data = data[off-aoff:]
+	r.Seek(int64(length), 1)
+	return data, true
+}

src/cmd/internal/bio/buf_nommap.go

+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd
+
+package bio
+
+func (r *Reader) sliceOS(length uint64) ([]byte, bool) {
+	return nil, false
+}

src/cmd/link/internal/ld/data.go

@@ -127,6 +127,15 @@ func trampoline(ctxt *Link, s *sym.Symbol) {
 // This is a performance-critical function for the linker; be careful
 // to avoid introducing unnecessary allocations in the main loop.
 func relocsym(ctxt *Link, s *sym.Symbol) {
+	if len(s.R) == 0 {
+		return
+	}
+	if s.Attr.ReadOnly() {
+		// The symbol's content is backed by read-only memory.
+		// Copy it to writable memory to apply relocations.
+		s.P = append([]byte(nil), s.P...)
+		s.Attr.Set(sym.AttrReadOnly, false)
+	}
 	for ri := int32(0); ri < int32(len(s.R)); ri++ {
 		r := &s.R[ri]
 		if r.Done {
@@ -2384,17 +2393,21 @@ func compressSyms(ctxt *Link, syms []*sym.Symbol) []byte {
 	if err != nil {
 		log.Fatalf("NewWriterLevel failed: %s", err)
 	}
-	for _, sym := range syms {
-		// sym.P may be read-only. Apply relocations in a
+	for _, s := range syms {
+		// s.P may be read-only. Apply relocations in a
 		// temporary buffer, and immediately write it out.
-		oldP := sym.P
-		ctxt.relocbuf = append(ctxt.relocbuf[:0], sym.P...)
-		sym.P = ctxt.relocbuf
-		relocsym(ctxt, sym)
-		if _, err := z.Write(sym.P); err != nil {
+		oldP := s.P
+		wasReadOnly := s.Attr.ReadOnly()
+		if len(s.R) != 0 && wasReadOnly {
+			ctxt.relocbuf = append(ctxt.relocbuf[:0], s.P...)
+			s.P = ctxt.relocbuf
+			s.Attr.Set(sym.AttrReadOnly, false)
+		}
+		relocsym(ctxt, s)
+		if _, err := z.Write(s.P); err != nil {
 			log.Fatalf("compression failed: %s", err)
 		}
-		for i := sym.Size - int64(len(sym.P)); i > 0; {
+		for i := s.Size - int64(len(s.P)); i > 0; {
 			b := zeros[:]
 			if i < int64(len(b)) {
 				b = b[:i]
@@ -2405,13 +2418,15 @@ func compressSyms(ctxt *Link, syms []*sym.Symbol) []byte {
 			}
 			i -= int64(n)
 		}
-		// Restore sym.P, for 1. not holding temp buffer live
-		// unnecessarily, 2. if compression is not beneficial,
-		// we'll go back to use the uncompressed contents, in
-		// which case we still need sym.P.
-		sym.P = oldP
-		for i := range sym.R {
-			sym.R[i].Done = false
+		// Restore s.P if a temporary buffer was used. If compression
+		// is not beneficial, we'll go back to use the uncompressed
+		// contents, in which case we still need s.P.
+		if len(s.R) != 0 && wasReadOnly {
+			s.P = oldP
+			s.Attr.Set(sym.AttrReadOnly, wasReadOnly)
+			for i := range s.R {
+				s.R[i].Done = false
+			}
 		}
 	}
 	if err := z.Close(); err != nil {

src/cmd/link/internal/ld/outbuf.go

@@ -158,6 +158,7 @@ func (out *OutBuf) WriteSym(s *sym.Symbol) {
 		start := out.off
 		out.Write(s.P)
 		s.P = out.buf[start:out.off]
+		s.Attr.Set(sym.AttrReadOnly, false)
 	} else {
 		out.Write(s.P)
 	}

src/cmd/link/internal/objfile/objfile.go

@@ -34,7 +34,7 @@ var emptyPkg = []byte(`"".`)
 
 // objReader reads Go object files.
 type objReader struct {
-	rd              *bufio.Reader
+	rd              *bio.Reader
 	arch            *sys.Arch
 	syms            *sym.Symbols
 	lib             *sym.Library
@@ -43,6 +43,7 @@ type objReader struct {
 	localSymVersion int
 	flags           int
 	strictDupMsgs   int
+	dataSize        int
 
 	// rdBuf is used by readString and readSymName as scratch for reading strings.
 	rdBuf []byte
@@ -56,6 +57,8 @@ type objReader struct {
 	funcdata    []*sym.Symbol
 	funcdataoff []int64
 	file        []*sym.Symbol
+
+	dataReadOnly bool // whether data is backed by read-only memory
 }
 
 // Flags to enable optional behavior during object loading/reading.
@@ -76,7 +79,7 @@ const (
 func Load(arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *sym.Library, length int64, pn string, flags int) int {
 	start := f.Offset()
 	r := &objReader{
-		rd:              f.Reader,
+		rd:              f,
 		lib:             lib,
 		arch:            arch,
 		syms:            syms,
@@ -133,7 +136,10 @@ func (r *objReader) loadObjFile() {
 	r.readSlices()
 
 	// Data section
-	r.readFull(r.data)
+	r.data, r.dataReadOnly, err = r.rd.Slice(uint64(r.dataSize))
+	if err != nil {
+		log.Fatalf("%s: error reading %s", r.pn, err)
+	}
 
 	// Defined symbols
 	for {
@@ -156,9 +162,8 @@ func (r *objReader) loadObjFile() {
 }
 
 func (r *objReader) readSlices() {
+	r.dataSize = r.readInt()
 	n := r.readInt()
-	r.data = make([]byte, n)
-	n = r.readInt()
 	r.reloc = make([]sym.Reloc, n)
 	n = r.readInt()
 	r.pcdata = make([]sym.Pcdata, n)
@@ -249,6 +254,7 @@ overwrite:
 		dup.Gotype = typ
 	}
 	s.P = data
+	s.Attr.Set(sym.AttrReadOnly, r.dataReadOnly)
 	if nreloc > 0 {
 		s.R = r.reloc[:nreloc:nreloc]
 		if !isdup {

src/cmd/link/internal/sym/attribute.go

@@ -78,7 +78,10 @@ const (
 	// AttrTopFrame means that the function is an entry point and unwinders
 	// should stop when they hit this function.
 	AttrTopFrame
-	// 18 attributes defined so far.
+	// AttrReadOnly indicates whether the symbol's content (Symbol.P) is backed by
+	// read-only memory.
+	AttrReadOnly
+	// 19 attributes defined so far.
 )
 
 func (a Attribute) DuplicateOK() bool      { return a&AttrDuplicateOK != 0 }
@@ -99,6 +102,7 @@ func (a Attribute) VisibilityHidden() bool { return a&AttrVisibilityHidden != 0
 func (a Attribute) SubSymbol() bool        { return a&AttrSubSymbol != 0 }
 func (a Attribute) Container() bool        { return a&AttrContainer != 0 }
 func (a Attribute) TopFrame() bool         { return a&AttrTopFrame != 0 }
+func (a Attribute) ReadOnly() bool         { return a&AttrReadOnly != 0 }
 
 func (a Attribute) CgoExport() bool {
 	return a.CgoExportDynamic() || a.CgoExportStatic()