[dev.ssa] Merge branch 'master' into dev.ssa

Change-Id: Iabc80b6e0734efbd234d998271e110d2eaad41dd

.gitignore

@@ -26,6 +26,7 @@ misc/cgo/stdio/run.out
 misc/cgo/testso/main
 src/cmd/cgo/zdefaultcc.go
 src/cmd/go/zdefaultcc.go
+src/cmd/go/zosarch.go
 src/cmd/internal/obj/zbootstrap.go
 src/go/build/zcgo.go
 src/go/doc/headscan

api/next.txt

@@ -17,7 +17,15 @@ pkg context, type Context interface, Err() error
 pkg context, type Context interface, Value(interface{}) interface{}
 pkg context, var Canceled error
 pkg context, var DeadlineExceeded error
+pkg crypto/tls, const RenegotiateFreelyAsClient = 2
+pkg crypto/tls, const RenegotiateFreelyAsClient RenegotiationSupport
+pkg crypto/tls, const RenegotiateNever = 0
+pkg crypto/tls, const RenegotiateNever RenegotiationSupport
+pkg crypto/tls, const RenegotiateOnceAsClient = 1
+pkg crypto/tls, const RenegotiateOnceAsClient RenegotiationSupport
 pkg crypto/tls, type Config struct, DynamicRecordSizingDisabled bool
+pkg crypto/tls, type Config struct, Renegotiation RenegotiationSupport
+pkg crypto/tls, type RenegotiationSupport int
 pkg crypto/x509, func SystemCertPool() (*CertPool, error)
 pkg crypto/x509, type SystemRootsError struct, Err error
 pkg debug/dwarf, method (*Data) Ranges(*Entry) ([][2]uint64, error)
@@ -147,8 +155,9 @@ pkg debug/elf, const R_390_TLS_TPOFF R_390
 pkg debug/elf, method (R_390) GoString() string
 pkg debug/elf, method (R_390) String() string
 pkg debug/elf, type R_390 int
-pkg encoding/json, method (*Encoder) DisableHTMLEscaping()
-pkg encoding/json, method (*Encoder) Indent(string, string)
+pkg encoding/json, method (*Encoder) SetEscapeHTML(bool)
+pkg encoding/json, method (*Encoder) SetIndent(string, string)
+pkg go/build, type Package struct, BinaryOnly bool
 pkg go/build, type Package struct, CgoFFLAGS []string
 pkg go/build, type Package struct, FFiles []string
 pkg go/doc, type Example struct, Unordered bool
@@ -158,22 +167,54 @@ pkg io, const SeekEnd = 2
 pkg io, const SeekEnd ideal-int
 pkg io, const SeekStart = 0
 pkg io, const SeekStart ideal-int
-pkg io, type SizedReaderAt interface { ReadAt, Size }
-pkg io, type SizedReaderAt interface, ReadAt([]uint8, int64) (int, error)
-pkg io, type SizedReaderAt interface, Size() int64
 pkg math/big, method (*Float) GobDecode([]uint8) error
 pkg math/big, method (*Float) GobEncode() ([]uint8, error)
 pkg net, method (*Dialer) DialContext(context.Context, string, string) (Conn, error)
-pkg net, type IPNet struct, Zone string
 pkg net/http, method (*Request) Context() context.Context
 pkg net/http, method (*Request) WithContext(context.Context) *Request
+pkg net/http, type Request struct, Response *Response
+pkg net/http, type Response struct, Uncompressed bool
 pkg net/http, type Transport struct, Dialer *net.Dialer
+pkg net/http, type Transport struct, IdleConnTimeout time.Duration
+pkg net/http, type Transport struct, MaxIdleConns int
 pkg net/http, type Transport struct, MaxResponseHeaderBytes int64
+pkg net/http, var ErrUseLastResponse error
+pkg net/http, var LocalAddrContextKey *contextKey
 pkg net/http, var ServerContextKey *contextKey
 pkg net/http/cgi, type Handler struct, Stderr io.Writer
 pkg net/http/httptest, func NewRequest(string, string, io.Reader) *http.Request
-pkg net/http/httptest, method (*ResponseRecorder) Trailers() http.Header
+pkg net/http/httptest, method (*ResponseRecorder) Result() *http.Response
+pkg net/http/httptrace, func ContextClientTrace(context.Context) *ClientTrace
+pkg net/http/httptrace, func WithClientTrace(context.Context, *ClientTrace) context.Context
+pkg net/http/httptrace, type ClientTrace struct
+pkg net/http/httptrace, type ClientTrace struct, ConnectDone func(string, string, error)
+pkg net/http/httptrace, type ClientTrace struct, ConnectStart func(string, string)
+pkg net/http/httptrace, type ClientTrace struct, DNSDone func(DNSDoneInfo)
+pkg net/http/httptrace, type ClientTrace struct, DNSStart func(DNSStartInfo)
+pkg net/http/httptrace, type ClientTrace struct, GetConn func(string)
+pkg net/http/httptrace, type ClientTrace struct, Got100Continue func()
+pkg net/http/httptrace, type ClientTrace struct, GotConn func(GotConnInfo)
+pkg net/http/httptrace, type ClientTrace struct, GotFirstResponseByte func()
+pkg net/http/httptrace, type ClientTrace struct, PutIdleConn func(error)
+pkg net/http/httptrace, type ClientTrace struct, Wait100Continue func()
+pkg net/http/httptrace, type ClientTrace struct, WroteHeaders func()
+pkg net/http/httptrace, type ClientTrace struct, WroteRequest func(WroteRequestInfo)
+pkg net/http/httptrace, type DNSDoneInfo struct
+pkg net/http/httptrace, type DNSDoneInfo struct, Addrs []net.IPAddr
+pkg net/http/httptrace, type DNSDoneInfo struct, Coalesced bool
+pkg net/http/httptrace, type DNSDoneInfo struct, Err error
+pkg net/http/httptrace, type DNSStartInfo struct
+pkg net/http/httptrace, type DNSStartInfo struct, Host string
+pkg net/http/httptrace, type GotConnInfo struct
+pkg net/http/httptrace, type GotConnInfo struct, Conn net.Conn
+pkg net/http/httptrace, type GotConnInfo struct, IdleTime time.Duration
+pkg net/http/httptrace, type GotConnInfo struct, Reused bool
+pkg net/http/httptrace, type GotConnInfo struct, WasIdle bool
+pkg net/http/httptrace, type WroteRequestInfo struct
+pkg net/http/httptrace, type WroteRequestInfo struct, Err error
 pkg net/url, type URL struct, ForceQuery bool
+pkg os, method (*File) Size() (int64, error)
+pkg os/exec, func CommandContext(context.Context, string, ...string) *Cmd
 pkg os/user, func LookupGroup(string) (*Group, error)
 pkg os/user, func LookupGroupId(string) (*Group, error)
 pkg os/user, method (*User) GroupIds() ([]string, error)
@@ -187,6 +228,7 @@ pkg os/user, type UnknownGroupIdError string
 pkg reflect, func StructOf([]StructField) Type
 pkg reflect, method (StructTag) Lookup(string) (string, bool)
 pkg runtime, func CallersFrames([]uintptr) *Frames
+pkg runtime, func KeepAlive(interface{})
 pkg runtime, func SetCgoTraceback(int, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer)
 pkg runtime, method (*Frames) Next() (Frame, bool)
 pkg runtime, type Frame struct
@@ -198,6 +240,12 @@ pkg runtime, type Frame struct, Line int
 pkg runtime, type Frame struct, PC uintptr
 pkg runtime, type Frames struct
 pkg strings, method (*Reader) Reset(string)
+pkg syscall (linux-386), type SysProcAttr struct, Unshare uintptr
+pkg syscall (linux-386-cgo), type SysProcAttr struct, Unshare uintptr
+pkg syscall (linux-amd64), type SysProcAttr struct, Unshare uintptr
+pkg syscall (linux-amd64-cgo), type SysProcAttr struct, Unshare uintptr
+pkg syscall (linux-arm), type SysProcAttr struct, Unshare uintptr
+pkg syscall (linux-arm-cgo), type SysProcAttr struct, Unshare uintptr
 pkg testing, method (*B) Run(string, func(*B)) bool
 pkg testing, method (*T) Run(string, func(*T)) bool
 pkg testing, type InternalExample struct, Unordered bool

doc/cmd.html

@@ -89,7 +89,7 @@ gofmt</a> command with more general options.</td>
 </tr>
 
 <tr>
-<td><a href="//godoc.org/golang.org/x/tools/cmd/vet/">vet</a></td>
+<td><a href="/cmd/vet/">vet</a></td>
 <td>&nbsp;&nbsp;&nbsp;&nbsp;</td>
 <td>Vet examines Go source code and reports suspicious constructs, such as Printf
 calls whose arguments do not align with the format string.</td>

doc/contribute.html

@@ -353,10 +353,13 @@ with a thorough description of your change.
 The first line of the change description is conventionally a one-line
 summary of the change, prefixed by the primary affected package,
 and is used as the subject for code review mail.
-The rest of the
-description elaborates and should provide context for the
+It should complete the sentence "This change modifies Go to _____."
+The rest of the description elaborates and should provide context for the
 change and explain what it does.
+Write in complete sentences with correct punctuation, just like
+for your comments in Go.
 If there is a helpful reference, mention it here.
+If you've fixed an issue, reference it by number with a # before it.
 </p>
 
 <p>
@@ -364,7 +367,7 @@ After editing, the template might now read:
 </p>
 
 <pre>
-math: improved Sin, Cos and Tan precision for very large arguments
+math: improve Sin, Cos and Tan precision for very large arguments
 
 The existing implementation has poor numerical properties for
 large arguments, so use the McGillicutty algorithm to improve

doc/effective_go.html

@@ -2014,7 +2014,7 @@ then make the receiver for the method a value of that type.
 type ByteSlice []byte
 
 func (slice ByteSlice) Append(data []byte) []byte {
-    // Body exactly the same as above
+    // Body exactly the same as the Append function defined above.
 }
 </pre>
 <p>

doc/go1.7.txt

-Tools:
-
-cmd/dist: add list subcommand to list all supported platforms (CL 19837)
-cmd/go: GO15VENDOREXPERIMENT gone, assumed on (CL 19615)
-cmd/link: "-X name value" form gone (CL 19614)
-cmd/compile: smaller binaries (many CLs)
-cmd/go, go/build: add support for Fortran (CL 19670, CL 4114)
-cmd/doc: group constructors with types (CL 22354)
-cmd/go, go/build: binary-only package support (CL 22433)
-
-Ports:
-
-We now require OpenBSD 5.6+ (CL 18219, crypto/rand using getentropy)
-plan9/arm support? Start at least.
-cgo and external linking support for linux/mips64 and linux/mips64le (CL 19809, ...)
-
-New packages:
-
-* context (and new support in net, net/http, os/exec, net/http/httptrace)
-* net/http/httptrace
-
-API additions and behavior changes:
-
-crypto/tls: allow renegotiation to be handled by a client (CL 22475)
-runtime: support symbolic backtrace of C code in a cgo crash (CL 17761)
-runtime: add CallerFrames and Frames (CL 19869)
-testing/quick: now generates nil values (CL 16470)
-net/http/httptest: ResponseRecorder supports trailer (CL 20047) (compat impact: issue 14928)
-net/url: support query string without values (CL 19931)
-net/textproto: permit all valid token chars in CanonicalMIMEHeaderKey input (CL 18725)
-go/doc: add Unordered boolean to Example struct (CL 19280)
-time: print zero duration as 0s, not 0 (CL 22357)

doc/install-source.html

@@ -33,7 +33,7 @@ compiler using the GCC back end, see
 </p>
 
 <p>
-The Go compilers support five instruction sets.
+The Go compilers support six instruction sets.
 There are important differences in the quality of the compilers for the different
 architectures.
 </p>
@@ -63,19 +63,19 @@ architectures.
 	<code>arm64</code> (<code>AArch64</code>)
 </dt>
 <dd>
-	Supports Linux and Darwin binaries. New in 1.5 and not as well excercised as other ports.
+	Supports Linux and Darwin binaries. New in 1.5 and not as well exercised as other ports.
 </dd>
 <dt>
 	<code>ppc64, ppc64le</code> (64-bit PowerPC big- and little-endian)
 </dt>
 <dd>
-	Supports Linux binaries. New in 1.5 and not as well excercised as other ports.
+	Supports Linux binaries. New in 1.5 and not as well exercised as other ports.
 </dd>
 <dt>
 	<code>mips64, mips64le</code> (64-bit MIPS big- and little-endian)
 </dt>
 <dd>
-	Supports Linux binaries. New in 1.6 and not as well excercised as other ports.
+	Supports Linux binaries. New in 1.6 and not as well exercised as other ports.
 </dd>
 </dl>
 

doc/install.html

@@ -221,7 +221,7 @@ and building a simple program, as follows.
 <p>
 Create a directory to contain your <a href="code.html#Workspaces">workspace</a>,
 <code class="testUnix">$HOME/work</code>
-<code class="testWindows" style="display: none">%HOME%\work</code>
+<code class="testWindows" style="display: none">C:\work</code>
 for example, and set the <code>GOPATH</code> environment
 variable to point to that location.
 </p>
@@ -231,7 +231,7 @@ $ <b>export GOPATH=$HOME/work</b>
 </pre>
 
 <pre class="testWindows" style="display: none">
-C:\&gt; <b>set GOPATH=%HOME%\work</b>
+C:\&gt; <b>set GOPATH=C:\work</b>
 </pre>
 
 <p>

misc/cgo/errors/issue14669.go

+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Issue 14669: test that fails when build with CGO_CFLAGS selecting
+// optimization.
+
+package p
+
+/*
+const int E = 1;
+
+typedef struct s {
+	int       c;
+} s;
+*/
+import "C"
+
+func F() {
+	_ = C.s{
+		c: C.E,
+	}
+}

misc/cgo/errors/ptr.go

@@ -290,6 +290,30 @@ var ptrTests = []ptrTest{
 		},
 		fail: true,
 	},
+	{
+		// Don't check non-pointer data.
+		// Uses unsafe code to get a pointer we shouldn't check.
+		// Although we use unsafe, the uintptr represents an integer
+		// that happens to have the same representation as a pointer;
+		// that is, we are testing something that is not unsafe.
+		name: "ptrdata1",
+		c: `#include <stdlib.h>
+                    void f(void* p) {}`,
+		imports: []string{"unsafe"},
+		support: `type S struct { p *int; a [8*8]byte; u uintptr }`,
+		body:    `i := 0; p := &S{u:uintptr(unsafe.Pointer(&i))}; q := (*S)(C.malloc(C.size_t(unsafe.Sizeof(*p)))); *q = *p; C.f(unsafe.Pointer(q))`,
+		fail:    false,
+	},
+	{
+		// Like ptrdata1, but with a type that uses a GC program.
+		name: "ptrdata2",
+		c: `#include <stdlib.h>
+                    void f(void* p) {}`,
+		imports: []string{"unsafe"},
+		support: `type S struct { p *int; a [32769*8]byte; q *int; u uintptr }`,
+		body:    `i := 0; p := S{u:uintptr(unsafe.Pointer(&i))}; q := (*S)(C.malloc(C.size_t(unsafe.Sizeof(p)))); *q = p; C.f(unsafe.Pointer(q))`,
+		fail:    false,
+	},
 }
 
 func main() {

misc/cgo/errors/test.bash

@@ -45,6 +45,13 @@ expect issue13129.go C.ushort
 check issue13423.go
 expect issue13635.go C.uchar C.schar C.ushort C.uint C.ulong C.longlong C.ulonglong C.complexfloat C.complexdouble
 
+if ! go build issue14669.go; then
+	exit 1
+fi
+if ! CGO_CFLAGS="-O" go build issue14669.go; then
+	exit 1
+fi
+
 if ! go run ptr.go; then
 	exit 1
 fi

misc/cgo/testcarchive/carchive_test.go

@@ -79,14 +79,12 @@ func init() {
 	}
 
 	if GOOS == "darwin" {
-		cc = append(cc, "-Wl,-no_pie")
-
 		// For Darwin/ARM.
 		// TODO(crawshaw): can we do better?
 		cc = append(cc, []string{"-framework", "CoreFoundation", "-framework", "Foundation"}...)
 	}
 	libgodir = GOOS + "_" + GOARCH
-	if GOOS == "darwin" && GOARCH == "arm" {
+	if GOOS == "darwin" && (GOARCH == "arm" || GOARCH == "arm64") {
 		libgodir = GOOS + "_" + GOARCH + "_shared"
 	}
 	cc = append(cc, "-I", filepath.Join("pkg", libgodir))

misc/cgo/testcarchive/main4.c

@@ -44,8 +44,7 @@ static void init() {
 
 // Test raising SIGIO on a C thread with an alternate signal stack
 // when there is a Go signal handler for SIGIO.
-static void* thread1(void* arg) {
-	pthread_t* ptid = (pthread_t*)(arg);
+static void* thread1(void* arg __attribute__ ((unused))) {
 	stack_t ss;
 	int i;
 	stack_t nss;
@@ -65,7 +64,7 @@ static void* thread1(void* arg) {
 	// Send ourselves a SIGIO.  This will be caught by the Go
 	// signal handler which should forward to the C signal
 	// handler.
-	i = pthread_kill(*ptid, SIGIO);
+	i = pthread_kill(pthread_self(), SIGIO);
 	if (i != 0) {
 		fprintf(stderr, "pthread_kill: %s\n", strerror(i));
 		exit(EXIT_FAILURE);
@@ -101,11 +100,11 @@ static void* thread1(void* arg) {
 
 // Test calling a Go function to raise SIGIO on a C thread with an
 // alternate signal stack when there is a Go signal handler for SIGIO.
-static void* thread2(void* arg) {
-	pthread_t* ptid = (pthread_t*)(arg);
+static void* thread2(void* arg __attribute__ ((unused))) {
 	stack_t ss;
 	int i;
 	int oldcount;
+	pthread_t tid;
 	stack_t nss;
 
 	// Set up an alternate signal stack for this thread.
@@ -124,7 +123,8 @@ static void* thread2(void* arg) {
 
 	// Call a Go function that will call a C function to send us a
 	// SIGIO.
-	GoRaiseSIGIO(ptid);
+	tid = pthread_self();
+	GoRaiseSIGIO(&tid);
 
 	// Wait until the signal has been delivered.
 	i = 0;
@@ -161,7 +161,7 @@ int main(int argc, char **argv) {
 	// Tell the Go library to start looking for SIGIO.
 	GoCatchSIGIO();
 
-	i = pthread_create(&tid, NULL, thread1, (void*)(&tid));
+	i = pthread_create(&tid, NULL, thread1, NULL);
 	if (i != 0) {
 		fprintf(stderr, "pthread_create: %s\n", strerror(i));
 		exit(EXIT_FAILURE);
@@ -173,7 +173,7 @@ int main(int argc, char **argv) {
 		exit(EXIT_FAILURE);
 	}
 
-	i = pthread_create(&tid, NULL, thread2, (void*)(&tid));
+	i = pthread_create(&tid, NULL, thread2, NULL);
 	if (i != 0) {
 		fprintf(stderr, "pthread_create: %s\n", strerror(i));
 		exit(EXIT_FAILURE);

misc/cgo/testsanitizers/test.bash

@@ -134,6 +134,26 @@ if test "$tsan" = "yes"; then
 	status=1
     fi
 
+    if ! go run tsan3.go 2>$err; then
+	cat $err
+	echo "FAIL: tsan3"
+	status=1
+    elif grep -i warning $err >/dev/null 2>&1; then
+	cat $err
+	echo "FAIL: tsan3"
+	status=1
+    fi
+
+    if ! go run tsan4.go 2>$err; then
+	cat $err
+	echo "FAIL: tsan4"
+	status=1
+    elif grep -i warning $err >/dev/null 2>&1; then
+	cat $err
+	echo "FAIL: tsan4"
+	status=1
+    fi
+
     rm -f $err
 fi
 

misc/cgo/testsanitizers/tsan3.go

+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+// The stubs for the C functions read and write the same slot on the
+// g0 stack when copying arguments in and out.
+
+/*
+#cgo CFLAGS: -fsanitize=thread
+#cgo LDFLAGS: -fsanitize=thread
+
+int Func1() {
+	return 0;
+}
+
+void Func2(int x) {
+	(void)x;
+}
+*/
+import "C"
+
+func main() {
+	const N = 10000
+	done := make(chan bool, N)
+	for i := 0; i < N; i++ {
+		go func() {
+			C.Func1()
+			done <- true
+		}()
+		go func() {
+			C.Func2(0)
+			done <- true
+		}()
+	}
+	for i := 0; i < 2*N; i++ {
+		<-done
+	}
+}

misc/cgo/testsanitizers/tsan4.go

+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+// Check that calls to C.malloc/C.free do not trigger TSAN false
+// positive reports.
+
+// #cgo CFLAGS: -fsanitize=thread
+// #cgo LDFLAGS: -fsanitize=thread
+// #include <stdlib.h>
+import "C"
+
+import (
+	"runtime"
+	"sync"
+)
+
+func main() {
+	var wg sync.WaitGroup
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 100; i++ {
+				p := C.malloc(C.size_t(i * 10))
+				runtime.Gosched()
+				C.free(p)
+			}
+		}()
+	}
+	wg.Wait()
+}

misc/nacl/testzip.proto

@@ -27,23 +27,23 @@ go	src=..
 			internal
 				objfile
 					objfile.go
-				unvendor
-					golang.org
-							x
-								arch
-									arm
-										armasm
-											testdata
-													+
-									x86
-										x86asm
-											testdata
-													+
 			gofmt
 				gofmt.go
 				gofmt_test.go
 				testdata
 					+
+			vendor
+				golang.org
+					x
+						arch
+							arm
+								armasm
+									testdata
+										+
+							x86
+								x86asm
+									testdata
+										+
 		archive
 			tar
 				testdata

src/archive/tar/common.go

@@ -21,10 +21,8 @@ import (
 	"time"
 )
 
+// Header type flags.
 const (
-	blockSize = 512
-
-	// Types
 	TypeReg           = '0'    // regular file
 	TypeRegA          = '\x00' // regular file
 	TypeLink          = '1'    // hard link
@@ -61,12 +59,6 @@ type Header struct {
 	Xattrs     map[string]string
 }
 
-// File name constants from the tar spec.
-const (
-	fileNameSize       = 100 // Maximum number of bytes in a standard tar name.
-	fileNamePrefixSize = 155 // Maximum number of ustar extension bytes.
-)
-
 // FileInfo returns an os.FileInfo for the Header.
 func (h *Header) FileInfo() os.FileInfo {
 	return headerFileInfo{h}
@@ -279,33 +271,6 @@ func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
 	return h, nil
 }
 
-var zeroBlock = make([]byte, blockSize)
-
-// POSIX specifies a sum of the unsigned byte values, but the Sun tar uses signed byte values.
-// We compute and return both.
-func checksum(header []byte) (unsigned int64, signed int64) {
-	for i := 0; i < len(header); i++ {
-		if i == 148 {
-			// The chksum field (header[148:156]) is special: it should be treated as space bytes.
-			unsigned += ' ' * 8
-			signed += ' ' * 8
-			i += 7
-			continue
-		}
-		unsigned += int64(header[i])
-		signed += int64(int8(header[i]))
-	}
-	return
-}
-
-type slicer []byte
-
-func (sp *slicer) next(n int) (b []byte) {
-	s := *sp
-	b, *sp = s[0:n], s[n:]
-	return
-}
-
 func isASCII(s string) bool {
 	for _, c := range s {
 		if c >= 0x80 {

src/archive/tar/format.go

+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tar
+
+// Constants to identify various tar formats.
+const (
+	// The format is unknown.
+	formatUnknown = (1 << iota) / 2 // Sequence of 0, 1, 2, 4, 8, etc...
+
+	// The format of the original Unix V7 tar tool prior to standardization.
+	formatV7
+
+	// The old and new GNU formats, which are incompatible with USTAR.
+	// This does cover the old GNU sparse extension.
+	// This does not cover the GNU sparse extensions using PAX headers,
+	// versions 0.0, 0.1, and 1.0; these fall under the PAX format.
+	formatGNU
+
+	// Schily's tar format, which is incompatible with USTAR.
+	// This does not cover STAR extensions to the PAX format; these fall under
+	// the PAX format.
+	formatSTAR
+
+	// USTAR is the former standardization of tar defined in POSIX.1-1988.
+	// This is incompatible with the GNU and STAR formats.
+	formatUSTAR
+
+	// PAX is the latest standardization of tar defined in POSIX.1-2001.
+	// This is an extension of USTAR and is "backwards compatible" with it.
+	//
+	// Some newer formats add their own extensions to PAX, such as GNU sparse
+	// files and SCHILY extended attributes. Since they are backwards compatible
+	// with PAX, they will be labelled as "PAX".
+	formatPAX
+)
+
+// Magics used to identify various formats.
+const (
+	magicGNU, versionGNU     = "ustar ", " \x00"
+	magicUSTAR, versionUSTAR = "ustar\x00", "00"
+	trailerSTAR              = "tar\x00"
+)
+
+// Size constants from various tar specifications.
+const (
+	blockSize  = 512 // Size of each block in a tar stream
+	nameSize   = 100 // Max length of the name field in USTAR format
+	prefixSize = 155 // Max length of the prefix field in USTAR format
+)
+
+var zeroBlock block
+
+type block [blockSize]byte
+
+// Convert block to any number of formats.
+func (b *block) V7() *headerV7       { return (*headerV7)(b) }
+func (b *block) GNU() *headerGNU     { return (*headerGNU)(b) }
+func (b *block) STAR() *headerSTAR   { return (*headerSTAR)(b) }
+func (b *block) USTAR() *headerUSTAR { return (*headerUSTAR)(b) }
+func (b *block) Sparse() sparseArray { return (sparseArray)(b[:]) }
+
+// GetFormat checks that the block is a valid tar header based on the checksum.
+// It then attempts to guess the specific format based on magic values.
+// If the checksum fails, then formatUnknown is returned.
+func (b *block) GetFormat() (format int) {
+	// Verify checksum.
+	var p parser
+	value := p.parseOctal(b.V7().Chksum())
+	chksum1, chksum2 := b.ComputeChecksum()
+	if p.err != nil || (value != chksum1 && value != chksum2) {
+		return formatUnknown
+	}
+
+	// Guess the magic values.
+	magic := string(b.USTAR().Magic())
+	version := string(b.USTAR().Version())
+	trailer := string(b.STAR().Trailer())
+	switch {
+	case magic == magicUSTAR && trailer == trailerSTAR:
+		return formatSTAR
+	case magic == magicUSTAR:
+		return formatUSTAR
+	case magic == magicGNU && version == versionGNU:
+		return formatGNU
+	default:
+		return formatV7
+	}
+}
+
+// SetFormat writes the magic values necessary for specified format
+// and then updates the checksum accordingly.
+func (b *block) SetFormat(format int) {
+	// Set the magic values.
+	switch format {
+	case formatV7:
+		// Do nothing.
+	case formatGNU:
+		copy(b.GNU().Magic(), magicGNU)
+		copy(b.GNU().Version(), versionGNU)
+	case formatSTAR:
+		copy(b.STAR().Magic(), magicUSTAR)
+		copy(b.STAR().Version(), versionUSTAR)
+		copy(b.STAR().Trailer(), trailerSTAR)
+	case formatUSTAR, formatPAX:
+		copy(b.USTAR().Magic(), magicUSTAR)
+		copy(b.USTAR().Version(), versionUSTAR)
+	default:
+		panic("invalid format")
+	}
+
+	// Update checksum.
+	// This field is special in that it is terminated by a NULL then space.
+	var f formatter
+	field := b.V7().Chksum()
+	chksum, _ := b.ComputeChecksum() // Possible values are 256..128776
+	f.formatOctal(field[:7], chksum) // Never fails since 128776 < 262143
+	field[7] = ' '
+}
+
+// ComputeChecksum computes the checksum for the header block.
+// POSIX specifies a sum of the unsigned byte values, but the Sun tar used
+// signed byte values.
+// We compute and return both.
+func (b *block) ComputeChecksum() (unsigned, signed int64) {
+	for i, c := range b {
+		if 148 <= i && i < 156 {
+			c = ' ' // Treat the checksum field itself as all spaces.
+		}
+		unsigned += int64(uint8(c))
+		signed += int64(int8(c))
+	}
+	return unsigned, signed
+}
+
+type headerV7 [blockSize]byte
+
+func (h *headerV7) Name() []byte     { return h[000:][:100] }
+func (h *headerV7) Mode() []byte     { return h[100:][:8] }
+func (h *headerV7) UID() []byte      { return h[108:][:8] }
+func (h *headerV7) GID() []byte      { return h[116:][:8] }
+func (h *headerV7) Size() []byte     { return h[124:][:12] }
+func (h *headerV7) ModTime() []byte  { return h[136:][:12] }
+func (h *headerV7) Chksum() []byte   { return h[148:][:8] }
+func (h *headerV7) TypeFlag() []byte { return h[156:][:1] }
+func (h *headerV7) LinkName() []byte { return h[157:][:100] }
+
+type headerGNU [blockSize]byte
+
+func (h *headerGNU) V7() *headerV7       { return (*headerV7)(h) }
+func (h *headerGNU) Magic() []byte       { return h[257:][:6] }
+func (h *headerGNU) Version() []byte     { return h[263:][:2] }
+func (h *headerGNU) UserName() []byte    { return h[265:][:32] }
+func (h *headerGNU) GroupName() []byte   { return h[297:][:32] }
+func (h *headerGNU) DevMajor() []byte    { return h[329:][:8] }
+func (h *headerGNU) DevMinor() []byte    { return h[337:][:8] }
+func (h *headerGNU) AccessTime() []byte  { return h[345:][:12] }
+func (h *headerGNU) ChangeTime() []byte  { return h[357:][:12] }
+func (h *headerGNU) Sparse() sparseArray { return (sparseArray)(h[386:][:24*4+1]) }
+func (h *headerGNU) RealSize() []byte    { return h[483:][:12] }
+
+type headerSTAR [blockSize]byte
+
+func (h *headerSTAR) V7() *headerV7      { return (*headerV7)(h) }
+func (h *headerSTAR) Magic() []byte      { return h[257:][:6] }
+func (h *headerSTAR) Version() []byte    { return h[263:][:2] }
+func (h *headerSTAR) UserName() []byte   { return h[265:][:32] }
+func (h *headerSTAR) GroupName() []byte  { return h[297:][:32] }
+func (h *headerSTAR) DevMajor() []byte   { return h[329:][:8] }
+func (h *headerSTAR) DevMinor() []byte   { return h[337:][:8] }
+func (h *headerSTAR) Prefix() []byte     { return h[345:][:131] }
+func (h *headerSTAR) AccessTime() []byte { return h[476:][:12] }
+func (h *headerSTAR) ChangeTime() []byte { return h[488:][:12] }
+func (h *headerSTAR) Trailer() []byte    { return h[508:][:4] }
+
+type headerUSTAR [blockSize]byte
+
+func (h *headerUSTAR) V7() *headerV7     { return (*headerV7)(h) }
+func (h *headerUSTAR) Magic() []byte     { return h[257:][:6] }
+func (h *headerUSTAR) Version() []byte   { return h[263:][:2] }
+func (h *headerUSTAR) UserName() []byte  { return h[265:][:32] }
+func (h *headerUSTAR) GroupName() []byte { return h[297:][:32] }
+func (h *headerUSTAR) DevMajor() []byte  { return h[329:][:8] }
+func (h *headerUSTAR) DevMinor() []byte  { return h[337:][:8] }
+func (h *headerUSTAR) Prefix() []byte    { return h[345:][:155] }
+
+type sparseArray []byte
+
+func (s sparseArray) Entry(i int) sparseNode { return (sparseNode)(s[i*24:]) }
+func (s sparseArray) IsExtended() []byte     { return s[24*s.MaxEntries():][:1] }
+func (s sparseArray) MaxEntries() int        { return len(s) / 24 }
+
+type sparseNode []byte
+
+func (s sparseNode) Offset() []byte   { return s[00:][:12] }
+func (s sparseNode) NumBytes() []byte { return s[12:][:12] }

src/archive/tar/reader.go

@@ -29,11 +29,11 @@ const maxNanoSecondIntSize = 9
 // The Next method advances to the next file in the archive (including the first),
 // and then it can be treated as an io.Reader to access the file's data.
 type Reader struct {
-	r       io.Reader
-	err     error
-	pad     int64           // amount of padding (ignored) after current file entry
-	curr    numBytesReader  // reader for current file entry
-	hdrBuff [blockSize]byte // buffer to use in readHeader
+	r    io.Reader
+	err  error
+	pad  int64          // amount of padding (ignored) after current file entry
+	curr numBytesReader // reader for current file entry
+	blk  block          // buffer to use as temporary local storage
 }
 
 type parser struct {
@@ -98,17 +98,6 @@ const (
 	paxGNUSparseRealSize  = "GNU.sparse.realsize"
 )
 
-// Keywords for old GNU sparse headers
-const (
-	oldGNUSparseMainHeaderOffset               = 386
-	oldGNUSparseMainHeaderIsExtendedOffset     = 482
-	oldGNUSparseMainHeaderNumEntries           = 4
-	oldGNUSparseExtendedHeaderIsExtendedOffset = 504
-	oldGNUSparseExtendedHeaderNumEntries       = 21
-	oldGNUSparseOffsetSize                     = 12
-	oldGNUSparseNumBytesSize                   = 12
-)
-
 // NewReader creates a new Reader reading from r.
 func NewReader(r io.Reader) *Reader { return &Reader{r: r} }
 
@@ -542,17 +531,6 @@ func (tr *Reader) skipUnread() error {
 	return tr.err
 }
 
-func (tr *Reader) verifyChecksum(header []byte) bool {
-	if tr.err != nil {
-		return false
-	}
-
-	var p parser
-	given := p.parseOctal(header[148:156])
-	unsigned, signed := checksum(header)
-	return p.err == nil && (given == unsigned || given == signed)
-}
-
 // readHeader reads the next block header and assumes that the underlying reader
 // is already aligned to a block boundary.
 //
@@ -561,19 +539,16 @@ func (tr *Reader) verifyChecksum(header []byte) bool {
 //	* Exactly 1 block of zeros is read and EOF is hit.
 //	* At least 2 blocks of zeros are read.
 func (tr *Reader) readHeader() *Header {
-	header := tr.hdrBuff[:]
-	copy(header, zeroBlock)
-
-	if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
+	if _, tr.err = io.ReadFull(tr.r, tr.blk[:]); tr.err != nil {
 		return nil // io.EOF is okay here
 	}
 
 	// Two blocks of zero bytes marks the end of the archive.
-	if bytes.Equal(header, zeroBlock[0:blockSize]) {
-		if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
+	if bytes.Equal(tr.blk[:], zeroBlock[:]) {
+		if _, tr.err = io.ReadFull(tr.r, tr.blk[:]); tr.err != nil {
 			return nil // io.EOF is okay here
 		}
-		if bytes.Equal(header, zeroBlock[0:blockSize]) {
+		if bytes.Equal(tr.blk[:], zeroBlock[:]) {
 			tr.err = io.EOF
 		} else {
 			tr.err = ErrHeader // zero block and then non-zero block
@@ -581,71 +556,55 @@ func (tr *Reader) readHeader() *Header {
 		return nil
 	}
 
-	if !tr.verifyChecksum(header) {
+	// Verify the header matches a known format.
+	format := tr.blk.GetFormat()
+	if format == formatUnknown {
 		tr.err = ErrHeader
 		return nil
 	}
 
-	// Unpack
 	var p parser
 	hdr := new(Header)
-	s := slicer(header)
-
-	hdr.Name = p.parseString(s.next(100))
-	hdr.Mode = p.parseNumeric(s.next(8))
-	hdr.Uid = int(p.parseNumeric(s.next(8)))
-	hdr.Gid = int(p.parseNumeric(s.next(8)))
-	hdr.Size = p.parseNumeric(s.next(12))
-	hdr.ModTime = time.Unix(p.parseNumeric(s.next(12)), 0)
-	s.next(8) // chksum
-	hdr.Typeflag = s.next(1)[0]
-	hdr.Linkname = p.parseString(s.next(100))
-
-	// The remainder of the header depends on the value of magic.
-	// The original (v7) version of tar had no explicit magic field,
-	// so its magic bytes, like the rest of the block, are NULs.
-	magic := string(s.next(8)) // contains version field as well.
-	var format string
-	switch {
-	case magic[:6] == "ustar\x00": // POSIX tar (1003.1-1988)
-		if string(header[508:512]) == "tar\x00" {
-			format = "star"
-		} else {
-			format = "posix"
-		}
-	case magic == "ustar  \x00": // old GNU tar
-		format = "gnu"
-	}
 
-	switch format {
-	case "posix", "gnu", "star":
-		hdr.Uname = p.parseString(s.next(32))
-		hdr.Gname = p.parseString(s.next(32))
-		devmajor := s.next(8)
-		devminor := s.next(8)
+	// Unpack the V7 header.
+	v7 := tr.blk.V7()
+	hdr.Name = p.parseString(v7.Name())
+	hdr.Mode = p.parseNumeric(v7.Mode())
+	hdr.Uid = int(p.parseNumeric(v7.UID()))
+	hdr.Gid = int(p.parseNumeric(v7.GID()))
+	hdr.Size = p.parseNumeric(v7.Size())
+	hdr.ModTime = time.Unix(p.parseNumeric(v7.ModTime()), 0)
+	hdr.Typeflag = v7.TypeFlag()[0]
+	hdr.Linkname = p.parseString(v7.LinkName())
+
+	// Unpack format specific fields.
+	if format > formatV7 {
+		ustar := tr.blk.USTAR()
+		hdr.Uname = p.parseString(ustar.UserName())
+		hdr.Gname = p.parseString(ustar.GroupName())
 		if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock {
-			hdr.Devmajor = p.parseNumeric(devmajor)
-			hdr.Devminor = p.parseNumeric(devminor)
+			hdr.Devmajor = p.parseNumeric(ustar.DevMajor())
+			hdr.Devminor = p.parseNumeric(ustar.DevMinor())
 		}
+
 		var prefix string
 		switch format {
-		case "posix", "gnu":
-			prefix = p.parseString(s.next(155))
-		case "star":
-			prefix = p.parseString(s.next(131))
-			hdr.AccessTime = time.Unix(p.parseNumeric(s.next(12)), 0)
-			hdr.ChangeTime = time.Unix(p.parseNumeric(s.next(12)), 0)
+		case formatUSTAR, formatGNU:
+			// TODO(dsnet): Do not use the prefix field for the GNU format!
+			// See golang.org/issues/12594
+			ustar := tr.blk.USTAR()
+			prefix = p.parseString(ustar.Prefix())
+		case formatSTAR:
+			star := tr.blk.STAR()
+			prefix = p.parseString(star.Prefix())
+			hdr.AccessTime = time.Unix(p.parseNumeric(star.AccessTime()), 0)
+			hdr.ChangeTime = time.Unix(p.parseNumeric(star.ChangeTime()), 0)
 		}
 		if len(prefix) > 0 {
 			hdr.Name = prefix + "/" + hdr.Name
 		}
 	}
 
-	if p.err != nil {
-		tr.err = p.err
-		return nil
-	}
-
 	nb := hdr.Size
 	if isHeaderOnlyType(hdr.Typeflag) {
 		nb = 0
@@ -662,14 +621,14 @@ func (tr *Reader) readHeader() *Header {
 	// Check for old GNU sparse format entry.
 	if hdr.Typeflag == TypeGNUSparse {
 		// Get the real size of the file.
-		hdr.Size = p.parseNumeric(header[483:495])
+		hdr.Size = p.parseNumeric(tr.blk.GNU().RealSize())
 		if p.err != nil {
 			tr.err = p.err
 			return nil
 		}
 
 		// Read the sparse map.
-		sp := tr.readOldGNUSparseMap(header)
+		sp := tr.readOldGNUSparseMap(&tr.blk)
 		if tr.err != nil {
 			return nil
 		}
@@ -681,26 +640,24 @@ func (tr *Reader) readHeader() *Header {
 		}
 	}
 
+	if p.err != nil {
+		tr.err = p.err
+		return nil
+	}
+
 	return hdr
 }
 
 // readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format.
 // The sparse map is stored in the tar header if it's small enough. If it's larger than four entries,
 // then one or more extension headers are used to store the rest of the sparse map.
-func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
+func (tr *Reader) readOldGNUSparseMap(blk *block) []sparseEntry {
 	var p parser
-	isExtended := header[oldGNUSparseMainHeaderIsExtendedOffset] != 0
-	spCap := oldGNUSparseMainHeaderNumEntries
-	if isExtended {
-		spCap += oldGNUSparseExtendedHeaderNumEntries
-	}
-	sp := make([]sparseEntry, 0, spCap)
-	s := slicer(header[oldGNUSparseMainHeaderOffset:])
-
-	// Read the four entries from the main tar header
-	for i := 0; i < oldGNUSparseMainHeaderNumEntries; i++ {
-		offset := p.parseNumeric(s.next(oldGNUSparseOffsetSize))
-		numBytes := p.parseNumeric(s.next(oldGNUSparseNumBytesSize))
+	var s sparseArray = blk.GNU().Sparse()
+	var sp = make([]sparseEntry, 0, s.MaxEntries())
+	for i := 0; i < s.MaxEntries(); i++ {
+		offset := p.parseOctal(s.Entry(i).Offset())
+		numBytes := p.parseOctal(s.Entry(i).NumBytes())
 		if p.err != nil {
 			tr.err = p.err
 			return nil
@@ -711,17 +668,17 @@ func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
 		sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
 	}
 
-	for isExtended {
+	for s.IsExtended()[0] > 0 {
 		// There are more entries. Read an extension header and parse its entries.
-		sparseHeader := make([]byte, blockSize)
-		if _, tr.err = io.ReadFull(tr.r, sparseHeader); tr.err != nil {
+		var blk block
+		if _, tr.err = io.ReadFull(tr.r, blk[:]); tr.err != nil {
 			return nil
 		}
-		isExtended = sparseHeader[oldGNUSparseExtendedHeaderIsExtendedOffset] != 0
-		s = slicer(sparseHeader)
-		for i := 0; i < oldGNUSparseExtendedHeaderNumEntries; i++ {
-			offset := p.parseNumeric(s.next(oldGNUSparseOffsetSize))
-			numBytes := p.parseNumeric(s.next(oldGNUSparseNumBytesSize))
+		s = blk.Sparse()
+
+		for i := 0; i < s.MaxEntries(); i++ {
+			offset := p.parseOctal(s.Entry(i).Offset())
+			numBytes := p.parseOctal(s.Entry(i).NumBytes())
 			if p.err != nil {
 				tr.err = p.err
 				return nil

src/archive/tar/writer.go

@@ -36,10 +36,10 @@ type Writer struct {
 	nb         int64 // number of unwritten bytes for current file entry
 	pad        int64 // amount of padding to write after current file entry
 	closed     bool
-	usedBinary bool            // whether the binary numeric field extension was used
-	preferPax  bool            // use pax header instead of binary numeric header
-	hdrBuff    [blockSize]byte // buffer to use in writeHeader when writing a regular header
-	paxHdrBuff [blockSize]byte // buffer to use in writeHeader when writing a pax header
+	usedBinary bool  // whether the binary numeric field extension was used
+	preferPax  bool  // use PAX header instead of binary numeric header
+	hdrBuff    block // buffer to use in writeHeader when writing a regular header
+	paxHdrBuff block // buffer to use in writeHeader when writing a PAX header
 }
 
 type formatter struct {
@@ -153,27 +153,24 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 	// a map to hold pax header records, if any are needed
 	paxHeaders := make(map[string]string)
 
-	// TODO(shanemhansen): we might want to use PAX headers for
+	// TODO(dsnet): we might want to use PAX headers for
 	// subsecond time resolution, but for now let's just capture
 	// too long fields or non ascii characters
 
-	var f formatter
-	var header []byte
-
 	// We need to select which scratch buffer to use carefully,
 	// since this method is called recursively to write PAX headers.
 	// If allowPax is true, this is the non-recursive call, and we will use hdrBuff.
 	// If allowPax is false, we are being called by writePAXHeader, and hdrBuff is
 	// already being used by the non-recursive call, so we must use paxHdrBuff.
-	header = tw.hdrBuff[:]
+	header := &tw.hdrBuff
 	if !allowPax {
-		header = tw.paxHdrBuff[:]
+		header = &tw.paxHdrBuff
 	}
-	copy(header, zeroBlock)
-	s := slicer(header)
+	copy(header[:], zeroBlock[:])
 
 	// Wrappers around formatter that automatically sets paxHeaders if the
 	// argument extends beyond the capacity of the input byte slice.
+	var f formatter
 	var formatString = func(b []byte, s string, paxKeyword string) {
 		needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s)
 		if needsPaxHeader {
@@ -202,44 +199,33 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 		f.formatNumeric(b, x)
 	}
 
-	// keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
-	pathHeaderBytes := s.next(fileNameSize)
-
-	formatString(pathHeaderBytes, hdr.Name, paxPath)
-
 	// Handle out of range ModTime carefully.
 	var modTime int64
 	if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
 		modTime = hdr.ModTime.Unix()
 	}
 
-	f.formatOctal(s.next(8), hdr.Mode)               // 100:108
-	formatNumeric(s.next(8), int64(hdr.Uid), paxUid) // 108:116
-	formatNumeric(s.next(8), int64(hdr.Gid), paxGid) // 116:124
-	formatNumeric(s.next(12), hdr.Size, paxSize)     // 124:136
-	formatNumeric(s.next(12), modTime, paxNone)      // 136:148 --- consider using pax for finer granularity
-	s.next(8)                                        // chksum (148:156)
-	s.next(1)[0] = hdr.Typeflag                      // 156:157
-
-	formatString(s.next(100), hdr.Linkname, paxLinkpath)
-
-	copy(s.next(8), []byte("ustar\x0000"))          // 257:265
-	formatString(s.next(32), hdr.Uname, paxUname)   // 265:297
-	formatString(s.next(32), hdr.Gname, paxGname)   // 297:329
-	formatNumeric(s.next(8), hdr.Devmajor, paxNone) // 329:337
-	formatNumeric(s.next(8), hdr.Devminor, paxNone) // 337:345
-
-	// keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
-	prefixHeaderBytes := s.next(155)
-	formatString(prefixHeaderBytes, "", paxNone) // 345:500  prefix
+	v7 := header.V7()
+	formatString(v7.Name(), hdr.Name, paxPath)
+	// TODO(dsnet): The GNU format permits the mode field to be encoded in
+	// base-256 format. Thus, we can use formatNumeric instead of formatOctal.
+	f.formatOctal(v7.Mode(), hdr.Mode)
+	formatNumeric(v7.UID(), int64(hdr.Uid), paxUid)
+	formatNumeric(v7.GID(), int64(hdr.Gid), paxGid)
+	formatNumeric(v7.Size(), hdr.Size, paxSize)
+	// TODO(dsnet): Consider using PAX for finer time granularity.
+	formatNumeric(v7.ModTime(), modTime, paxNone)
+	v7.TypeFlag()[0] = hdr.Typeflag
+	formatString(v7.LinkName(), hdr.Linkname, paxLinkpath)
+
+	ustar := header.USTAR()
+	formatString(ustar.UserName(), hdr.Uname, paxUname)
+	formatString(ustar.GroupName(), hdr.Gname, paxGname)
+	formatNumeric(ustar.DevMajor(), hdr.Devmajor, paxNone)
+	formatNumeric(ustar.DevMinor(), hdr.Devminor, paxNone)
 
-	// Use the GNU magic instead of POSIX magic if we used any GNU extensions.
-	if tw.usedBinary {
-		copy(header[257:265], []byte("ustar  \x00"))
-	}
-
-	_, paxPathUsed := paxHeaders[paxPath]
 	// try to use a ustar header when only the name is too long
+	_, paxPathUsed := paxHeaders[paxPath]
 	if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed {
 		prefix, suffix, ok := splitUSTARPath(hdr.Name)
 		if ok {
@@ -247,16 +233,16 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 			delete(paxHeaders, paxPath)
 
 			// Update the path fields
-			formatString(pathHeaderBytes, suffix, paxNone)
-			formatString(prefixHeaderBytes, prefix, paxNone)
+			formatString(v7.Name(), suffix, paxNone)
+			formatString(ustar.Prefix(), prefix, paxNone)
 		}
 	}
 
-	// The chksum field is terminated by a NUL and a space.
-	// This is different from the other octal fields.
-	chksum, _ := checksum(header)
-	f.formatOctal(header[148:155], chksum) // Never fails
-	header[155] = ' '
+	if tw.usedBinary {
+		header.SetFormat(formatGNU)
+	} else {
+		header.SetFormat(formatUSTAR)
+	}
 
 	// Check if there were any formatting errors.
 	if f.err != nil {
@@ -281,7 +267,7 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 	tw.nb = hdr.Size
 	tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize
 
-	_, tw.err = tw.w.Write(header)
+	_, tw.err = tw.w.Write(header[:])
 	return tw.err
 }
 
@@ -289,10 +275,10 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 // If the path is not splittable, then it will return ("", "", false).
 func splitUSTARPath(name string) (prefix, suffix string, ok bool) {
 	length := len(name)
-	if length <= fileNameSize || !isASCII(name) {
+	if length <= nameSize || !isASCII(name) {
 		return "", "", false
-	} else if length > fileNamePrefixSize+1 {
-		length = fileNamePrefixSize + 1
+	} else if length > prefixSize+1 {
+		length = prefixSize + 1
 	} else if name[length-1] == '/' {
 		length--
 	}
@@ -300,7 +286,7 @@ func splitUSTARPath(name string) (prefix, suffix string, ok bool) {
 	i := strings.LastIndex(name[:length], "/")
 	nlen := len(name) - i - 1 // nlen is length of suffix
 	plen := i                 // plen is length of prefix
-	if i <= 0 || nlen > fileNameSize || nlen == 0 || plen > fileNamePrefixSize {
+	if i <= 0 || nlen > nameSize || nlen == 0 || plen > prefixSize {
 		return "", "", false
 	}
 	return name[:i], name[i+1:], true
@@ -323,8 +309,8 @@ func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) erro
 	fullName := path.Join(dir, "PaxHeaders.0", file)
 
 	ascii := toASCII(fullName)
-	if len(ascii) > 100 {
-		ascii = ascii[:100]
+	if len(ascii) > nameSize {
+		ascii = ascii[:nameSize]
 	}
 	ext.Name = ascii
 	// Construct the body
@@ -407,7 +393,7 @@ func (tw *Writer) Close() error {
 
 	// trailer: two zero blocks
 	for i := 0; i < 2; i++ {
-		_, tw.err = tw.w.Write(zeroBlock)
+		_, tw.err = tw.w.Write(zeroBlock[:])
 		if tw.err != nil {
 			break
 		}

src/archive/tar/writer_test.go

@@ -587,17 +587,17 @@ func TestSplitUSTARPath(t *testing.T) {
 		{"", "", "", false},
 		{"abc", "", "", false},
 		{"用戶名", "", "", false},
-		{sr("a", fileNameSize), "", "", false},
-		{sr("a", fileNameSize) + "/", "", "", false},
-		{sr("a", fileNameSize) + "/a", sr("a", fileNameSize), "a", true},
-		{sr("a", fileNamePrefixSize) + "/", "", "", false},
-		{sr("a", fileNamePrefixSize) + "/a", sr("a", fileNamePrefixSize), "a", true},
-		{sr("a", fileNameSize+1), "", "", false},
-		{sr("/", fileNameSize+1), sr("/", fileNameSize-1), "/", true},
-		{sr("a", fileNamePrefixSize) + "/" + sr("b", fileNameSize),
-			sr("a", fileNamePrefixSize), sr("b", fileNameSize), true},
-		{sr("a", fileNamePrefixSize) + "//" + sr("b", fileNameSize), "", "", false},
-		{sr("a/", fileNameSize), sr("a/", 77) + "a", sr("a/", 22), true},
+		{sr("a", nameSize), "", "", false},
+		{sr("a", nameSize) + "/", "", "", false},
+		{sr("a", nameSize) + "/a", sr("a", nameSize), "a", true},
+		{sr("a", prefixSize) + "/", "", "", false},
+		{sr("a", prefixSize) + "/a", sr("a", prefixSize), "a", true},
+		{sr("a", nameSize+1), "", "", false},
+		{sr("/", nameSize+1), sr("/", nameSize-1), "/", true},
+		{sr("a", prefixSize) + "/" + sr("b", nameSize),
+			sr("a", prefixSize), sr("b", nameSize), true},
+		{sr("a", prefixSize) + "//" + sr("b", nameSize), "", "", false},
+		{sr("a/", nameSize), sr("a/", 77) + "a", sr("a/", 22), true},
 	}
 
 	for _, v := range vectors {

src/archive/zip/struct.go

@@ -5,7 +5,7 @@
 /*
 Package zip provides support for reading and writing ZIP archives.
 
-See: http://www.pkware.com/documents/casestudies/APPNOTE.TXT
+See: https://www.pkware.com/documents/casestudies/APPNOTE.TXT
 
 This package does not support disk spanning.
 

src/bufio/bufio_test.go

@@ -1475,7 +1475,7 @@ func BenchmarkReaderWriteToOptimal(b *testing.B) {
 		b.Fatal("ioutil.Discard doesn't support ReaderFrom")
 	}
 	for i := 0; i < b.N; i++ {
-		r.Seek(0, 0)
+		r.Seek(0, io.SeekStart)
 		srcReader.Reset(onlyReader{r})
 		n, err := srcReader.WriteTo(ioutil.Discard)
 		if err != nil {

src/bytes/reader.go

@@ -108,11 +108,11 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) {
 	r.prevRune = -1
 	var abs int64
 	switch whence {
-	case 0:
+	case io.SeekStart:
 		abs = offset
-	case 1:
+	case io.SeekCurrent:
 		abs = r.i + offset
-	case 2:
+	case io.SeekEnd:
 		abs = int64(len(r.s)) + offset
 	default:
 		return 0, errors.New("bytes.Reader.Seek: invalid whence")

src/bytes/reader_test.go

@@ -188,7 +188,7 @@ var UnreadRuneErrorTests = []struct {
 	{"Read", func(r *Reader) { r.Read([]byte{0}) }},
 	{"ReadByte", func(r *Reader) { r.ReadByte() }},
 	{"UnreadRune", func(r *Reader) { r.UnreadRune() }},
-	{"Seek", func(r *Reader) { r.Seek(0, 1) }},
+	{"Seek", func(r *Reader) { r.Seek(0, io.SeekCurrent) }},
 	{"WriteTo", func(r *Reader) { r.WriteTo(&Buffer{}) }},
 }
 

src/cmd/asm/internal/arch/amd64.go

+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file encapsulates some of the odd characteristics of the
+// AMD64 instruction set, to minimize its interaction
+// with the core of the assembler.
+
+package arch
+
+import (
+	"cmd/internal/obj"
+	"cmd/internal/obj/x86"
+)
+
+// IsAMD4OP reports whether the op (as defined by an ppc64.A* constant) is
+// The FMADD-like instructions behave similarly.
+func IsAMD4OP(op obj.As) bool {
+	switch op {
+	case x86.AVPERM2F128,
+		x86.AVPALIGNR,
+		x86.AVPERM2I128,
+		x86.AVINSERTI128,
+		x86.AVPBLENDD:
+		return true
+	}
+	return false
+}

src/cmd/asm/internal/asm/asm.go

@@ -568,6 +568,15 @@ func (p *Parser) asmInstruction(op obj.As, cond string, a []obj.Addr) {
 			prog.From = a[0]
 			prog.Reg = p.getRegister(prog, op, &a[1])
 			prog.To = a[2]
+		case sys.AMD64:
+			// Catch missing operand here, because we store immediate as part of From3, and can't distinguish
+			// missing operand from legal value 0 in obj/x86/asm6.
+			if arch.IsAMD4OP(op) {
+				p.errorf("4 operands required, but only 3 are provided for %s instruction", obj.Aconv(op))
+			}
+			prog.From = a[0]
+			prog.From3 = newAddr(a[1])
+			prog.To = a[2]
 		case sys.ARM64:
 			// ARM64 instructions with one input and two outputs.
 			if arch.IsARM64STLXR(op) {
@@ -583,7 +592,7 @@ func (p *Parser) asmInstruction(op obj.As, cond string, a []obj.Addr) {
 			prog.From = a[0]
 			prog.Reg = p.getRegister(prog, op, &a[1])
 			prog.To = a[2]
-		case sys.AMD64, sys.I386:
+		case sys.I386:
 			prog.From = a[0]
 			prog.From3 = newAddr(a[1])
 			prog.To = a[2]
@@ -640,6 +649,23 @@ func (p *Parser) asmInstruction(op obj.As, cond string, a []obj.Addr) {
 			prog.Reg = r1
 			break
 		}
+		if p.arch.Family == sys.AMD64 {
+			// 4 operand instruction have form  ymm1, ymm2, ymm3/m256, imm8
+			// So From3 is always just a register, so we store imm8 in Offset field,
+			// to avoid increasing size of Prog.
+			prog.From = a[1]
+			prog.From3 = newAddr(a[2])
+			if a[0].Type != obj.TYPE_CONST {
+				p.errorf("first operand must be an immediate in %s instruction", obj.Aconv(op))
+			}
+			if prog.From3.Type != obj.TYPE_REG {
+				p.errorf("third operand must be a register in %s instruction", obj.Aconv(op))
+			}
+			prog.From3.Offset = int64(p.getImmediate(prog, op, &a[0]))
+			prog.To = a[3]
+			prog.RegTo2 = -1
+			break
+		}
 		if p.arch.Family == sys.ARM64 {
 			prog.From = a[0]
 			prog.Reg = p.getRegister(prog, op, &a[1])

src/cmd/cgo/gcc.go

@@ -1089,6 +1089,8 @@ func (p *Package) gccMachine() []string {
 		return []string{"-m31"}
 	case "s390x":
 		return []string{"-m64"}
+	case "mips64", "mips64le":
+		return []string{"-mabi=64"}
 	}
 	return nil
 }
@@ -1241,12 +1243,20 @@ func (p *Package) gccErrors(stdin []byte) string {
 	// TODO(rsc): require failure
 	args := p.gccCmd()
 
+	// Optimization options can confuse the error messages; remove them.
+	nargs := make([]string, 0, len(args))
+	for _, arg := range args {
+		if !strings.HasPrefix(arg, "-O") {
+			nargs = append(nargs, arg)
+		}
+	}
+
 	if *debugGcc {
-		fmt.Fprintf(os.Stderr, "$ %s <<EOF\n", strings.Join(args, " "))
+		fmt.Fprintf(os.Stderr, "$ %s <<EOF\n", strings.Join(nargs, " "))
 		os.Stderr.Write(stdin)
 		fmt.Fprint(os.Stderr, "EOF\n")
 	}
-	stdout, stderr, _ := run(stdin, args)
+	stdout, stderr, _ := run(stdin, nargs)
 	if *debugGcc {
 		os.Stderr.Write(stdout)
 		os.Stderr.Write(stderr)

src/cmd/cgo/out.go

@@ -175,10 +175,11 @@ func (p *Package) writeDefs() {
 	}
 	fmt.Fprintf(fgo2, "\n")
 
+	callsMalloc := false
 	for _, key := range nameKeys(p.Name) {
 		n := p.Name[key]
 		if n.FuncType != nil {
-			p.writeDefsFunc(fgo2, n)
+			p.writeDefsFunc(fgo2, n, &callsMalloc)
 		}
 	}
 
@@ -189,6 +190,12 @@ func (p *Package) writeDefs() {
 	} else {
 		p.writeExports(fgo2, fm, fgcc, fgcch)
 	}
+
+	if callsMalloc && !*gccgo {
+		fmt.Fprint(fgo2, strings.Replace(cMallocDefGo, "PREFIX", cPrefix, -1))
+		fmt.Fprint(fgcc, strings.Replace(strings.Replace(cMallocDefC, "PREFIX", cPrefix, -1), "PACKED", p.packedAttribute(), -1))
+	}
+
 	if err := fgcc.Close(); err != nil {
 		fatalf("%s", err)
 	}
@@ -352,7 +359,7 @@ func (p *Package) structType(n *Name) (string, int64) {
 	return buf.String(), off
 }
 
-func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name) {
+func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name, callsMalloc *bool) {
 	name := n.Go
 	gtype := n.FuncType.Go
 	void := gtype.Results == nil || len(gtype.Results.List) == 0
@@ -441,6 +448,9 @@ func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name) {
 
 	if inProlog {
 		fmt.Fprint(fgo2, builtinDefs[name])
+		if strings.Contains(builtinDefs[name], "_cgo_cmalloc") {
+			*callsMalloc = true
+		}
 		return
 	}
 
@@ -560,6 +570,7 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
 
 	// Gcc wrapper unpacks the C argument struct
 	// and calls the actual C function.
+	fmt.Fprintf(fgcc, "CGO_NO_SANITIZE_THREAD\n")
 	if n.AddError {
 		fmt.Fprintf(fgcc, "int\n")
 	} else {
@@ -635,6 +646,7 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
 // wrapper, we can't refer to the function, since the reference is in
 // a different file.
 func (p *Package) writeGccgoOutputFunc(fgcc *os.File, n *Name) {
+	fmt.Fprintf(fgcc, "CGO_NO_SANITIZE_THREAD\n")
 	if t := n.FuncType.Result; t != nil {
 		fmt.Fprintf(fgcc, "%s\n", t.C.String())
 	} else {
@@ -710,11 +722,13 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
 	p.writeExportHeader(fgcch)
 
 	fmt.Fprintf(fgcc, "/* Created by cgo - DO NOT EDIT. */\n")
+	fmt.Fprintf(fgcc, "#include <stdlib.h>\n")
 	fmt.Fprintf(fgcc, "#include \"_cgo_export.h\"\n\n")
 
 	fmt.Fprintf(fgcc, "extern void crosscall2(void (*fn)(void *, int, __SIZE_TYPE__), void *, int, __SIZE_TYPE__);\n")
 	fmt.Fprintf(fgcc, "extern __SIZE_TYPE__ _cgo_wait_runtime_init_done();\n")
 	fmt.Fprintf(fgcc, "extern void _cgo_release_context(__SIZE_TYPE__);\n\n")
+	fmt.Fprintf(fgcc, "extern char* _cgo_topofstack(void);")
 	fmt.Fprintf(fgcc, "%s\n", tsanProlog)
 
 	for _, exp := range p.ExpFunc {
@@ -817,6 +831,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
 		fmt.Fprintf(fgcch, "\nextern %s;\n", s)
 
 		fmt.Fprintf(fgcc, "extern void _cgoexp%s_%s(void *, int, __SIZE_TYPE__);\n", cPrefix, exp.ExpName)
+		fmt.Fprintf(fgcc, "\nCGO_NO_SANITIZE_THREAD")
 		fmt.Fprintf(fgcc, "\n%s\n", s)
 		fmt.Fprintf(fgcc, "{\n")
 		fmt.Fprintf(fgcc, "\t__SIZE_TYPE__ _cgo_ctxt = _cgo_wait_runtime_init_done();\n")
@@ -1020,7 +1035,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) {
 		fmt.Fprintf(fgcc, `extern %s %s %s __asm__("%s.%s");`, cRet, goName, cParams, gccgoSymbolPrefix, goName)
 		fmt.Fprint(fgcc, "\n")
 
-		fmt.Fprint(fgcc, "\n")
+		fmt.Fprint(fgcc, "\nCGO_NO_SANITIZE_THREAD\n")
 		fmt.Fprintf(fgcc, "%s %s %s {\n", cRet, exp.ExpName, cParams)
 		if resultCount > 0 {
 			fmt.Fprintf(fgcc, "\t%s r;\n", cRet)
@@ -1304,11 +1319,14 @@ extern char* _cgo_topofstack(void);
 
 // Prologue defining TSAN functions in C.
 const noTsanProlog = `
+#define CGO_NO_SANITIZE_THREAD
 #define _cgo_tsan_acquire()
 #define _cgo_tsan_release()
 `
 
 const yesTsanProlog = `
+#define CGO_NO_SANITIZE_THREAD __attribute__ ((no_sanitize_thread))
+
 long long _cgo_sync __attribute__ ((common));
 
 extern void __tsan_acquire(void*);
@@ -1346,9 +1364,6 @@ const goProlog = `
 //go:linkname _cgo_runtime_cgocall runtime.cgocall
 func _cgo_runtime_cgocall(unsafe.Pointer, uintptr) int32
 
-//go:linkname _cgo_runtime_cmalloc runtime.cmalloc
-func _cgo_runtime_cmalloc(uintptr) unsafe.Pointer
-
 //go:linkname _cgo_runtime_cgocallback runtime.cgocallback
 func _cgo_runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr, uintptr)
 
@@ -1360,10 +1375,8 @@ func _cgoCheckResult(interface{})
 `
 
 const gccgoGoProlog = `
-//extern runtime.cgoCheckPointer
 func _cgoCheckPointer(interface{}, ...interface{}) interface{}
 
-//extern runtime.cgoCheckResult
 func _cgoCheckResult(interface{})
 `
 
@@ -1396,7 +1409,7 @@ func _Cfunc_GoBytes(p unsafe.Pointer, l _Ctype_int) []byte {
 
 const cStringDef = `
 func _Cfunc_CString(s string) *_Ctype_char {
-	p := _cgo_runtime_cmalloc(uintptr(len(s)+1))
+	p := _cgo_cmalloc(uint64(len(s)+1))
 	pp := (*[1<<30]byte)(p)
 	copy(pp[:], s)
 	pp[len(s)] = 0
@@ -1406,7 +1419,7 @@ func _Cfunc_CString(s string) *_Ctype_char {
 
 const cBytesDef = `
 func _Cfunc_CBytes(b []byte) unsafe.Pointer {
-	p := _cgo_runtime_cmalloc(uintptr(len(b)))
+	p := _cgo_cmalloc(uint64(len(b)))
 	pp := (*[1<<30]byte)(p)
 	copy(pp[:], b)
 	return p
@@ -1415,7 +1428,7 @@ func _Cfunc_CBytes(b []byte) unsafe.Pointer {
 
 const cMallocDef = `
 func _Cfunc__CMalloc(n _Ctype_size_t) unsafe.Pointer {
-	return _cgo_runtime_cmalloc(uintptr(n))
+	return _cgo_cmalloc(uint64(n))
 }
 `
 
@@ -1428,6 +1441,50 @@ var builtinDefs = map[string]string{
 	"_CMalloc":  cMallocDef,
 }
 
+// Definitions for C.malloc in Go and in C. We define it ourselves
+// since we call it from functions we define, such as C.CString.
+// Also, we have historically ensured that C.malloc does not return
+// nil even for an allocation of 0.
+
+const cMallocDefGo = `
+//go:cgo_import_static _cgoPREFIX_Cfunc__Cmalloc
+//go:linkname __cgofn__cgoPREFIX_Cfunc__Cmalloc _cgoPREFIX_Cfunc__Cmalloc
+var __cgofn__cgoPREFIX_Cfunc__Cmalloc byte
+var _cgoPREFIX_Cfunc__Cmalloc = unsafe.Pointer(&__cgofn__cgoPREFIX_Cfunc__Cmalloc)
+
+//go:cgo_unsafe_args
+func _cgo_cmalloc(p0 uint64) (r1 unsafe.Pointer) {
+	_cgo_runtime_cgocall(_cgoPREFIX_Cfunc__Cmalloc, uintptr(unsafe.Pointer(&p0)))
+	return
+}
+`
+
+// cMallocDefC defines the C version of C.malloc for the gc compiler.
+// It is defined here because C.CString and friends need a definition.
+// We define it by hand, rather than simply inventing a reference to
+// C.malloc, because <stdlib.h> may not have been included.
+// This is approximately what writeOutputFunc would generate, but
+// skips the cgo_topofstack code (which is only needed if the C code
+// calls back into Go). This also avoids returning nil for an
+// allocation of 0 bytes.
+const cMallocDefC = `
+CGO_NO_SANITIZE_THREAD
+void _cgoPREFIX_Cfunc__Cmalloc(void *v) {
+	struct {
+		unsigned long long p0;
+		void *r1;
+	} PACKED *a = v;
+	void *ret;
+	_cgo_tsan_acquire();
+	ret = malloc(a->p0);
+	if (ret == 0 && a->p0 == 0) {
+		ret = malloc(1);
+	}
+	a->r1 = ret;
+	_cgo_tsan_release();
+}
+`
+
 func (p *Package) cPrologGccgo() string {
 	return strings.Replace(strings.Replace(cPrologGccgo, "PREFIX", cPrefix, -1),
 		"GCCGOSYMBOLPREF", p.gccgoSymbolPrefix(), -1)

src/cmd/compile/doc.go

@@ -60,8 +60,15 @@ Flags:
 	-installsuffix suffix
 		Look for packages in $GOROOT/pkg/$GOOS_$GOARCH_suffix
 		instead of $GOROOT/pkg/$GOOS_$GOARCH.
+	-l
+		Disable inlining.
 	-largemodel
-		Generated code that assumes a large memory model.
+		Generate code that assumes a large memory model.
+	-linkobj file
+		Write linker-specific object to file and compiler-specific
+		object to usual output file (as specified by -o).
+		Without this flag, the -o output is a combination of both
+		linker and compiler input.
 	-memprofile file
 		Write memory profile for the compilation to file.
 	-memprofilerate rate

src/cmd/compile/internal/amd64/galign.go

@@ -25,18 +25,16 @@ func betypeinit() {
 		cmpptr = x86.ACMPL
 	}
 
-	if gc.Ctxt.Flag_dynlink {
-		gc.Thearch.ReservedRegs = append(gc.Thearch.ReservedRegs, x86.REG_R15)
+	if gc.Ctxt.Flag_dynlink || obj.Getgoos() == "nacl" {
+		resvd = append(resvd, x86.REG_R15)
 	}
-}
-
-func Main() {
-	if obj.Getgoos() == "nacl" {
-		resvd = append(resvd, x86.REG_BP, x86.REG_R15)
-	} else if obj.Framepointer_enabled != 0 {
+	if gc.Ctxt.Framepointer_enabled || obj.Getgoos() == "nacl" {
 		resvd = append(resvd, x86.REG_BP)
 	}
+	gc.Thearch.ReservedRegs = resvd
+}
 
+func Main() {
 	gc.Thearch.LinkArch = &x86.Linkamd64
 	if obj.Getgoarch() == "amd64p32" {
 		gc.Thearch.LinkArch = &x86.Linkamd64p32
@@ -51,7 +49,6 @@ func Main() {
 	gc.Thearch.FREGMIN = x86.REG_X0
 	gc.Thearch.FREGMAX = x86.REG_X15
 	gc.Thearch.MAXWIDTH = 1 << 50
-	gc.Thearch.ReservedRegs = resvd
 
 	gc.Thearch.AddIndex = addindex
 	gc.Thearch.Betypeinit = betypeinit

src/cmd/compile/internal/amd64/gsubr.go

@@ -116,7 +116,7 @@ func ginscmp(op gc.Op, t *gc.Type, n1, n2 *gc.Node, likely int) *obj.Prog {
 		base = n1.Left
 	}
 
-	if base.Op == gc.ONAME && base.Class&gc.PHEAP == 0 || n1.Op == gc.OINDREG {
+	if base.Op == gc.ONAME && base.Class != gc.PAUTOHEAP || n1.Op == gc.OINDREG {
 		r1 = *n1
 	} else {
 		gc.Regalloc(&r1, t, n1)
@@ -229,6 +229,8 @@ func gmove(f *gc.Node, t *gc.Node) {
 
 	switch uint32(ft)<<16 | uint32(tt) {
 	default:
+		gc.Dump("f", f)
+		gc.Dump("t", t)
 		gc.Fatalf("gmove %v -> %v", gc.Tconv(f.Type, gc.FmtLong), gc.Tconv(t.Type, gc.FmtLong))
 
 		/*

src/cmd/compile/internal/amd64/reg.go

@@ -32,7 +32,6 @@ package amd64
 
 import (
 	"cmd/compile/internal/gc"
-	"cmd/internal/obj"
 	"cmd/internal/obj/x86"
 )
 
@@ -121,7 +120,7 @@ func BtoR(b uint64) int {
 	b &= 0xffff
 	if gc.Nacl {
 		b &^= (1<<(x86.REG_BP-x86.REG_AX) | 1<<(x86.REG_R15-x86.REG_AX))
-	} else if obj.Framepointer_enabled != 0 {
+	} else if gc.Ctxt.Framepointer_enabled {
 		// BP is part of the calling convention if framepointer_enabled.
 		b &^= (1 << (x86.REG_BP - x86.REG_AX))
 	}

src/cmd/compile/internal/amd64/ssa.go

@@ -878,6 +878,18 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 		gc.Gvarkill(v.Aux.(*gc.Node))
 	case ssa.OpVarLive:
 		gc.Gvarlive(v.Aux.(*gc.Node))
+	case ssa.OpKeepAlive:
+		if !v.Args[0].Type.IsPtrShaped() {
+			v.Fatalf("keeping non-pointer alive %v", v.Args[0])
+		}
+		n, off := gc.AutoVar(v.Args[0])
+		if n == nil {
+			v.Fatalf("KeepLive with non-spilled value %s %s", v, v.Args[0])
+		}
+		if off != 0 {
+			v.Fatalf("KeepLive with non-zero offset spill location %s:%d", n, off)
+		}
+		gc.Gvarlive(n)
 	case ssa.OpAMD64LoweredNilCheck:
 		// Optimization - if the subsequent block has a load or store
 		// at the same address, we don't need to issue this instruction.

src/cmd/compile/internal/arm/gsubr.go

@@ -86,17 +86,8 @@ func split64(n *gc.Node, lo *gc.Node, hi *gc.Node) {
 
 			n = &n1
 
-		case gc.ONAME:
-			if n.Class == gc.PPARAMREF {
-				var n1 gc.Node
-				gc.Cgen(n.Name.Heapaddr, &n1)
-				sclean[nsclean-1] = n1
-				n = &n1
-			}
-
+		case gc.ONAME, gc.OINDREG:
 			// nothing
-		case gc.OINDREG:
-			break
 		}
 
 		*lo = *n

src/cmd/compile/internal/gc/align.go

@@ -55,12 +55,15 @@ func widstruct(errtype *Type, t *Type, o int64, flag int) int64 {
 		}
 		f.Offset = o
 		if f.Nname != nil {
-			// this same stackparam logic is in addrescapes
-			// in typecheck.go.  usually addrescapes runs after
-			// widstruct, in which case we could drop this,
+			// addrescapes has similar code to update these offsets.
+			// Usually addrescapes runs after widstruct,
+			// in which case we could drop this,
 			// but function closure functions are the exception.
-			if f.Nname.Name.Param.Stackparam != nil {
-				f.Nname.Name.Param.Stackparam.Xoffset = o
+			// NOTE(rsc): This comment may be stale.
+			// It's possible the ordering has changed and this is
+			// now the common case. I'm not sure.
+			if f.Nname.Name.Param.Stackcopy != nil {
+				f.Nname.Name.Param.Stackcopy.Xoffset = o
 				f.Nname.Xoffset = 0
 			} else {
 				f.Nname.Xoffset = o

src/cmd/compile/internal/gc/asm_test.go

+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gc
+
+import (
+	"bytes"
+	"fmt"
+	"internal/testenv"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+// TestAssembly checks to make sure the assembly generated for
+// functions contains certain expected instructions.
+// Note: this test will fail if -ssa=0.
+func TestAssembly(t *testing.T) {
+	testenv.MustHaveGoBuild(t)
+	if runtime.GOOS == "windows" {
+		// TODO: remove if we can get "go tool compile -S" to work on windows.
+		t.Skipf("skipping test: recursive windows compile not working")
+	}
+	dir, err := ioutil.TempDir("", "TestAssembly")
+	if err != nil {
+		t.Fatalf("could not create directory: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	for _, test := range asmTests {
+		asm := compileToAsm(dir, test.arch, fmt.Sprintf(template, test.function))
+		// Get rid of code for "".init. Also gets rid of type algorithms & other junk.
+		if i := strings.Index(asm, "\n\"\".init "); i >= 0 {
+			asm = asm[:i+1]
+		}
+		for _, r := range test.regexps {
+			if b, err := regexp.MatchString(r, asm); !b || err != nil {
+				t.Errorf("expected:%s\ngo:%s\nasm:%s\n", r, test.function, asm)
+			}
+		}
+	}
+}
+
+// compile compiles the package pkg for architecture arch and
+// returns the generated assembly.  dir is a scratch directory.
+func compileToAsm(dir, arch, pkg string) string {
+	// Create source.
+	src := filepath.Join(dir, "test.go")
+	f, err := os.Create(src)
+	if err != nil {
+		panic(err)
+	}
+	f.Write([]byte(pkg))
+	f.Close()
+
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Command("go", "tool", "compile", "-S", "-o", filepath.Join(dir, "out.o"), src)
+	cmd.Env = append(cmd.Env, "GOARCH="+arch)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		panic(err)
+	}
+	if s := stderr.String(); s != "" {
+		panic(fmt.Errorf("Stderr = %s\nWant empty", s))
+	}
+	return stdout.String()
+}
+
+// template to convert a function to a full file
+const template = `
+package main
+%s
+`
+
+type asmTest struct {
+	// architecture to compile to
+	arch string
+	// function to compile
+	function string
+	// regexps that must match the generated assembly
+	regexps []string
+}
+
+var asmTests = [...]asmTest{
+	{"amd64", `
+func f(x int) int {
+	return x * 64
+}
+`,
+		[]string{"\tSHLQ\t\\$6,"},
+	},
+	{"amd64", `
+func f(x int) int {
+	return x * 96
+}`,
+		[]string{"\tSHLQ\t\\$5,", "\tLEAQ\t\\(.*\\)\\(.*\\*2\\),"},
+	},
+}

src/cmd/compile/internal/gc/bimport.go

@@ -3,7 +3,8 @@
 // license that can be found in the LICENSE file.
 
 // Binary package import.
-// Based loosely on x/tools/go/importer.
+// See bexport.go for the export data format and how
+// to make a format change.
 
 package gc
 
@@ -24,10 +25,11 @@ type importer struct {
 	buf []byte // reused for reading strings
 
 	// object lists, in order of deserialization
-	strList  []string
-	pkgList  []*Pkg
-	typList  []*Type
-	funcList []*Node // nil entry means already declared
+	strList       []string
+	pkgList       []*Pkg
+	typList       []*Type
+	funcList      []*Node // nil entry means already declared
+	trackAllTypes bool
 
 	// for delayed type verification
 	cmpList []struct{ pt, t *Type }
@@ -59,6 +61,8 @@ func Import(in *bufio.Reader) {
 		Fatalf("importer: invalid encoding format in export data: got %q; want 'c' or 'd'", format)
 	}
 
+	p.trackAllTypes = p.rawByte() == 'a'
+
 	p.posInfoFormat = p.bool()
 
 	// --- generic export data ---
@@ -100,7 +104,9 @@ func Import(in *bufio.Reader) {
 	// --- compiler-specific export data ---
 
 	// read compiler-specific flags
-	importpkg.Safe = p.bool()
+
+	// read but ignore safemode bit (see issue #15772)
+	p.bool() // formerly: importpkg.Safe = p.bool()
 
 	// phase 2
 	objcount = 0
@@ -230,7 +236,7 @@ func (p *importer) pkg() *Pkg {
 	// an empty path denotes the package we are currently importing;
 	// it must be the first package we see
 	if (path == "") != (len(p.pkgList) == 0) {
-		panic(fmt.Sprintf("package path %q for pkg index %d", path, len(p.pkgList)))
+		Fatalf("importer: package path %q for pkg index %d", path, len(p.pkgList))
 	}
 
 	pkg := importpkg
@@ -331,7 +337,9 @@ func (p *importer) pos() {
 
 func (p *importer) newtyp(etype EType) *Type {
 	t := typ(etype)
-	p.typList = append(p.typList, t)
+	if p.trackAllTypes {
+		p.typList = append(p.typList, t)
+	}
 	return t
 }
 
@@ -389,7 +397,13 @@ func (p *importer) typ() *Type {
 		// read underlying type
 		// parser.go:hidden_type
 		t0 := p.typ()
-		p.importtype(t, t0) // parser.go:hidden_import
+		if p.trackAllTypes {
+			// If we track all types, we cannot check equality of previously
+			// imported types until later. Use customized version of importtype.
+			p.importtype(t, t0)
+		} else {
+			importtype(t, t0)
+		}
 
 		// interfaces don't have associated methods
 		if t0.IsInterface() {
@@ -788,16 +802,11 @@ func (p *importer) node() *Node {
 		return n
 
 	case ONAME:
-		if p.bool() {
-			// "_"
-			// TODO(gri) avoid repeated "_" lookup
-			return mkname(Pkglookup("_", localpkg))
-		}
-		return NodSym(OXDOT, typenod(p.typ()), p.fieldSym())
-
-	case OPACK, ONONAME:
 		return mkname(p.sym())
 
+	// case OPACK, ONONAME:
+	// 	unreachable - should have been resolved by typechecking
+
 	case OTYPE:
 		if p.bool() {
 			return mkname(p.sym())
@@ -810,12 +819,9 @@ func (p *importer) node() *Node {
 	// case OCLOSURE:
 	//	unimplemented
 
-	// case OCOMPLIT:
-	//	unimplemented
-
 	case OPTRLIT:
 		n := p.expr()
-		if !p.bool() /* !implicit, i.e. '&' operator*/ {
+		if !p.bool() /* !implicit, i.e. '&' operator */ {
 			if n.Op == OCOMPLIT {
 				// Special case for &T{...}: turn into (*T){...}.
 				n.Right = Nod(OIND, n.Right, nil)
@@ -827,18 +833,15 @@ func (p *importer) node() *Node {
 		return n
 
 	case OSTRUCTLIT:
-		n := Nod(OCOMPLIT, nil, nil)
-		if !p.bool() {
-			n.Right = typenod(p.typ())
-		}
-		n.List.Set(p.elemList())
+		n := Nod(OCOMPLIT, nil, typenod(p.typ()))
+		n.List.Set(p.elemList()) // special handling of field names
 		return n
 
-	case OARRAYLIT, OMAPLIT:
-		n := Nod(OCOMPLIT, nil, nil)
-		if !p.bool() {
-			n.Right = typenod(p.typ())
-		}
+	// case OARRAYLIT, OMAPLIT:
+	// 	unreachable - mapped to case OCOMPLIT below by exporter
+
+	case OCOMPLIT:
+		n := Nod(OCOMPLIT, nil, typenod(p.typ()))
 		n.List.Set(p.exprList())
 		return n
 
@@ -854,14 +857,7 @@ func (p *importer) node() *Node {
 
 	case OXDOT:
 		// see parser.new_dotname
-		obj := p.expr()
-		sel := p.fieldSym()
-		if obj.Op == OPACK {
-			s := restrictlookup(sel.Name, obj.Name.Pkg)
-			obj.Used = true
-			return oldname(s)
-		}
-		return NodSym(OXDOT, obj, sel)
+		return NodSym(OXDOT, p.expr(), p.fieldSym())
 
 	// case ODOTTYPE, ODOTTYPE2:
 	// 	unreachable - mapped to case ODOTTYPE below by exporter
@@ -891,29 +887,18 @@ func (p *importer) node() *Node {
 		n.SetSliceBounds(low, high, max)
 		return n
 
-	case OCOPY, OCOMPLEX:
-		n := builtinCall(op)
-		n.List.Set([]*Node{p.expr(), p.expr()})
-		return n
-
 	// case OCONV, OCONVIFACE, OCONVNOP, OARRAYBYTESTR, OARRAYRUNESTR, OSTRARRAYBYTE, OSTRARRAYRUNE, ORUNESTR:
 	// 	unreachable - mapped to OCONV case below by exporter
 
 	case OCONV:
 		n := Nod(OCALL, typenod(p.typ()), nil)
-		if p.bool() {
-			n.List.Set1(p.expr())
-		} else {
-			n.List.Set(p.exprList())
-		}
+		n.List.Set(p.exprList())
 		return n
 
-	case OREAL, OIMAG, OAPPEND, OCAP, OCLOSE, ODELETE, OLEN, OMAKE, ONEW, OPANIC, ORECOVER, OPRINT, OPRINTN:
+	case OCOPY, OCOMPLEX, OREAL, OIMAG, OAPPEND, OCAP, OCLOSE, ODELETE, OLEN, OMAKE, ONEW, OPANIC, ORECOVER, OPRINT, OPRINTN:
 		n := builtinCall(op)
-		if p.bool() {
-			n.List.Set1(p.expr())
-		} else {
-			n.List.Set(p.exprList())
+		n.List.Set(p.exprList())
+		if op == OAPPEND {
 			n.Isddd = p.bool()
 		}
 		return n
@@ -1053,6 +1038,7 @@ func (p *importer) node() *Node {
 	case OXCASE:
 		markdcl()
 		n := Nod(OXCASE, nil, nil)
+		n.Xoffset = int64(block)
 		n.List.Set(p.exprList())
 		// TODO(gri) eventually we must declare variables for type switch
 		// statements (type switch statements are not yet exported)
@@ -1063,23 +1049,32 @@ func (p *importer) node() *Node {
 	// case OFALL:
 	// 	unreachable - mapped to OXFALL case below by exporter
 
-	case OBREAK, OCONTINUE, OGOTO, OXFALL:
+	case OXFALL:
+		n := Nod(OXFALL, nil, nil)
+		n.Xoffset = int64(block)
+		return n
+
+	case OBREAK, OCONTINUE:
 		left, _ := p.exprsOrNil()
+		if left != nil {
+			left = newname(left.Sym)
+		}
 		return Nod(op, left, nil)
 
 	// case OEMPTY:
 	// 	unreachable - not emitted by exporter
 
-	case OLABEL:
-		n := Nod(OLABEL, p.expr(), nil)
-		n.Left.Sym = dclstack // context, for goto restrictions
+	case OGOTO, OLABEL:
+		n := Nod(op, newname(p.expr().Sym), nil)
+		n.Sym = dclstack // context, for goto restrictions
 		return n
 
 	case OEND:
 		return nil
 
 	default:
-		Fatalf("importer: %s (%d) node not yet supported", op, op)
+		Fatalf("cannot import %s (%d) node\n"+
+			"==> please file an issue and assign to gri@\n", op, op)
 		panic("unreachable") // satisfy compiler
 	}
 }