cmd/compile: make HasHeapPointer recursive

Currently (*Type).HasHeapPointer only ignores pointers go:notinheap
types if the type itself is a pointer to a go:notinheap type. However,
if it's some other type that contains pointers where all of those
pointers are go:notinheap, it will conservatively return true. As a
result, we'll use write barriers where they aren't needed, for example
calling typedmemmove instead of just memmove on structs that contain
only go:notinheap pointers.

Fix this by making HasHeapPointer walk the whole type looking for
pointers that aren't marked go:notinheap.

Change-Id: Ib8c6abf6f7a20f34969d1d402c5498e0b990be59
Reviewed-on: https://go-review.googlesource.com/73412
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>

src/cmd/compile/internal/types/type.go

@@ -1364,7 +1364,14 @@ func (t *Type) IsUntyped() bool {
 	return false
 }
 
+// TODO(austin): We probably only need HasHeapPointer. See
+// golang.org/cl/73412 for discussion.
+
 func Haspointers(t *Type) bool {
+	return Haspointers1(t, false)
+}
+
+func Haspointers1(t *Type, ignoreNotInHeap bool) bool {
 	switch t.Etype {
 	case TINT, TUINT, TINT8, TUINT8, TINT16, TUINT16, TINT32, TUINT32, TINT64,
 		TUINT64, TUINTPTR, TFLOAT32, TFLOAT64, TCOMPLEX64, TCOMPLEX128, TBOOL:
@@ -1374,28 +1381,28 @@ func Haspointers(t *Type) bool {
 		if t.NumElem() == 0 { // empty array has no pointers
 			return false
 		}
-		return Haspointers(t.Elem())
+		return Haspointers1(t.Elem(), ignoreNotInHeap)
 
 	case TSTRUCT:
 		for _, t1 := range t.Fields().Slice() {
-			if Haspointers(t1.Type) {
+			if Haspointers1(t1.Type, ignoreNotInHeap) {
 				return true
 			}
 		}
 		return false
+
+	case TPTR32, TPTR64:
+		return !(ignoreNotInHeap && t.Elem().NotInHeap())
 	}
 
 	return true
 }
 
 // HasHeapPointer returns whether t contains a heap pointer.
-// This is used for write barrier insertion, so we ignore
+// This is used for write barrier insertion, so it ignores
 // pointers to go:notinheap types.
 func (t *Type) HasHeapPointer() bool {
-	if t.IsPtr() && t.Elem().NotInHeap() {
-		return false
-	}
-	return Haspointers(t)
+	return Haspointers1(t, true)
 }
 
 func (t *Type) Symbol() *obj.LSym {

test/notinheap3.go

+// errorcheck -+ -0 -l -d=wb
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Test write barrier elimination for notinheap.
+
+package p
+
+type t1 struct {
+	x *nih
+	y [1024]byte // Prevent write decomposition
+}
+
+type t2 struct {
+	x *ih
+	y [1024]byte
+}
+
+//go:notinheap
+type nih struct {
+	x uintptr
+}
+
+type ih struct { // In-heap type
+	x uintptr
+}
+
+var (
+	v1 t1
+	v2 t2
+)
+
+func f() {
+	// Test direct writes
+	v1.x = nil // no barrier
+	v2.x = nil // ERROR "write barrier"
+}
+
+func g() {
+	// Test aggregate writes
+	v1 = t1{x: nil} // no barrier
+	v2 = t2{x: nil} // ERROR "write barrier"
+}