[dev.cc] runtime: fix lfstack for amd64 addresses in top half of addr space

While we are here, add the linux/power64 version.

LGTM=austin
R=austin
CC=aram, dvyukov, golang-codereviews
https://golang.org/cl/177750043

src/runtime/lfstack.go

@@ -9,23 +9,12 @@ package runtime
 
 import "unsafe"
 
-const (
-	// lfPtrBits and lfCountMask are defined in lfstack_*.go.
-	lfPtrMask = 1<<lfPtrBits - 1
-)
-
 func lfstackpush(head *uint64, node *lfnode) {
-	unode := uintptr(unsafe.Pointer(node))
-	if unode&^lfPtrMask != 0 {
-		print("p=", node, "\n")
-		gothrow("lfstackpush: invalid pointer")
-	}
-
 	node.pushcnt++
-	new := uint64(unode) | (uint64(node.pushcnt)&lfCountMask)<<lfPtrBits
+	new := lfstackPack(node, node.pushcnt)
 	for {
 		old := atomicload64(head)
-		node.next = (*lfnode)(unsafe.Pointer(uintptr(old & lfPtrMask)))
+		node.next, _ = lfstackUnpack(old)
 		if cas64(head, old, new) {
 			break
 		}
@@ -38,11 +27,11 @@ func lfstackpop(head *uint64) unsafe.Pointer {
 		if old == 0 {
 			return nil
 		}
-		node := (*lfnode)(unsafe.Pointer(uintptr(old & lfPtrMask)))
+		node, _ := lfstackUnpack(old)
 		node2 := (*lfnode)(atomicloadp(unsafe.Pointer(&node.next)))
 		new := uint64(0)
 		if node2 != nil {
-			new = uint64(uintptr(unsafe.Pointer(node2))) | uint64(node2.pushcnt&lfCountMask)<<lfPtrBits
+			new = lfstackPack(node2, node2.pushcnt)
 		}
 		if cas64(head, old, new) {
 			return unsafe.Pointer(node)

src/runtime/lfstack_32bit.go

@@ -6,8 +6,16 @@
 
 package runtime
 
+import "unsafe"
+
 // On 32-bit systems, the stored uint64 has a 32-bit pointer and 32-bit count.
-const (
-	lfPtrBits   = 32
-	lfCountMask = 1<<32 - 1
-)
+
+func lfstackPack(node *lfnode, cnt uintptr) uint64 {
+	return uint64(uintptr(unsafe.Pointer(node)))<<32 | uint64(cnt)
+}
+
+func lfstackUnpack(val uint64) (node *lfnode, cnt uintptr) {
+	node = (*lfnode)(unsafe.Pointer(uintptr(val >> 32)))
+	cnt = uintptr(val)
+	return
+}

src/runtime/lfstack_amd64.go

@@ -4,9 +4,21 @@
 
 package runtime
 
-// Amd64 uses 48-bit virtual addresses, 47-th bit is used as kernel/user flag.
-// So we use 17msb of pointers as ABA counter.
-const (
-	lfPtrBits   = 47
-	lfCountMask = 1<<17 - 1
-)
+import "unsafe"
+
+// On AMD64, virtual addresses are 48-bit numbers sign extended to 64.
+// We shift the address left 16 to eliminate the sign extended part and make
+// room in the bottom for the count.
+// In addition to the 16 bits taken from the top, we can take 3 from the
+// bottom, because node must be pointer-aligned, giving a total of 19 bits
+// of count.
+
+func lfstackPack(node *lfnode, cnt uintptr) uint64 {
+	return uint64(uintptr(unsafe.Pointer(node)))<<16 | uint64(cnt&(1<<19-1))
+}
+
+func lfstackUnpack(val uint64) (node *lfnode, cnt uintptr) {
+	node = (*lfnode)(unsafe.Pointer(uintptr(int64(val) >> 19 << 3)))
+	cnt = uintptr(val & (1<<19 - 1))
+	return
+}

src/runtime/lfstack_linux_power64x.go

+// Copyright 2014 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build power64 power64le
+// +build linux
+
+package runtime
+
+import "unsafe"
+
+// On Power64, Linux limits the user address space to 43 bits.
+// (https://www.kernel.org/doc/ols/2001/ppc64.pdf)
+// In addition to the 21 bits taken from the top, we can take 3 from the
+// bottom, because node must be pointer-aligned, giving a total of 24 bits
+// of count.
+
+func lfstackPack(node *lfnode, cnt uintptr) uint64 {
+	return uint64(uintptr(unsafe.Pointer(node)))<<21 | uint64(cnt&(1<<24-1))
+}
+
+func lfstackUnpack(val uint64) (node *lfnode, cnt uintptr) {
+	node = (*lfnode)(unsafe.Pointer(uintptr(val >> 24 << 3)))
+	cnt = uintptr(val & (1<<24 - 1))
+	return
+}