sync/atomic: add Swap functions

Fixes #5722.

R=golang-dev, khr, cshapiro, rsc, r
CC=golang-dev
https://golang.org/cl/12670045

src/pkg/sync/atomic/64bit_arm.go

@@ -34,3 +34,13 @@ func addUint64(val *uint64, delta uint64) (new uint64) {
 	}
 	return
 }
+
+func swapUint64(addr *uint64, new uint64) (old uint64) {
+	for {
+		old := *addr
+		if CompareAndSwapUint64(addr, old, new) {
+			break
+		}
+	}
+	return
+}

src/pkg/sync/atomic/asm_386.s

@@ -6,6 +6,53 @@
 
 #include "../../../cmd/ld/textflag.h"
 
+TEXT ·SwapInt32(SB),NOSPLIT,$0-12
+	JMP	·SwapUint32(SB)
+
+TEXT ·SwapUint32(SB),NOSPLIT,$0-12
+	MOVL	addr+0(FP), BP
+	MOVL	new+4(FP), AX
+	XCHGL	AX, 0(BP)
+	MOVL	AX, new+8(FP)
+	RET
+
+TEXT ·SwapInt64(SB),NOSPLIT,$0-20
+	JMP	·SwapUint64(SB)
+
+TEXT ·SwapUint64(SB),NOSPLIT,$0-20
+	// no XCHGQ so use CMPXCHG8B loop
+	MOVL	addr+0(FP), BP
+	TESTL	$7, BP
+	JZ	2(PC)
+	MOVL	0, AX // crash with nil ptr deref
+	// CX:BX = new
+	MOVL	new_lo+4(FP), BX
+	MOVL	new_hi+8(FP), CX
+	// DX:AX = *addr
+	MOVL	0(BP), AX
+	MOVL	4(BP), DX
+swaploop:
+	// if *addr == DX:AX
+	//	*addr = CX:BX
+	// else
+	//	DX:AX = *addr
+	// all in one instruction
+	LOCK
+	CMPXCHG8B	0(BP)
+	JNZ	swaploop
+
+	// success
+	// return DX:AX
+	MOVL	AX, new_lo+12(FP)
+	MOVL	DX, new_hi+16(FP)
+	RET
+
+TEXT ·SwapUintptr(SB),NOSPLIT,$0-12
+	JMP	·SwapUint32(SB)
+
+TEXT ·SwapPointer(SB),NOSPLIT,$0-12
+	JMP	·SwapUint32(SB)
+
 TEXT ·CompareAndSwapInt32(SB),NOSPLIT,$0-13
 	JMP	·CompareAndSwapUint32(SB)
 

src/pkg/sync/atomic/asm_amd64.s

@@ -6,6 +6,32 @@
 
 #include "../../../cmd/ld/textflag.h"
 
+TEXT ·SwapInt32(SB),NOSPLIT,$0-20
+	JMP	·SwapUint32(SB)
+
+TEXT ·SwapUint32(SB),NOSPLIT,$0-20
+	MOVQ	addr+0(FP), BP
+	MOVL	new+8(FP), AX
+	XCHGL	AX, 0(BP)
+	MOVL	AX, new+16(FP)
+	RET
+
+TEXT ·SwapInt64(SB),NOSPLIT,$0-24
+	JMP	·SwapUint64(SB)
+
+TEXT ·SwapUint64(SB),NOSPLIT,$0-24
+	MOVQ	addr+0(FP), BP
+	MOVQ	new+8(FP), AX
+	XCHGQ	AX, 0(BP)
+	MOVQ	AX, new+16(FP)
+	RET
+
+TEXT ·SwapUintptr(SB),NOSPLIT,$0-24
+	JMP	·SwapUint64(SB)
+
+TEXT ·SwapPointer(SB),NOSPLIT,$0-24
+	JMP	·SwapUint64(SB)
+
 TEXT ·CompareAndSwapInt32(SB),NOSPLIT,$0-17
 	JMP	·CompareAndSwapUint32(SB)
 

src/pkg/sync/atomic/asm_arm.s

@@ -91,6 +91,37 @@ add64loop:
 	MOVW	R5, rethi+16(FP)
 	RET
 
+TEXT ·armSwapUint32(SB),NOSPLIT,$0-12
+	MOVW	addr+0(FP), R1
+	MOVW	new+4(FP), R2
+swaploop:
+	// LDREX and STREX were introduced in ARM 6.
+	LDREX	(R1), R3
+	STREX	R2, (R1), R0
+	CMP	$0, R0
+	BNE	swaploop
+	MOVW	R3, old+8(FP)
+	RET
+
+TEXT ·armSwapUint64(SB),NOSPLIT,$0-20
+	BL	fastCheck64<>(SB)
+	MOVW	addr+0(FP), R1
+	// make unaligned atomic access panic
+	AND.S	$7, R1, R2
+	BEQ 	2(PC)
+	MOVW	R2, (R2)
+	MOVW	newlo+4(FP), R2
+	MOVW	newhi+8(FP), R3
+swap64loop:
+	// LDREXD and STREXD were introduced in ARM 11.
+	LDREXD	(R1), R4	// loads R4 and R5
+	STREXD	R2, (R1), R0	// stores R2 and R3
+	CMP	$0, R0
+	BNE	swap64loop
+	MOVW	R4, oldlo+12(FP)
+	MOVW	R5, oldhi+16(FP)
+	RET
+
 TEXT ·armLoadUint64(SB),NOSPLIT,$0-12
 	BL	fastCheck64<>(SB)
 	MOVW	addr+0(FP), R1

src/pkg/sync/atomic/asm_freebsd_arm.s

@@ -28,6 +28,18 @@ TEXT ·AddUint32(SB),NOSPLIT,$0
 TEXT ·AddUintptr(SB),NOSPLIT,$0
 	B ·AddUint32(SB)
 
+TEXT ·SwapInt32(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
+TEXT ·SwapUint32(SB),NOSPLIT,$0
+	B ·armSwapUint32(SB)
+
+TEXT ·SwapUintptr(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
+TEXT ·SwapPointer(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0
 	B ·CompareAndSwapUint64(SB)
 
@@ -40,6 +52,12 @@ TEXT ·AddInt64(SB),NOSPLIT,$0
 TEXT ·AddUint64(SB),NOSPLIT,$0
 	B ·addUint64(SB)
 
+TEXT ·SwapInt64(SB),NOSPLIT,$0
+	B ·swapUint64(SB)
+
+TEXT ·SwapUint64(SB),NOSPLIT,$0
+	B ·swapUint64(SB)
+
 TEXT ·LoadInt32(SB),NOSPLIT,$0
 	B ·LoadUint32(SB)
 

src/pkg/sync/atomic/asm_linux_arm.s

@@ -76,6 +76,26 @@ addloop1:
 TEXT ·AddUintptr(SB),NOSPLIT,$0
 	B	·AddUint32(SB)
 
+TEXT ·SwapInt32(SB),NOSPLIT,$0
+	B	·SwapUint32(SB)
+
+// Implement using kernel cas for portability.
+TEXT ·SwapUint32(SB),NOSPLIT,$0-12
+	MOVW	addr+0(FP), R2
+	MOVW	new+4(FP), R1
+swaploop1:
+	MOVW	0(R2), R0
+	BL	cas<>(SB)
+	BCC	swaploop1
+	MOVW	R0, old+8(FP)
+	RET
+
+TEXT ·SwapUintptr(SB),NOSPLIT,$0
+	B	·SwapUint32(SB)
+
+TEXT ·SwapPointer(SB),NOSPLIT,$0
+	B	·SwapUint32(SB)
+
 TEXT cas64<>(SB),NOSPLIT,$0
 	MOVW	$0xffff0f60, PC // __kuser_cmpxchg64: Linux-3.1 and above
 
@@ -148,6 +168,12 @@ TEXT ·AddInt64(SB),NOSPLIT,$0
 TEXT ·AddUint64(SB),NOSPLIT,$0
 	B	·addUint64(SB)
 
+TEXT ·SwapInt64(SB),NOSPLIT,$0
+	B	·swapUint64(SB)
+
+TEXT ·SwapUint64(SB),NOSPLIT,$0
+	B	·swapUint64(SB)
+
 TEXT ·LoadInt32(SB),NOSPLIT,$0
 	B	·LoadUint32(SB)
 

src/pkg/sync/atomic/asm_netbsd_arm.s

@@ -28,6 +28,18 @@ TEXT ·AddUint32(SB),NOSPLIT,$0
 TEXT ·AddUintptr(SB),NOSPLIT,$0
 	B ·AddUint32(SB)
 
+TEXT ·SwapInt32(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
+TEXT ·SwapUint32(SB),NOSPLIT,$0
+	B ·armSwapUint32(SB)
+
+TEXT ·SwapUintptr(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
+TEXT ·SwapPointer(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0
 	B ·CompareAndSwapUint64(SB)
 
@@ -40,6 +52,12 @@ TEXT ·AddInt64(SB),NOSPLIT,$0
 TEXT ·AddUint64(SB),NOSPLIT,$0
 	B ·addUint64(SB)
 
+TEXT ·SwapInt64(SB),NOSPLIT,$0
+	B ·swapUint64(SB)
+
+TEXT ·SwapUint64(SB),NOSPLIT,$0
+	B ·swapUint64(SB)
+
 TEXT ·LoadInt32(SB),NOSPLIT,$0
 	B ·LoadUint32(SB)
 

src/pkg/sync/atomic/doc.go

@@ -13,6 +13,13 @@
 // Share memory by communicating;
 // don't communicate by sharing memory.
 //
+// The swap operation, implemented by the SwapT functions, is the atomic
+// equivalent of:
+//
+//	old = *addr
+//	*addr = new
+//	return old
+//
 // The compare-and-swap operation, implemented by the CompareAndSwapT
 // functions, is the atomic equivalent of:
 //
@@ -45,6 +52,24 @@ import (
 // variable or in an allocated struct or slice can be relied upon to be
 // 64-bit aligned.
 
+// SwapInt32 atomically stores new into *addr and returns the previous *addr value.
+func SwapInt32(addr *int32, new int32) (old int32)
+
+// SwapInt64 atomically stores new into *addr and returns the previous *addr value.
+func SwapInt64(addr *int64, new int64) (old int64)
+
+// SwapUint32 atomically stores new into *addr and returns the previous *addr value.
+func SwapUint32(addr *uint32, new uint32) (old uint32)
+
+// SwapUint64 atomically stores new into *addr and returns the previous *addr value.
+func SwapUint64(addr *uint64, new uint64) (old uint64)
+
+// SwapUintptr atomically stores new into *addr and returns the previous *addr value.
+func SwapUintptr(addr *uintptr, new uintptr) (old uintptr)
+
+// SwapPointer atomically stores new into *addr and returns the previous *addr value.
+func SwapPointer(addr *unsafe.Pointer, new unsafe.Pointer) (old unsafe.Pointer)
+
 // CompareAndSwapInt32 executes the compare-and-swap operation for an int32 value.
 func CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool)
 

src/pkg/sync/atomic/race.go

@@ -20,6 +20,54 @@ import (
 
 var mtx uint32 = 1 // same for all
 
+func SwapInt32(addr *int32, new int32) (old int32) {
+	return int32(SwapUint32((*uint32)(unsafe.Pointer(addr)), uint32(new)))
+}
+
+func SwapUint32(addr *uint32, new uint32) (old uint32) {
+	_ = *addr
+	runtime.RaceSemacquire(&mtx)
+	runtime.RaceRead(unsafe.Pointer(addr))
+	runtime.RaceAcquire(unsafe.Pointer(addr))
+	old = *addr
+	*addr = new
+	runtime.RaceReleaseMerge(unsafe.Pointer(addr))
+	runtime.RaceSemrelease(&mtx)
+	return
+}
+
+func SwapInt64(addr *int64, new int64) (old int64) {
+	return int64(SwapUint64((*uint64)(unsafe.Pointer(addr)), uint64(new)))
+}
+
+func SwapUint64(addr *uint64, new uint64) (old uint64) {
+	_ = *addr
+	runtime.RaceSemacquire(&mtx)
+	runtime.RaceRead(unsafe.Pointer(addr))
+	runtime.RaceAcquire(unsafe.Pointer(addr))
+	old = *addr
+	*addr = new
+	runtime.RaceReleaseMerge(unsafe.Pointer(addr))
+	runtime.RaceSemrelease(&mtx)
+	return
+}
+
+func SwapUintptr(addr *uintptr, new uintptr) (old uintptr) {
+	return uintptr(SwapPointer((*unsafe.Pointer)(unsafe.Pointer(addr)), unsafe.Pointer(new)))
+}
+
+func SwapPointer(addr *unsafe.Pointer, new unsafe.Pointer) (old unsafe.Pointer) {
+	_ = *addr
+	runtime.RaceSemacquire(&mtx)
+	runtime.RaceRead(unsafe.Pointer(addr))
+	runtime.RaceAcquire(unsafe.Pointer(addr))
+	old = *addr
+	*addr = new
+	runtime.RaceReleaseMerge(unsafe.Pointer(addr))
+	runtime.RaceSemrelease(&mtx)
+	return
+}
+
 func CompareAndSwapInt32(val *int32, old, new int32) bool {
 	return CompareAndSwapUint32((*uint32)(unsafe.Pointer(val)), uint32(old), uint32(new))
 }