Commit 26760fc1 authored by Boqun Feng's avatar Boqun Feng Committed by Michael Ellerman

powerpc: atomic: Implement acquire/release/relaxed variants for xchg

Implement xchg{,64}_relaxed and atomic{,64}_xchg_relaxed, based on these
_relaxed variants, release/acquire variants and fully ordered versions
can be built.

Note that xchg{,64}_relaxed and atomic_{,64}_xchg_relaxed are not
compiler barriers.
Signed-off-by: default avatarBoqun Feng <boqun.feng@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent dc53617c
...@@ -177,6 +177,7 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v) ...@@ -177,6 +177,7 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v)
#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
#define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
/** /**
* __atomic_add_unless - add unless the number is a given value * __atomic_add_unless - add unless the number is a given value
...@@ -444,6 +445,7 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v) ...@@ -444,6 +445,7 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v)
#define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
#define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
/** /**
* atomic64_add_unless - add unless the number is a given value * atomic64_add_unless - add unless the number is a given value
......
...@@ -9,21 +9,20 @@ ...@@ -9,21 +9,20 @@
/* /*
* Atomic exchange * Atomic exchange
* *
* Changes the memory location '*ptr' to be val and returns * Changes the memory location '*p' to be val and returns
* the previous value stored there. * the previous value stored there.
*/ */
static __always_inline unsigned long static __always_inline unsigned long
__xchg_u32(volatile void *p, unsigned long val) __xchg_u32_local(volatile void *p, unsigned long val)
{ {
unsigned long prev; unsigned long prev;
__asm__ __volatile__( __asm__ __volatile__(
PPC_ATOMIC_ENTRY_BARRIER
"1: lwarx %0,0,%2 \n" "1: lwarx %0,0,%2 \n"
PPC405_ERR77(0,%2) PPC405_ERR77(0,%2)
" stwcx. %3,0,%2 \n\ " stwcx. %3,0,%2 \n\
bne- 1b" bne- 1b"
PPC_ATOMIC_EXIT_BARRIER
: "=&r" (prev), "+m" (*(volatile unsigned int *)p) : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
: "r" (p), "r" (val) : "r" (p), "r" (val)
: "cc", "memory"); : "cc", "memory");
...@@ -31,42 +30,34 @@ __xchg_u32(volatile void *p, unsigned long val) ...@@ -31,42 +30,34 @@ __xchg_u32(volatile void *p, unsigned long val)
return prev; return prev;
} }
/*
* Atomic exchange
*
* Changes the memory location '*ptr' to be val and returns
* the previous value stored there.
*/
static __always_inline unsigned long static __always_inline unsigned long
__xchg_u32_local(volatile void *p, unsigned long val) __xchg_u32_relaxed(u32 *p, unsigned long val)
{ {
unsigned long prev; unsigned long prev;
__asm__ __volatile__( __asm__ __volatile__(
"1: lwarx %0,0,%2 \n" "1: lwarx %0,0,%2\n"
PPC405_ERR77(0,%2) PPC405_ERR77(0, %2)
" stwcx. %3,0,%2 \n\ " stwcx. %3,0,%2\n"
bne- 1b" " bne- 1b"
: "=&r" (prev), "+m" (*(volatile unsigned int *)p) : "=&r" (prev), "+m" (*p)
: "r" (p), "r" (val) : "r" (p), "r" (val)
: "cc", "memory"); : "cc");
return prev; return prev;
} }
#ifdef CONFIG_PPC64 #ifdef CONFIG_PPC64
static __always_inline unsigned long static __always_inline unsigned long
__xchg_u64(volatile void *p, unsigned long val) __xchg_u64_local(volatile void *p, unsigned long val)
{ {
unsigned long prev; unsigned long prev;
__asm__ __volatile__( __asm__ __volatile__(
PPC_ATOMIC_ENTRY_BARRIER
"1: ldarx %0,0,%2 \n" "1: ldarx %0,0,%2 \n"
PPC405_ERR77(0,%2) PPC405_ERR77(0,%2)
" stdcx. %3,0,%2 \n\ " stdcx. %3,0,%2 \n\
bne- 1b" bne- 1b"
PPC_ATOMIC_EXIT_BARRIER
: "=&r" (prev), "+m" (*(volatile unsigned long *)p) : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
: "r" (p), "r" (val) : "r" (p), "r" (val)
: "cc", "memory"); : "cc", "memory");
...@@ -75,18 +66,18 @@ __xchg_u64(volatile void *p, unsigned long val) ...@@ -75,18 +66,18 @@ __xchg_u64(volatile void *p, unsigned long val)
} }
static __always_inline unsigned long static __always_inline unsigned long
__xchg_u64_local(volatile void *p, unsigned long val) __xchg_u64_relaxed(u64 *p, unsigned long val)
{ {
unsigned long prev; unsigned long prev;
__asm__ __volatile__( __asm__ __volatile__(
"1: ldarx %0,0,%2 \n" "1: ldarx %0,0,%2\n"
PPC405_ERR77(0,%2) PPC405_ERR77(0, %2)
" stdcx. %3,0,%2 \n\ " stdcx. %3,0,%2\n"
bne- 1b" " bne- 1b"
: "=&r" (prev), "+m" (*(volatile unsigned long *)p) : "=&r" (prev), "+m" (*p)
: "r" (p), "r" (val) : "r" (p), "r" (val)
: "cc", "memory"); : "cc");
return prev; return prev;
} }
...@@ -99,14 +90,14 @@ __xchg_u64_local(volatile void *p, unsigned long val) ...@@ -99,14 +90,14 @@ __xchg_u64_local(volatile void *p, unsigned long val)
extern void __xchg_called_with_bad_pointer(void); extern void __xchg_called_with_bad_pointer(void);
static __always_inline unsigned long static __always_inline unsigned long
__xchg(volatile void *ptr, unsigned long x, unsigned int size) __xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
{ {
switch (size) { switch (size) {
case 4: case 4:
return __xchg_u32(ptr, x); return __xchg_u32_local(ptr, x);
#ifdef CONFIG_PPC64 #ifdef CONFIG_PPC64
case 8: case 8:
return __xchg_u64(ptr, x); return __xchg_u64_local(ptr, x);
#endif #endif
} }
__xchg_called_with_bad_pointer(); __xchg_called_with_bad_pointer();
...@@ -114,25 +105,19 @@ __xchg(volatile void *ptr, unsigned long x, unsigned int size) ...@@ -114,25 +105,19 @@ __xchg(volatile void *ptr, unsigned long x, unsigned int size)
} }
static __always_inline unsigned long static __always_inline unsigned long
__xchg_local(volatile void *ptr, unsigned long x, unsigned int size) __xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
{ {
switch (size) { switch (size) {
case 4: case 4:
return __xchg_u32_local(ptr, x); return __xchg_u32_relaxed(ptr, x);
#ifdef CONFIG_PPC64 #ifdef CONFIG_PPC64
case 8: case 8:
return __xchg_u64_local(ptr, x); return __xchg_u64_relaxed(ptr, x);
#endif #endif
} }
__xchg_called_with_bad_pointer(); __xchg_called_with_bad_pointer();
return x; return x;
} }
#define xchg(ptr,x) \
({ \
__typeof__(*(ptr)) _x_ = (x); \
(__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
})
#define xchg_local(ptr,x) \ #define xchg_local(ptr,x) \
({ \ ({ \
__typeof__(*(ptr)) _x_ = (x); \ __typeof__(*(ptr)) _x_ = (x); \
...@@ -140,6 +125,12 @@ __xchg_local(volatile void *ptr, unsigned long x, unsigned int size) ...@@ -140,6 +125,12 @@ __xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
(unsigned long)_x_, sizeof(*(ptr))); \ (unsigned long)_x_, sizeof(*(ptr))); \
}) })
#define xchg_relaxed(ptr, x) \
({ \
__typeof__(*(ptr)) _x_ = (x); \
(__typeof__(*(ptr))) __xchg_relaxed((ptr), \
(unsigned long)_x_, sizeof(*(ptr))); \
})
/* /*
* Compare and exchange - if *p == old, set it to new, * Compare and exchange - if *p == old, set it to new,
* and return the old value of *p. * and return the old value of *p.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment