Commit 4d1dbfe6 authored by Paul Burton's avatar Paul Burton

MIPS: atomic: Emit Loongson3 sync workarounds within asm

Generate the sync instructions required to workaround Loongson3 LL/SC
errata within inline asm blocks, which feels a little safer than doing
it from C where strictly speaking the compiler would be well within its
rights to insert a memory access between the separate asm statements we
previously had, containing sync & ll instructions respectively.
Signed-off-by: default avatarPaul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
Cc: Huacai Chen <chenhc@lemote.com>
Cc: Jiaxun Yang <jiaxun.yang@flygoat.com>
Cc: linux-kernel@vger.kernel.org
parent a38ee6bb
...@@ -21,6 +21,7 @@ ...@@ -21,6 +21,7 @@
#include <asm/cpu-features.h> #include <asm/cpu-features.h>
#include <asm/cmpxchg.h> #include <asm/cmpxchg.h>
#include <asm/llsc.h> #include <asm/llsc.h>
#include <asm/sync.h>
#include <asm/war.h> #include <asm/war.h>
#define ATOMIC_INIT(i) { (i) } #define ATOMIC_INIT(i) { (i) }
...@@ -56,10 +57,10 @@ static __inline__ void pfx##_##op(type i, pfx##_t * v) \ ...@@ -56,10 +57,10 @@ static __inline__ void pfx##_##op(type i, pfx##_t * v) \
return; \ return; \
} \ } \
\ \
loongson_llsc_mb(); \
__asm__ __volatile__( \ __asm__ __volatile__( \
" .set push \n" \ " .set push \n" \
" .set " MIPS_ISA_LEVEL " \n" \ " .set " MIPS_ISA_LEVEL " \n" \
" " __SYNC(full, loongson3_war) " \n" \
"1: " #ll " %0, %1 # " #pfx "_" #op " \n" \ "1: " #ll " %0, %1 # " #pfx "_" #op " \n" \
" " #asm_op " %0, %2 \n" \ " " #asm_op " %0, %2 \n" \
" " #sc " %0, %1 \n" \ " " #sc " %0, %1 \n" \
...@@ -85,10 +86,10 @@ static __inline__ type pfx##_##op##_return_relaxed(type i, pfx##_t * v) \ ...@@ -85,10 +86,10 @@ static __inline__ type pfx##_##op##_return_relaxed(type i, pfx##_t * v) \
return result; \ return result; \
} \ } \
\ \
loongson_llsc_mb(); \
__asm__ __volatile__( \ __asm__ __volatile__( \
" .set push \n" \ " .set push \n" \
" .set " MIPS_ISA_LEVEL " \n" \ " .set " MIPS_ISA_LEVEL " \n" \
" " __SYNC(full, loongson3_war) " \n" \
"1: " #ll " %1, %2 # " #pfx "_" #op "_return\n" \ "1: " #ll " %1, %2 # " #pfx "_" #op "_return\n" \
" " #asm_op " %0, %1, %3 \n" \ " " #asm_op " %0, %1, %3 \n" \
" " #sc " %0, %2 \n" \ " " #sc " %0, %2 \n" \
...@@ -117,10 +118,10 @@ static __inline__ type pfx##_fetch_##op##_relaxed(type i, pfx##_t * v) \ ...@@ -117,10 +118,10 @@ static __inline__ type pfx##_fetch_##op##_relaxed(type i, pfx##_t * v) \
return result; \ return result; \
} \ } \
\ \
loongson_llsc_mb(); \
__asm__ __volatile__( \ __asm__ __volatile__( \
" .set push \n" \ " .set push \n" \
" .set " MIPS_ISA_LEVEL " \n" \ " .set " MIPS_ISA_LEVEL " \n" \
" " __SYNC(full, loongson3_war) " \n" \
"1: " #ll " %1, %2 # " #pfx "_fetch_" #op "\n" \ "1: " #ll " %1, %2 # " #pfx "_fetch_" #op "\n" \
" " #asm_op " %0, %1, %3 \n" \ " " #asm_op " %0, %1, %3 \n" \
" " #sc " %0, %2 \n" \ " " #sc " %0, %2 \n" \
...@@ -200,10 +201,10 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) ...@@ -200,10 +201,10 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
if (kernel_uses_llsc) { if (kernel_uses_llsc) {
int temp; int temp;
loongson_llsc_mb();
__asm__ __volatile__( __asm__ __volatile__(
" .set push \n" " .set push \n"
" .set "MIPS_ISA_LEVEL" \n" " .set "MIPS_ISA_LEVEL" \n"
" " __SYNC(full, loongson3_war) " \n"
"1: ll %1, %2 # atomic_sub_if_positive\n" "1: ll %1, %2 # atomic_sub_if_positive\n"
" .set pop \n" " .set pop \n"
" subu %0, %1, %3 \n" " subu %0, %1, %3 \n"
...@@ -213,7 +214,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) ...@@ -213,7 +214,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
" .set "MIPS_ISA_LEVEL" \n" " .set "MIPS_ISA_LEVEL" \n"
" sc %1, %2 \n" " sc %1, %2 \n"
"\t" __SC_BEQZ "%1, 1b \n" "\t" __SC_BEQZ "%1, 1b \n"
"2: \n" "2: " __SYNC(full, loongson3_war) " \n"
" .set pop \n" " .set pop \n"
: "=&r" (result), "=&r" (temp), : "=&r" (result), "=&r" (temp),
"+" GCC_OFF_SMALL_ASM() (v->counter) "+" GCC_OFF_SMALL_ASM() (v->counter)
...@@ -229,6 +230,13 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) ...@@ -229,6 +230,13 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
raw_local_irq_restore(flags); raw_local_irq_restore(flags);
} }
/*
* In the Loongson3 workaround case we already have a completion
* barrier at 2: above, which is needed due to the bltz that can branch
* to code outside of the LL/SC loop. As such, we don't need to emit
* another barrier here.
*/
if (!__SYNC_loongson3_war)
smp_llsc_mb(); smp_llsc_mb();
return result; return result;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment