redonkable
/
alistair23-linux
1
0
Fork 0
Code Releases Activity

arm64: cmpxchg: avoid memory barrier on comparison failure 

cmpxchg doesn't require memory barrier semantics when the value
comparison fails, so make the barrier conditional on success.

Reviewed-by: Steve Capper <steve.capper@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
hifive-unleashed-5.1
Will Deacon 2015-05-29 14:47:59 +01:00
parent 0bc671d3f4
commit 4e39715f4b
1 changed files with 22 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
arch/arm64/include/asm/atomic_ll_sc.h
View File

@ -97,19 +97,18 @@ __LL_SC_PREFIX(atomic_cmpxchg(atomic_t *ptr, int old, int new))
unsigned long tmp;
int oldval;
smp_mb();
asm volatile("// atomic_cmpxchg\n"
"1: ldxr %w1, %2\n"
" eor %w0, %w1, %w3\n"
" cbnz %w0, 2f\n"
" stxr %w0, %w4, %2\n"
" stlxr %w0, %w4, %2\n"
" cbnz %w0, 1b\n"
" dmb ish\n"
"2:"
: "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
: "Lr" (old), "r" (new));
: "Lr" (old), "r" (new)
: "memory");
smp_mb();
return oldval;
}
__LL_SC_EXPORT(atomic_cmpxchg);
@ -174,19 +173,18 @@ __LL_SC_PREFIX(atomic64_cmpxchg(atomic64_t *ptr, long old, long new))
long oldval;
unsigned long res;
smp_mb();
asm volatile("// atomic64_cmpxchg\n"
"1: ldxr %1, %2\n"
" eor %0, %1, %3\n"
" cbnz %w0, 2f\n"
" stxr %w0, %4, %2\n"
" stlxr %w0, %4, %2\n"
" cbnz %w0, 1b\n"
" dmb ish\n"
"2:"
: "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
: "Lr" (old), "r" (new));
: "Lr" (old), "r" (new)
: "memory");
smp_mb();
return oldval;
}
__LL_SC_EXPORT(atomic64_cmpxchg);
@ -213,7 +211,7 @@ __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
}
__LL_SC_EXPORT(atomic64_dec_if_positive);
#define __CMPXCHG_CASE(w, sz, name, mb, cl) \
#define __CMPXCHG_CASE(w, sz, name, mb, rel, cl) \
__LL_SC_INLINE unsigned long \
__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
unsigned long old, \
@ -222,11 +220,10 @@ __LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
unsigned long tmp, oldval; \
\
asm volatile( \
" " #mb "\n" \
"1: ldxr" #sz "\t%" #w "[oldval], %[v]\n" \
" eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
" cbnz %" #w "[tmp], 2f\n" \
" stxr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
" st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
" cbnz %w[tmp], 1b\n" \
" " #mb "\n" \
" mov %" #w "[oldval], %" #w "[old]\n" \
@ -240,18 +237,18 @@ __LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
} \
__LL_SC_EXPORT(__cmpxchg_case_##name);
__CMPXCHG_CASE(w, b, 1, , )
__CMPXCHG_CASE(w, h, 2, , )
__CMPXCHG_CASE(w, , 4, , )
__CMPXCHG_CASE( , , 8, , )
__CMPXCHG_CASE(w, b, mb_1, dmb ish, "memory")
__CMPXCHG_CASE(w, h, mb_2, dmb ish, "memory")
__CMPXCHG_CASE(w, , mb_4, dmb ish, "memory")
__CMPXCHG_CASE( , , mb_8, dmb ish, "memory")
__CMPXCHG_CASE(w, b, 1, , , )
__CMPXCHG_CASE(w, h, 2, , , )
__CMPXCHG_CASE(w, , 4, , , )
__CMPXCHG_CASE( , , 8, , , )
__CMPXCHG_CASE(w, b, mb_1, dmb ish, l, "memory")
__CMPXCHG_CASE(w, h, mb_2, dmb ish, l, "memory")
__CMPXCHG_CASE(w, , mb_4, dmb ish, l, "memory")
__CMPXCHG_CASE( , , mb_8, dmb ish, l, "memory")
#undef __CMPXCHG_CASE
#define __CMPXCHG_DBL(name, mb, cl) \
#define __CMPXCHG_DBL(name, mb, rel, cl) \
__LL_SC_INLINE int \
__LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \
unsigned long old2, \
@ -262,13 +259,12 @@ __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \
unsigned long tmp, ret; \
\
asm volatile("// __cmpxchg_double" #name "\n" \
" " #mb "\n" \
"1: ldxp %0, %1, %2\n" \
" eor %0, %0, %3\n" \
" eor %1, %1, %4\n" \
" orr %1, %0, %1\n" \
" cbnz %1, 2f\n" \
" stxp %w0, %5, %6, %2\n" \
" st" #rel "xp %w0, %5, %6, %2\n" \
" cbnz %w0, 1b\n" \
" " #mb "\n" \
"2:" \
@ -280,8 +276,8 @@ __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \
} \
__LL_SC_EXPORT(__cmpxchg_double##name);
__CMPXCHG_DBL( , , )
__CMPXCHG_DBL(_mb, dmb ish, "memory")
__CMPXCHG_DBL( , , , )
__CMPXCHG_DBL(_mb, dmb ish, l, "memory")
#undef __CMPXCHG_DBL