49e9cf3f0c
According to memory-barriers.txt:
> Any atomic operation that modifies some state in memory and returns
> information about the state (old or new) implies an SMP-conditional
> general memory barrier (smp_mb()) on each side of the actual
> operation ...
Which mean these operations should be fully ordered. However on PPC,
PPC_ATOMIC_ENTRY_BARRIER is the barrier before the actual operation,
which is currently "lwsync" if SMP=y. The leading "lwsync" can not
guarantee fully ordered atomics, according to Paul Mckenney:
https://lkml.org/lkml/2015/10/14/970
To fix this, we define PPC_ATOMIC_ENTRY_BARRIER as "sync" to guarantee
the fully-ordered semantics.
This also makes futex atomics fully ordered, which can avoid possible
memory ordering problems if userspace code relies on futex system call
for fully ordered semantics.
Fixes: b97021f855 ("powerpc: Fix atomic_xxx_return barrier semantics")
Cc: stable@vger.kernel.org # 3.2+
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
58 lines
1.4 KiB
C
58 lines
1.4 KiB
C
#ifndef _ASM_POWERPC_SYNCH_H
|
|
#define _ASM_POWERPC_SYNCH_H
|
|
#ifdef __KERNEL__
|
|
|
|
#include <linux/stringify.h>
|
|
#include <asm/feature-fixups.h>
|
|
|
|
#if defined(__powerpc64__) || defined(CONFIG_PPC_E500MC)
|
|
#define __SUBARCH_HAS_LWSYNC
|
|
#endif
|
|
|
|
#ifndef __ASSEMBLY__
|
|
extern unsigned int __start___lwsync_fixup, __stop___lwsync_fixup;
|
|
extern void do_lwsync_fixups(unsigned long value, void *fixup_start,
|
|
void *fixup_end);
|
|
extern void do_final_fixups(void);
|
|
|
|
static inline void eieio(void)
|
|
{
|
|
__asm__ __volatile__ ("eieio" : : : "memory");
|
|
}
|
|
|
|
static inline void isync(void)
|
|
{
|
|
__asm__ __volatile__ ("isync" : : : "memory");
|
|
}
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#if defined(__powerpc64__)
|
|
# define LWSYNC lwsync
|
|
#elif defined(CONFIG_E500)
|
|
# define LWSYNC \
|
|
START_LWSYNC_SECTION(96); \
|
|
sync; \
|
|
MAKE_LWSYNC_SECTION_ENTRY(96, __lwsync_fixup);
|
|
#else
|
|
# define LWSYNC sync
|
|
#endif
|
|
|
|
#ifdef CONFIG_SMP
|
|
#define __PPC_ACQUIRE_BARRIER \
|
|
START_LWSYNC_SECTION(97); \
|
|
isync; \
|
|
MAKE_LWSYNC_SECTION_ENTRY(97, __lwsync_fixup);
|
|
#define PPC_ACQUIRE_BARRIER "\n" stringify_in_c(__PPC_ACQUIRE_BARRIER)
|
|
#define PPC_RELEASE_BARRIER stringify_in_c(LWSYNC) "\n"
|
|
#define PPC_ATOMIC_ENTRY_BARRIER "\n" stringify_in_c(sync) "\n"
|
|
#define PPC_ATOMIC_EXIT_BARRIER "\n" stringify_in_c(sync) "\n"
|
|
#else
|
|
#define PPC_ACQUIRE_BARRIER
|
|
#define PPC_RELEASE_BARRIER
|
|
#define PPC_ATOMIC_ENTRY_BARRIER
|
|
#define PPC_ATOMIC_EXIT_BARRIER
|
|
#endif
|
|
|
|
#endif /* __KERNEL__ */
|
|
#endif /* _ASM_POWERPC_SYNCH_H */
|