2019-06-02 23:44:50 -06:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2015-02-03 05:39:03 -07:00
|
|
|
/*
|
|
|
|
|
* Based on arch/arm/include/asm/atomic.h
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 1996 Russell King.
|
|
|
|
|
* Copyright (C) 2002 Deep Blue Solutions Ltd.
|
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef __ASM_ATOMIC_LSE_H
|
|
|
|
|
#define __ASM_ATOMIC_LSE_H
|
|
|
|
|
|
|
|
|
|
#ifndef __ARM64_IN_ATOMIC_IMPL
|
|
|
|
|
#error "please don't include this file directly"
|
|
|
|
|
#endif
|
|
|
|
|
|
2018-09-04 04:48:30 -06:00
|
|
|
#define __LL_SC_ATOMIC(op) __LL_SC_CALL(arch_atomic_##op)
|
2016-04-22 11:01:32 -06:00
|
|
|
#define ATOMIC_OP(op, asm_op) \
|
2018-09-04 04:48:30 -06:00
|
|
|
static inline void arch_atomic_##op(int i, atomic_t *v) \
|
2016-04-22 11:01:32 -06:00
|
|
|
{ \
|
|
|
|
|
register int w0 asm ("w0") = i; \
|
|
|
|
|
register atomic_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(op), \
|
|
|
|
|
" " #asm_op " %w[i], %[v]\n") \
|
|
|
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter) \
|
|
|
|
|
: "r" (x1) \
|
|
|
|
|
: __LL_SC_CLOBBERS); \
|
2015-02-03 09:14:13 -07:00
|
|
|
}
|
|
|
|
|
|
2016-04-22 11:01:32 -06:00
|
|
|
ATOMIC_OP(andnot, stclr)
|
|
|
|
|
ATOMIC_OP(or, stset)
|
|
|
|
|
ATOMIC_OP(xor, steor)
|
|
|
|
|
ATOMIC_OP(add, stadd)
|
2015-02-03 09:14:13 -07:00
|
|
|
|
2016-04-22 11:01:32 -06:00
|
|
|
#undef ATOMIC_OP
|
2015-02-03 09:14:13 -07:00
|
|
|
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
#define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \
|
2018-09-04 04:48:30 -06:00
|
|
|
static inline int arch_atomic_fetch_##op##name(int i, atomic_t *v) \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
{ \
|
|
|
|
|
register int w0 asm ("w0") = i; \
|
|
|
|
|
register atomic_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
|
|
|
|
__LL_SC_ATOMIC(fetch_##op##name), \
|
|
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" " #asm_op #mb " %w[i], %w[i], %[v]") \
|
|
|
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter) \
|
|
|
|
|
: "r" (x1) \
|
|
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
|
|
|
|
\
|
|
|
|
|
return w0; \
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define ATOMIC_FETCH_OPS(op, asm_op) \
|
|
|
|
|
ATOMIC_FETCH_OP(_relaxed, , op, asm_op) \
|
|
|
|
|
ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \
|
|
|
|
|
ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \
|
|
|
|
|
ATOMIC_FETCH_OP( , al, op, asm_op, "memory")
|
|
|
|
|
|
|
|
|
|
ATOMIC_FETCH_OPS(andnot, ldclr)
|
|
|
|
|
ATOMIC_FETCH_OPS(or, ldset)
|
|
|
|
|
ATOMIC_FETCH_OPS(xor, ldeor)
|
|
|
|
|
ATOMIC_FETCH_OPS(add, ldadd)
|
|
|
|
|
|
|
|
|
|
#undef ATOMIC_FETCH_OP
|
|
|
|
|
#undef ATOMIC_FETCH_OPS
|
|
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
#define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
|
2018-09-04 04:48:30 -06:00
|
|
|
static inline int arch_atomic_add_return##name(int i, atomic_t *v) \
|
2015-10-08 13:15:18 -06:00
|
|
|
{ \
|
|
|
|
|
register int w0 asm ("w0") = i; \
|
|
|
|
|
register atomic_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_ATOMIC(add_return##name) \
|
|
|
|
|
__nops(1), \
|
2015-10-08 13:15:18 -06:00
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" ldadd" #mb " %w[i], w30, %[v]\n" \
|
|
|
|
|
" add %w[i], %w[i], w30") \
|
|
|
|
|
: [i] "+r" (w0), [v] "+Q" (v->counter) \
|
|
|
|
|
: "r" (x1) \
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
2015-10-08 13:15:18 -06:00
|
|
|
\
|
|
|
|
|
return w0; \
|
|
|
|
|
}
|
2015-02-03 09:14:13 -07:00
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
ATOMIC_OP_ADD_RETURN(_relaxed, )
|
|
|
|
|
ATOMIC_OP_ADD_RETURN(_acquire, a, "memory")
|
|
|
|
|
ATOMIC_OP_ADD_RETURN(_release, l, "memory")
|
|
|
|
|
ATOMIC_OP_ADD_RETURN( , al, "memory")
|
2015-02-03 09:14:13 -07:00
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
#undef ATOMIC_OP_ADD_RETURN
|
2015-02-03 09:14:13 -07:00
|
|
|
|
2018-09-04 04:48:30 -06:00
|
|
|
static inline void arch_atomic_and(int i, atomic_t *v)
|
2015-02-03 09:14:13 -07:00
|
|
|
{
|
|
|
|
|
register int w0 asm ("w0") = i;
|
|
|
|
|
register atomic_t *x1 asm ("x1") = v;
|
|
|
|
|
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
|
|
|
/* LL/SC */
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_ATOMIC(and)
|
|
|
|
|
__nops(1),
|
2015-02-03 09:14:13 -07:00
|
|
|
/* LSE atomics */
|
|
|
|
|
" mvn %w[i], %w[i]\n"
|
|
|
|
|
" stclr %w[i], %[v]")
|
2018-05-21 10:44:57 -06:00
|
|
|
: [i] "+&r" (w0), [v] "+Q" (v->counter)
|
2015-02-03 09:14:13 -07:00
|
|
|
: "r" (x1)
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS);
|
2015-02-03 09:14:13 -07:00
|
|
|
}
|
|
|
|
|
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
#define ATOMIC_FETCH_OP_AND(name, mb, cl...) \
|
2018-09-04 04:48:30 -06:00
|
|
|
static inline int arch_atomic_fetch_and##name(int i, atomic_t *v) \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
{ \
|
|
|
|
|
register int w0 asm ("w0") = i; \
|
|
|
|
|
register atomic_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_ATOMIC(fetch_and##name) \
|
|
|
|
|
__nops(1), \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" mvn %w[i], %w[i]\n" \
|
|
|
|
|
" ldclr" #mb " %w[i], %w[i], %[v]") \
|
2018-05-21 10:44:57 -06:00
|
|
|
: [i] "+&r" (w0), [v] "+Q" (v->counter) \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
: "r" (x1) \
|
|
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
|
|
|
|
\
|
|
|
|
|
return w0; \
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_FETCH_OP_AND(_relaxed, )
|
|
|
|
|
ATOMIC_FETCH_OP_AND(_acquire, a, "memory")
|
|
|
|
|
ATOMIC_FETCH_OP_AND(_release, l, "memory")
|
|
|
|
|
ATOMIC_FETCH_OP_AND( , al, "memory")
|
|
|
|
|
|
|
|
|
|
#undef ATOMIC_FETCH_OP_AND
|
|
|
|
|
|
2018-09-04 04:48:30 -06:00
|
|
|
static inline void arch_atomic_sub(int i, atomic_t *v)
|
2015-02-03 09:14:13 -07:00
|
|
|
{
|
|
|
|
|
register int w0 asm ("w0") = i;
|
|
|
|
|
register atomic_t *x1 asm ("x1") = v;
|
|
|
|
|
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
|
|
|
/* LL/SC */
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_ATOMIC(sub)
|
|
|
|
|
__nops(1),
|
2015-02-03 09:14:13 -07:00
|
|
|
/* LSE atomics */
|
|
|
|
|
" neg %w[i], %w[i]\n"
|
|
|
|
|
" stadd %w[i], %[v]")
|
2018-05-21 10:44:57 -06:00
|
|
|
: [i] "+&r" (w0), [v] "+Q" (v->counter)
|
2015-02-03 09:14:13 -07:00
|
|
|
: "r" (x1)
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS);
|
2015-02-03 09:14:13 -07:00
|
|
|
}
|
|
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
#define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
|
2018-09-04 04:48:30 -06:00
|
|
|
static inline int arch_atomic_sub_return##name(int i, atomic_t *v) \
|
2015-10-08 13:15:18 -06:00
|
|
|
{ \
|
|
|
|
|
register int w0 asm ("w0") = i; \
|
|
|
|
|
register atomic_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
|
|
|
|
__LL_SC_ATOMIC(sub_return##name) \
|
2016-09-06 09:42:58 -06:00
|
|
|
__nops(2), \
|
2015-10-08 13:15:18 -06:00
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" neg %w[i], %w[i]\n" \
|
|
|
|
|
" ldadd" #mb " %w[i], w30, %[v]\n" \
|
|
|
|
|
" add %w[i], %w[i], w30") \
|
2018-05-21 10:44:57 -06:00
|
|
|
: [i] "+&r" (w0), [v] "+Q" (v->counter) \
|
2015-10-08 13:15:18 -06:00
|
|
|
: "r" (x1) \
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS , ##cl); \
|
2015-10-08 13:15:18 -06:00
|
|
|
\
|
|
|
|
|
return w0; \
|
2015-02-03 09:14:13 -07:00
|
|
|
}
|
2015-02-03 05:39:03 -07:00
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
ATOMIC_OP_SUB_RETURN(_relaxed, )
|
|
|
|
|
ATOMIC_OP_SUB_RETURN(_acquire, a, "memory")
|
|
|
|
|
ATOMIC_OP_SUB_RETURN(_release, l, "memory")
|
|
|
|
|
ATOMIC_OP_SUB_RETURN( , al, "memory")
|
|
|
|
|
|
|
|
|
|
#undef ATOMIC_OP_SUB_RETURN
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
|
|
|
|
|
#define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \
|
2018-09-04 04:48:30 -06:00
|
|
|
static inline int arch_atomic_fetch_sub##name(int i, atomic_t *v) \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
{ \
|
|
|
|
|
register int w0 asm ("w0") = i; \
|
|
|
|
|
register atomic_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_ATOMIC(fetch_sub##name) \
|
|
|
|
|
__nops(1), \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" neg %w[i], %w[i]\n" \
|
|
|
|
|
" ldadd" #mb " %w[i], %w[i], %[v]") \
|
2018-05-21 10:44:57 -06:00
|
|
|
: [i] "+&r" (w0), [v] "+Q" (v->counter) \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
: "r" (x1) \
|
|
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
|
|
|
|
\
|
|
|
|
|
return w0; \
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_FETCH_OP_SUB(_relaxed, )
|
|
|
|
|
ATOMIC_FETCH_OP_SUB(_acquire, a, "memory")
|
|
|
|
|
ATOMIC_FETCH_OP_SUB(_release, l, "memory")
|
|
|
|
|
ATOMIC_FETCH_OP_SUB( , al, "memory")
|
|
|
|
|
|
|
|
|
|
#undef ATOMIC_FETCH_OP_SUB
|
2015-02-03 09:14:13 -07:00
|
|
|
#undef __LL_SC_ATOMIC
|
2015-02-03 05:39:03 -07:00
|
|
|
|
2018-09-04 04:48:30 -06:00
|
|
|
#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(arch_atomic64_##op)
|
2016-04-22 11:01:32 -06:00
|
|
|
#define ATOMIC64_OP(op, asm_op) \
|
2019-05-22 07:22:39 -06:00
|
|
|
static inline void arch_atomic64_##op(s64 i, atomic64_t *v) \
|
2016-04-22 11:01:32 -06:00
|
|
|
{ \
|
2019-05-22 07:22:39 -06:00
|
|
|
register s64 x0 asm ("x0") = i; \
|
2016-04-22 11:01:32 -06:00
|
|
|
register atomic64_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(op), \
|
|
|
|
|
" " #asm_op " %[i], %[v]\n") \
|
|
|
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter) \
|
|
|
|
|
: "r" (x1) \
|
|
|
|
|
: __LL_SC_CLOBBERS); \
|
2015-02-03 09:14:13 -07:00
|
|
|
}
|
|
|
|
|
|
2016-04-22 11:01:32 -06:00
|
|
|
ATOMIC64_OP(andnot, stclr)
|
|
|
|
|
ATOMIC64_OP(or, stset)
|
|
|
|
|
ATOMIC64_OP(xor, steor)
|
|
|
|
|
ATOMIC64_OP(add, stadd)
|
2015-02-03 09:14:13 -07:00
|
|
|
|
2016-04-22 11:01:32 -06:00
|
|
|
#undef ATOMIC64_OP
|
2015-02-03 09:14:13 -07:00
|
|
|
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
#define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \
|
2019-05-22 07:22:39 -06:00
|
|
|
static inline s64 arch_atomic64_fetch_##op##name(s64 i, atomic64_t *v) \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
{ \
|
2019-05-22 07:22:39 -06:00
|
|
|
register s64 x0 asm ("x0") = i; \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
register atomic64_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
|
|
|
|
__LL_SC_ATOMIC64(fetch_##op##name), \
|
|
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" " #asm_op #mb " %[i], %[i], %[v]") \
|
|
|
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter) \
|
|
|
|
|
: "r" (x1) \
|
|
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
|
|
|
|
\
|
|
|
|
|
return x0; \
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define ATOMIC64_FETCH_OPS(op, asm_op) \
|
|
|
|
|
ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \
|
|
|
|
|
ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \
|
|
|
|
|
ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \
|
|
|
|
|
ATOMIC64_FETCH_OP( , al, op, asm_op, "memory")
|
|
|
|
|
|
|
|
|
|
ATOMIC64_FETCH_OPS(andnot, ldclr)
|
|
|
|
|
ATOMIC64_FETCH_OPS(or, ldset)
|
|
|
|
|
ATOMIC64_FETCH_OPS(xor, ldeor)
|
|
|
|
|
ATOMIC64_FETCH_OPS(add, ldadd)
|
|
|
|
|
|
|
|
|
|
#undef ATOMIC64_FETCH_OP
|
|
|
|
|
#undef ATOMIC64_FETCH_OPS
|
|
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
|
2019-05-22 07:22:39 -06:00
|
|
|
static inline s64 arch_atomic64_add_return##name(s64 i, atomic64_t *v) \
|
2015-10-08 13:15:18 -06:00
|
|
|
{ \
|
2019-05-22 07:22:39 -06:00
|
|
|
register s64 x0 asm ("x0") = i; \
|
2015-10-08 13:15:18 -06:00
|
|
|
register atomic64_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_ATOMIC64(add_return##name) \
|
|
|
|
|
__nops(1), \
|
2015-10-08 13:15:18 -06:00
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" ldadd" #mb " %[i], x30, %[v]\n" \
|
|
|
|
|
" add %[i], %[i], x30") \
|
|
|
|
|
: [i] "+r" (x0), [v] "+Q" (v->counter) \
|
|
|
|
|
: "r" (x1) \
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
2015-10-08 13:15:18 -06:00
|
|
|
\
|
|
|
|
|
return x0; \
|
|
|
|
|
}
|
2015-02-03 09:14:13 -07:00
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
ATOMIC64_OP_ADD_RETURN(_relaxed, )
|
|
|
|
|
ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory")
|
|
|
|
|
ATOMIC64_OP_ADD_RETURN(_release, l, "memory")
|
|
|
|
|
ATOMIC64_OP_ADD_RETURN( , al, "memory")
|
2015-02-03 09:14:13 -07:00
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
#undef ATOMIC64_OP_ADD_RETURN
|
2015-02-03 09:14:13 -07:00
|
|
|
|
2019-05-22 07:22:39 -06:00
|
|
|
static inline void arch_atomic64_and(s64 i, atomic64_t *v)
|
2015-02-03 09:14:13 -07:00
|
|
|
{
|
2019-05-22 07:22:39 -06:00
|
|
|
register s64 x0 asm ("x0") = i;
|
2015-02-03 09:14:13 -07:00
|
|
|
register atomic64_t *x1 asm ("x1") = v;
|
|
|
|
|
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
|
|
|
/* LL/SC */
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_ATOMIC64(and)
|
|
|
|
|
__nops(1),
|
2015-02-03 09:14:13 -07:00
|
|
|
/* LSE atomics */
|
|
|
|
|
" mvn %[i], %[i]\n"
|
|
|
|
|
" stclr %[i], %[v]")
|
2018-05-21 10:44:57 -06:00
|
|
|
: [i] "+&r" (x0), [v] "+Q" (v->counter)
|
2015-02-03 09:14:13 -07:00
|
|
|
: "r" (x1)
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS);
|
2015-02-03 09:14:13 -07:00
|
|
|
}
|
|
|
|
|
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
#define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \
|
2019-05-22 07:22:39 -06:00
|
|
|
static inline s64 arch_atomic64_fetch_and##name(s64 i, atomic64_t *v) \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
{ \
|
2019-05-22 07:22:39 -06:00
|
|
|
register s64 x0 asm ("x0") = i; \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
register atomic64_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_ATOMIC64(fetch_and##name) \
|
|
|
|
|
__nops(1), \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" mvn %[i], %[i]\n" \
|
|
|
|
|
" ldclr" #mb " %[i], %[i], %[v]") \
|
2018-05-21 10:44:57 -06:00
|
|
|
: [i] "+&r" (x0), [v] "+Q" (v->counter) \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
: "r" (x1) \
|
|
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
|
|
|
|
\
|
|
|
|
|
return x0; \
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC64_FETCH_OP_AND(_relaxed, )
|
|
|
|
|
ATOMIC64_FETCH_OP_AND(_acquire, a, "memory")
|
|
|
|
|
ATOMIC64_FETCH_OP_AND(_release, l, "memory")
|
|
|
|
|
ATOMIC64_FETCH_OP_AND( , al, "memory")
|
|
|
|
|
|
|
|
|
|
#undef ATOMIC64_FETCH_OP_AND
|
|
|
|
|
|
2019-05-22 07:22:39 -06:00
|
|
|
static inline void arch_atomic64_sub(s64 i, atomic64_t *v)
|
2015-02-03 09:14:13 -07:00
|
|
|
{
|
2019-05-22 07:22:39 -06:00
|
|
|
register s64 x0 asm ("x0") = i;
|
2015-02-03 09:14:13 -07:00
|
|
|
register atomic64_t *x1 asm ("x1") = v;
|
|
|
|
|
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
|
|
|
/* LL/SC */
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_ATOMIC64(sub)
|
|
|
|
|
__nops(1),
|
2015-02-03 09:14:13 -07:00
|
|
|
/* LSE atomics */
|
|
|
|
|
" neg %[i], %[i]\n"
|
|
|
|
|
" stadd %[i], %[v]")
|
2018-05-21 10:44:57 -06:00
|
|
|
: [i] "+&r" (x0), [v] "+Q" (v->counter)
|
2015-02-03 09:14:13 -07:00
|
|
|
: "r" (x1)
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS);
|
2015-02-03 09:14:13 -07:00
|
|
|
}
|
|
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
|
2019-05-22 07:22:39 -06:00
|
|
|
static inline s64 arch_atomic64_sub_return##name(s64 i, atomic64_t *v) \
|
2015-10-08 13:15:18 -06:00
|
|
|
{ \
|
2019-05-22 07:22:39 -06:00
|
|
|
register s64 x0 asm ("x0") = i; \
|
2015-10-08 13:15:18 -06:00
|
|
|
register atomic64_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
|
|
|
|
__LL_SC_ATOMIC64(sub_return##name) \
|
2016-09-06 09:42:58 -06:00
|
|
|
__nops(2), \
|
2015-10-08 13:15:18 -06:00
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" neg %[i], %[i]\n" \
|
|
|
|
|
" ldadd" #mb " %[i], x30, %[v]\n" \
|
|
|
|
|
" add %[i], %[i], x30") \
|
2018-05-21 10:44:57 -06:00
|
|
|
: [i] "+&r" (x0), [v] "+Q" (v->counter) \
|
2015-10-08 13:15:18 -06:00
|
|
|
: "r" (x1) \
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
2015-10-08 13:15:18 -06:00
|
|
|
\
|
|
|
|
|
return x0; \
|
|
|
|
|
}
|
2015-02-03 09:14:13 -07:00
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
ATOMIC64_OP_SUB_RETURN(_relaxed, )
|
|
|
|
|
ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory")
|
|
|
|
|
ATOMIC64_OP_SUB_RETURN(_release, l, "memory")
|
|
|
|
|
ATOMIC64_OP_SUB_RETURN( , al, "memory")
|
2015-02-03 09:14:13 -07:00
|
|
|
|
2015-10-08 13:15:18 -06:00
|
|
|
#undef ATOMIC64_OP_SUB_RETURN
|
2015-02-03 05:39:03 -07:00
|
|
|
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
#define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \
|
2019-05-22 07:22:39 -06:00
|
|
|
static inline s64 arch_atomic64_fetch_sub##name(s64 i, atomic64_t *v) \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
{ \
|
2019-05-22 07:22:39 -06:00
|
|
|
register s64 x0 asm ("x0") = i; \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
register atomic64_t *x1 asm ("x1") = v; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_ATOMIC64(fetch_sub##name) \
|
|
|
|
|
__nops(1), \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" neg %[i], %[i]\n" \
|
|
|
|
|
" ldadd" #mb " %[i], %[i], %[v]") \
|
2018-05-21 10:44:57 -06:00
|
|
|
: [i] "+&r" (x0), [v] "+Q" (v->counter) \
|
locking/atomic, arch/arm64: Implement atomic{,64}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}() for LSE instructions
Implement FETCH-OP atomic primitives, these are very similar to the
existing OP-RETURN primitives we already have, except they return the
value of the atomic variable _before_ modification.
This is especially useful for irreversible operations -- such as
bitops (because it becomes impossible to reconstruct the state prior
to modification).
This patch implements the LSE variants.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/1461344493-8262-2-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-04-22 11:01:33 -06:00
|
|
|
: "r" (x1) \
|
|
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
|
|
|
|
\
|
|
|
|
|
return x0; \
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC64_FETCH_OP_SUB(_relaxed, )
|
|
|
|
|
ATOMIC64_FETCH_OP_SUB(_acquire, a, "memory")
|
|
|
|
|
ATOMIC64_FETCH_OP_SUB(_release, l, "memory")
|
|
|
|
|
ATOMIC64_FETCH_OP_SUB( , al, "memory")
|
|
|
|
|
|
|
|
|
|
#undef ATOMIC64_FETCH_OP_SUB
|
|
|
|
|
|
2019-05-22 07:22:39 -06:00
|
|
|
static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
|
2015-02-03 05:39:03 -07:00
|
|
|
{
|
2015-02-03 09:14:13 -07:00
|
|
|
register long x0 asm ("x0") = (long)v;
|
2015-02-03 05:39:03 -07:00
|
|
|
|
2015-02-03 09:14:13 -07:00
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
|
|
|
/* LL/SC */
|
|
|
|
|
__LL_SC_ATOMIC64(dec_if_positive)
|
2016-09-06 09:42:58 -06:00
|
|
|
__nops(6),
|
2015-02-03 09:14:13 -07:00
|
|
|
/* LSE atomics */
|
|
|
|
|
"1: ldr x30, %[v]\n"
|
|
|
|
|
" subs %[ret], x30, #1\n"
|
2015-05-29 07:44:06 -06:00
|
|
|
" b.lt 2f\n"
|
2015-02-03 09:14:13 -07:00
|
|
|
" casal x30, %[ret], %[v]\n"
|
|
|
|
|
" sub x30, x30, #1\n"
|
|
|
|
|
" sub x30, x30, %[ret]\n"
|
|
|
|
|
" cbnz x30, 1b\n"
|
|
|
|
|
"2:")
|
2018-05-21 10:44:57 -06:00
|
|
|
: [ret] "+&r" (x0), [v] "+Q" (v->counter)
|
2015-02-03 05:39:03 -07:00
|
|
|
:
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS, "cc", "memory");
|
2015-02-03 05:39:03 -07:00
|
|
|
|
|
|
|
|
return x0;
|
|
|
|
|
}
|
|
|
|
|
|
2015-02-03 09:14:13 -07:00
|
|
|
#undef __LL_SC_ATOMIC64
|
|
|
|
|
|
2015-04-23 13:08:49 -06:00
|
|
|
#define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
|
|
|
|
|
|
2018-09-13 06:30:45 -06:00
|
|
|
#define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...) \
|
|
|
|
|
static inline u##sz __cmpxchg_case_##name##sz(volatile void *ptr, \
|
2018-09-13 07:28:33 -06:00
|
|
|
u##sz old, \
|
2018-09-13 06:30:45 -06:00
|
|
|
u##sz new) \
|
2015-04-23 13:08:49 -06:00
|
|
|
{ \
|
|
|
|
|
register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
|
2018-09-13 07:28:33 -06:00
|
|
|
register u##sz x1 asm ("x1") = old; \
|
2018-09-13 06:30:45 -06:00
|
|
|
register u##sz x2 asm ("x2") = new; \
|
2015-04-23 13:08:49 -06:00
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
2018-09-13 06:30:45 -06:00
|
|
|
__LL_SC_CMPXCHG(name##sz) \
|
2016-09-06 09:42:58 -06:00
|
|
|
__nops(2), \
|
2015-04-23 13:08:49 -06:00
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" mov " #w "30, %" #w "[old]\n" \
|
2018-09-13 06:30:45 -06:00
|
|
|
" cas" #mb #sfx "\t" #w "30, %" #w "[new], %[v]\n" \
|
2015-04-23 13:08:49 -06:00
|
|
|
" mov %" #w "[ret], " #w "30") \
|
|
|
|
|
: [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \
|
|
|
|
|
: [old] "r" (x1), [new] "r" (x2) \
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
2015-04-23 13:08:49 -06:00
|
|
|
\
|
|
|
|
|
return x0; \
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-13 06:30:45 -06:00
|
|
|
__CMPXCHG_CASE(w, b, , 8, )
|
|
|
|
|
__CMPXCHG_CASE(w, h, , 16, )
|
|
|
|
|
__CMPXCHG_CASE(w, , , 32, )
|
|
|
|
|
__CMPXCHG_CASE(x, , , 64, )
|
|
|
|
|
__CMPXCHG_CASE(w, b, acq_, 8, a, "memory")
|
|
|
|
|
__CMPXCHG_CASE(w, h, acq_, 16, a, "memory")
|
|
|
|
|
__CMPXCHG_CASE(w, , acq_, 32, a, "memory")
|
|
|
|
|
__CMPXCHG_CASE(x, , acq_, 64, a, "memory")
|
|
|
|
|
__CMPXCHG_CASE(w, b, rel_, 8, l, "memory")
|
|
|
|
|
__CMPXCHG_CASE(w, h, rel_, 16, l, "memory")
|
|
|
|
|
__CMPXCHG_CASE(w, , rel_, 32, l, "memory")
|
|
|
|
|
__CMPXCHG_CASE(x, , rel_, 64, l, "memory")
|
|
|
|
|
__CMPXCHG_CASE(w, b, mb_, 8, al, "memory")
|
|
|
|
|
__CMPXCHG_CASE(w, h, mb_, 16, al, "memory")
|
|
|
|
|
__CMPXCHG_CASE(w, , mb_, 32, al, "memory")
|
|
|
|
|
__CMPXCHG_CASE(x, , mb_, 64, al, "memory")
|
2015-04-23 13:08:49 -06:00
|
|
|
|
|
|
|
|
#undef __LL_SC_CMPXCHG
|
|
|
|
|
#undef __CMPXCHG_CASE
|
|
|
|
|
|
2015-05-14 11:05:50 -06:00
|
|
|
#define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
|
|
|
|
|
|
|
|
|
|
#define __CMPXCHG_DBL(name, mb, cl...) \
|
2015-11-05 07:00:56 -07:00
|
|
|
static inline long __cmpxchg_double##name(unsigned long old1, \
|
2015-05-14 11:05:50 -06:00
|
|
|
unsigned long old2, \
|
|
|
|
|
unsigned long new1, \
|
|
|
|
|
unsigned long new2, \
|
|
|
|
|
volatile void *ptr) \
|
|
|
|
|
{ \
|
|
|
|
|
unsigned long oldval1 = old1; \
|
|
|
|
|
unsigned long oldval2 = old2; \
|
|
|
|
|
register unsigned long x0 asm ("x0") = old1; \
|
|
|
|
|
register unsigned long x1 asm ("x1") = old2; \
|
|
|
|
|
register unsigned long x2 asm ("x2") = new1; \
|
|
|
|
|
register unsigned long x3 asm ("x3") = new2; \
|
|
|
|
|
register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
|
|
|
|
|
\
|
|
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN( \
|
|
|
|
|
/* LL/SC */ \
|
2016-09-06 09:42:58 -06:00
|
|
|
__LL_SC_CMPXCHG_DBL(name) \
|
|
|
|
|
__nops(3), \
|
2015-05-14 11:05:50 -06:00
|
|
|
/* LSE atomics */ \
|
|
|
|
|
" casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
|
|
|
|
|
" eor %[old1], %[old1], %[oldval1]\n" \
|
|
|
|
|
" eor %[old2], %[old2], %[oldval2]\n" \
|
|
|
|
|
" orr %[old1], %[old1], %[old2]") \
|
2018-05-21 10:44:57 -06:00
|
|
|
: [old1] "+&r" (x0), [old2] "+&r" (x1), \
|
2015-05-14 11:05:50 -06:00
|
|
|
[v] "+Q" (*(unsigned long *)ptr) \
|
|
|
|
|
: [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
|
|
|
|
|
[oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
|
arm64: lse: deal with clobbered IP registers after branch via PLT
The LSE atomics implementation uses runtime patching to patch in calls
to out of line non-LSE atomics implementations on cores that lack hardware
support for LSE. To avoid paying the overhead cost of a function call even
if no call ends up being made, the bl instruction is kept invisible to the
compiler, and the out of line implementations preserve all registers, not
just the ones that they are required to preserve as per the AAPCS64.
However, commit fd045f6cd98e ("arm64: add support for module PLTs") added
support for routing branch instructions via veneers if the branch target
offset exceeds the range of the ordinary relative branch instructions.
Since this deals with jump and call instructions that are exposed to ELF
relocations, the PLT code uses x16 to hold the address of the branch target
when it performs an indirect branch-to-register, something which is
explicitly allowed by the AAPCS64 (and ordinary compiler generated code
does not expect register x16 or x17 to retain their values across a bl
instruction).
Since the lse runtime patched bl instructions don't adhere to the AAPCS64,
they don't deal with this clobbering of registers x16 and x17. So add them
to the clobber list of the asm() statements that perform the call
instructions, and drop x16 and x17 from the list of registers that are
callee saved in the out of line non-LSE implementations.
In addition, since we have given these functions two scratch registers,
they no longer need to stack/unstack temp registers.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[will: factored clobber list into #define, updated Makefile comment]
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-02-25 12:48:53 -07:00
|
|
|
: __LL_SC_CLOBBERS, ##cl); \
|
2015-05-14 11:05:50 -06:00
|
|
|
\
|
|
|
|
|
return x0; \
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__CMPXCHG_DBL( , )
|
|
|
|
|
__CMPXCHG_DBL(_mb, al, "memory")
|
|
|
|
|
|
|
|
|
|
#undef __LL_SC_CMPXCHG_DBL
|
|
|
|
|
#undef __CMPXCHG_DBL
|
|
|
|
|
|
2015-02-03 05:39:03 -07:00
|
|
|
#endif /* __ASM_ATOMIC_LSE_H */
|