x86, percpu: Optimize this_cpu_ptr
Allow arches to implement __this_cpu_ptr, and provide an x86 version. Before: movq $foo, %rax movq %gs:this_cpu_off, %rdx addq %rdx, %rax After: movq $foo, %rax addq %gs:this_cpu_off, %rax The benefit is doing it in one less instruction and not clobbering a temporary register. tj: * Beefed up the comment a bit and renamed in-macro temp variable to match neighboring macros. * Folded fix for const pointer case found in linux-next. * Fixed sparse notation. Signed-off-by: Brian Gerst <brgerst@gmail.com> Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
parent
fc1481a956
commit
db7829c6cc
|
@ -47,6 +47,20 @@
|
|||
#ifdef CONFIG_SMP
|
||||
#define __percpu_arg(x) "%%"__stringify(__percpu_seg)":%P" #x
|
||||
#define __my_cpu_offset percpu_read(this_cpu_off)
|
||||
|
||||
/*
|
||||
* Compared to the generic __my_cpu_offset version, the following
|
||||
* saves one instruction and avoids clobbering a temp register.
|
||||
*/
|
||||
#define __this_cpu_ptr(ptr) \
|
||||
({ \
|
||||
unsigned long tcp_ptr__; \
|
||||
__verify_pcpu_ptr(ptr); \
|
||||
asm volatile("add " __percpu_arg(1) ", %0" \
|
||||
: "=r" (tcp_ptr__) \
|
||||
: "m" (this_cpu_off), "0" (ptr)); \
|
||||
(typeof(*(ptr)) __kernel __force *)tcp_ptr__; \
|
||||
})
|
||||
#else
|
||||
#define __percpu_arg(x) "%P" #x
|
||||
#endif
|
||||
|
|
|
@ -60,9 +60,14 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
|
|||
#define __raw_get_cpu_var(var) \
|
||||
(*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset))
|
||||
|
||||
#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)
|
||||
#ifndef __this_cpu_ptr
|
||||
#define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
|
||||
|
||||
#endif
|
||||
#ifdef CONFIG_DEBUG_PREEMPT
|
||||
#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)
|
||||
#else
|
||||
#define this_cpu_ptr(ptr) __this_cpu_ptr(ptr)
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
|
||||
extern void setup_per_cpu_areas(void);
|
||||
|
|
Loading…
Reference in a new issue