x86-64: Move current task from PDA to per-cpu and consolidate with 32-bit.
Signed-off-by: Brian Gerst <brgerst@gmail.com> Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
Brian Gerst
Tejun Heo
parent
ea9279066d
commit
c6f5e0acd5
|
|
@ -1,39 +1,21 @@
|
|||
#ifndef _ASM_X86_CURRENT_H
|
||||
#define _ASM_X86_CURRENT_H
|
||||
|
||||
#ifdef CONFIG_X86_32
|
||||
#include <linux/compiler.h>
|
||||
#include <asm/percpu.h>
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
struct task_struct;
|
||||
|
||||
DECLARE_PER_CPU(struct task_struct *, current_task);
|
||||
|
||||
static __always_inline struct task_struct *get_current(void)
|
||||
{
|
||||
return percpu_read(current_task);
|
||||
}
|
||||
|
||||
#else /* X86_32 */
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
#include <asm/pda.h>
|
||||
|
||||
struct task_struct;
|
||||
|
||||
static __always_inline struct task_struct *get_current(void)
|
||||
{
|
||||
return read_pda(pcurrent);
|
||||
}
|
||||
|
||||
#else /* __ASSEMBLY__ */
|
||||
|
||||
#include <asm/asm-offsets.h>
|
||||
#define GET_CURRENT(reg) movq %gs:(pda_pcurrent),reg
|
||||
#define current get_current()
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
#endif /* X86_32 */
|
||||
|
||||
#define current get_current()
|
||||
|
||||
#endif /* _ASM_X86_CURRENT_H */
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@
|
|||
|
||||
/* Per processor datastructure. %gs points to it while the kernel runs */
|
||||
struct x8664_pda {
|
||||
struct task_struct *pcurrent; /* 0 Current process */
|
||||
unsigned long dummy;
|
||||
unsigned long unused1;
|
||||
unsigned long unused2;
|
||||
unsigned long kernelstack; /* 16 top of kernel stack for current */
|
||||
unsigned long oldrsp; /* 24 user rsp for system call */
|
||||
int irqcount; /* 32 Irq nesting counter. Starts -1 */
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ do { \
|
|||
"call __switch_to\n\t" \
|
||||
".globl thread_return\n" \
|
||||
"thread_return:\n\t" \
|
||||
"movq %%gs:%P[pda_pcurrent],%%rsi\n\t" \
|
||||
"movq "__percpu_seg_str"%P[current_task],%%rsi\n\t" \
|
||||
"movq %P[thread_info](%%rsi),%%r8\n\t" \
|
||||
LOCK_PREFIX "btr %[tif_fork],%P[ti_flags](%%r8)\n\t" \
|
||||
"movq %%rax,%%rdi\n\t" \
|
||||
|
|
@ -106,7 +106,7 @@ do { \
|
|||
[ti_flags] "i" (offsetof(struct thread_info, flags)), \
|
||||
[tif_fork] "i" (TIF_FORK), \
|
||||
[thread_info] "i" (offsetof(struct task_struct, stack)), \
|
||||
[pda_pcurrent] "i" (offsetof(struct x8664_pda, pcurrent)) \
|
||||
[current_task] "m" (per_cpu_var(current_task)) \
|
||||
: "memory", "cc" __EXTRA_CLOBBER)
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,6 @@ int main(void)
|
|||
#define ENTRY(entry) DEFINE(pda_ ## entry, offsetof(struct x8664_pda, entry))
|
||||
ENTRY(kernelstack);
|
||||
ENTRY(oldrsp);
|
||||
ENTRY(pcurrent);
|
||||
ENTRY(irqcount);
|
||||
DEFINE(pda_size, sizeof(struct x8664_pda));
|
||||
BLANK();
|
||||
|
|
|
|||
|
|
@ -903,10 +903,7 @@ void __cpuinit pda_init(int cpu)
|
|||
pda->kernelstack = (unsigned long)stack_thread_info() -
|
||||
PDA_STACKOFFSET + THREAD_SIZE;
|
||||
|
||||
if (cpu == 0) {
|
||||
/* others are initialized in smpboot.c */
|
||||
pda->pcurrent = &init_task;
|
||||
} else {
|
||||
if (cpu != 0) {
|
||||
if (pda->nodenumber == 0 && cpu_to_node(cpu) != NUMA_NO_NODE)
|
||||
pda->nodenumber = cpu_to_node(cpu);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -242,7 +242,7 @@ void show_registers(struct pt_regs *regs)
|
|||
int i;
|
||||
unsigned long sp;
|
||||
const int cpu = smp_processor_id();
|
||||
struct task_struct *cur = cpu_pda(cpu)->pcurrent;
|
||||
struct task_struct *cur = current;
|
||||
|
||||
sp = regs->sp;
|
||||
printk("CPU %d ", cpu);
|
||||
|
|
|
|||
|
|
@ -57,6 +57,9 @@
|
|||
|
||||
asmlinkage extern void ret_from_fork(void);
|
||||
|
||||
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
|
||||
EXPORT_PER_CPU_SYMBOL(current_task);
|
||||
|
||||
unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
|
||||
|
||||
static ATOMIC_NOTIFIER_HEAD(idle_notifier);
|
||||
|
|
@ -615,7 +618,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
|
|||
*/
|
||||
prev->usersp = read_pda(oldrsp);
|
||||
write_pda(oldrsp, next->usersp);
|
||||
write_pda(pcurrent, next_p);
|
||||
percpu_write(current_task, next_p);
|
||||
|
||||
write_pda(kernelstack,
|
||||
(unsigned long)task_stack_page(next_p) +
|
||||
|
|
|
|||
|
|
@ -790,13 +790,12 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
|
|||
|
||||
set_idle_for_cpu(cpu, c_idle.idle);
|
||||
do_rest:
|
||||
#ifdef CONFIG_X86_32
|
||||
per_cpu(current_task, cpu) = c_idle.idle;
|
||||
#ifdef CONFIG_X86_32
|
||||
init_gdt(cpu);
|
||||
/* Stack for startup_32 can be just as for start_secondary onwards */
|
||||
irq_ctx_init(cpu);
|
||||
#else
|
||||
cpu_pda(cpu)->pcurrent = c_idle.idle;
|
||||
clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
|
||||
initial_gs = per_cpu_offset(cpu);
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -279,12 +279,11 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)
|
|||
struct task_struct *idle = idle_task(cpu);
|
||||
int rc;
|
||||
|
||||
per_cpu(current_task, cpu) = idle;
|
||||
#ifdef CONFIG_X86_32
|
||||
init_gdt(cpu);
|
||||
per_cpu(current_task, cpu) = idle;
|
||||
irq_ctx_init(cpu);
|
||||
#else
|
||||
cpu_pda(cpu)->pcurrent = idle;
|
||||
clear_tsk_thread_flag(idle, TIF_FORK);
|
||||
#endif
|
||||
xen_setup_timer(cpu);
|
||||
|
|
|
|||
Loading…
Reference in a new issue