2008-03-10 16:28:04 -06:00
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/smp.h>
|
2009-02-27 14:25:28 -07:00
|
|
|
#include <linux/prctl.h>
|
2008-03-10 16:28:04 -06:00
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/sched.h>
|
2008-04-25 09:39:01 -06:00
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/pm.h>
|
2008-06-09 11:15:00 -06:00
|
|
|
#include <linux/clockchips.h>
|
2009-05-11 20:05:28 -06:00
|
|
|
#include <linux/random.h>
|
2009-09-19 00:40:22 -06:00
|
|
|
#include <linux/user-return-notifier.h>
|
2009-12-08 01:29:42 -07:00
|
|
|
#include <linux/dmi.h>
|
|
|
|
#include <linux/utsname.h>
|
2009-09-17 08:11:28 -06:00
|
|
|
#include <trace/events/power.h>
|
2009-09-09 11:22:48 -06:00
|
|
|
#include <linux/hw_breakpoint.h>
|
2011-01-20 07:42:52 -07:00
|
|
|
#include <asm/cpu.h>
|
2008-06-24 03:58:53 -06:00
|
|
|
#include <asm/system.h>
|
2008-11-11 06:33:44 -07:00
|
|
|
#include <asm/apic.h>
|
2009-04-10 12:33:10 -06:00
|
|
|
#include <asm/syscalls.h>
|
2009-02-27 14:25:28 -07:00
|
|
|
#include <asm/idle.h>
|
|
|
|
#include <asm/uaccess.h>
|
|
|
|
#include <asm/i387.h>
|
2009-06-01 12:14:55 -06:00
|
|
|
#include <asm/debugreg.h>
|
2008-06-24 03:58:53 -06:00
|
|
|
|
2008-03-10 16:28:05 -06:00
|
|
|
struct kmem_cache *task_xstate_cachep;
|
2010-05-17 03:22:23 -06:00
|
|
|
EXPORT_SYMBOL_GPL(task_xstate_cachep);
|
2008-03-10 16:28:04 -06:00
|
|
|
|
|
|
|
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
|
|
|
|
{
|
2010-05-06 02:45:46 -06:00
|
|
|
int ret;
|
|
|
|
|
2008-03-10 16:28:04 -06:00
|
|
|
*dst = *src;
|
2010-05-06 02:45:46 -06:00
|
|
|
if (fpu_allocated(&src->thread.fpu)) {
|
|
|
|
memset(&dst->thread.fpu, 0, sizeof(dst->thread.fpu));
|
|
|
|
ret = fpu_alloc(&dst->thread.fpu);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
fpu_copy(&dst->thread.fpu, &src->thread.fpu);
|
2008-03-10 16:28:05 -06:00
|
|
|
}
|
2008-03-10 16:28:04 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-03-10 16:28:05 -06:00
|
|
|
void free_thread_xstate(struct task_struct *tsk)
|
2008-03-10 16:28:04 -06:00
|
|
|
{
|
2010-05-06 02:45:46 -06:00
|
|
|
fpu_free(&tsk->thread.fpu);
|
2008-03-10 16:28:05 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void free_thread_info(struct thread_info *ti)
|
|
|
|
{
|
|
|
|
free_thread_xstate(ti->task);
|
2008-04-16 02:27:53 -06:00
|
|
|
free_pages((unsigned long)ti, get_order(THREAD_SIZE));
|
2008-03-10 16:28:04 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void arch_task_cache_init(void)
|
|
|
|
{
|
|
|
|
task_xstate_cachep =
|
|
|
|
kmem_cache_create("task_xstate", xstate_size,
|
|
|
|
__alignof__(union thread_xstate),
|
kmemcheck: add mm functions
With kmemcheck enabled, the slab allocator needs to do this:
1. Tell kmemcheck to allocate the shadow memory which stores the status of
each byte in the allocation proper, e.g. whether it is initialized or
uninitialized.
2. Tell kmemcheck which parts of memory that should be marked uninitialized.
There are actually a few more states, such as "not yet allocated" and
"recently freed".
If a slab cache is set up using the SLAB_NOTRACK flag, it will never return
memory that can take page faults because of kmemcheck.
If a slab cache is NOT set up using the SLAB_NOTRACK flag, callers can still
request memory with the __GFP_NOTRACK flag. This does not prevent the page
faults from occuring, however, but marks the object in question as being
initialized so that no warnings will ever be produced for this object.
In addition to (and in contrast to) __GFP_NOTRACK, the
__GFP_NOTRACK_FALSE_POSITIVE flag indicates that the allocation should
not be tracked _because_ it would produce a false positive. Their values
are identical, but need not be so in the future (for example, we could now
enable/disable false positives with a config option).
Parts of this patch were contributed by Pekka Enberg but merged for
atomicity.
Signed-off-by: Vegard Nossum <vegard.nossum@gmail.com>
Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
[rebased for mainline inclusion]
Signed-off-by: Vegard Nossum <vegard.nossum@gmail.com>
2008-05-31 07:56:17 -06:00
|
|
|
SLAB_PANIC | SLAB_NOTRACK, NULL);
|
2008-03-10 16:28:04 -06:00
|
|
|
}
|
2008-04-25 09:39:01 -06:00
|
|
|
|
2009-02-27 14:25:28 -07:00
|
|
|
/*
|
|
|
|
* Free current thread data structures etc..
|
|
|
|
*/
|
|
|
|
void exit_thread(void)
|
|
|
|
{
|
|
|
|
struct task_struct *me = current;
|
|
|
|
struct thread_struct *t = &me->thread;
|
2009-03-16 06:07:21 -06:00
|
|
|
unsigned long *bp = t->io_bitmap_ptr;
|
2009-02-27 14:25:28 -07:00
|
|
|
|
2009-03-16 06:07:21 -06:00
|
|
|
if (bp) {
|
2009-02-27 14:25:28 -07:00
|
|
|
struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
|
|
|
|
|
|
|
|
t->io_bitmap_ptr = NULL;
|
|
|
|
clear_thread_flag(TIF_IO_BITMAP);
|
|
|
|
/*
|
|
|
|
* Careful, clear this in the TSS too:
|
|
|
|
*/
|
|
|
|
memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
|
|
|
|
t->io_bitmap_max = 0;
|
|
|
|
put_cpu();
|
2009-03-16 06:07:21 -06:00
|
|
|
kfree(bp);
|
2009-02-27 14:25:28 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-01-13 08:45:55 -07:00
|
|
|
void show_regs(struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
show_registers(regs);
|
2011-03-17 20:40:06 -06:00
|
|
|
show_trace(NULL, regs, (unsigned long *)kernel_stack_pointer(regs), 0);
|
2010-01-13 08:45:55 -07:00
|
|
|
}
|
|
|
|
|
2009-12-08 01:29:42 -07:00
|
|
|
void show_regs_common(void)
|
|
|
|
{
|
2011-02-14 15:47:17 -07:00
|
|
|
const char *vendor, *product, *board;
|
2009-12-08 01:29:42 -07:00
|
|
|
|
2011-02-14 15:47:17 -07:00
|
|
|
vendor = dmi_get_system_info(DMI_SYS_VENDOR);
|
|
|
|
if (!vendor)
|
|
|
|
vendor = "";
|
2009-12-08 01:30:21 -07:00
|
|
|
product = dmi_get_system_info(DMI_PRODUCT_NAME);
|
|
|
|
if (!product)
|
|
|
|
product = "";
|
2009-12-08 01:29:42 -07:00
|
|
|
|
2011-02-14 15:47:17 -07:00
|
|
|
/* Board Name is optional */
|
|
|
|
board = dmi_get_system_info(DMI_BOARD_NAME);
|
|
|
|
|
2009-12-28 01:26:59 -07:00
|
|
|
printk(KERN_CONT "\n");
|
2011-02-14 15:47:17 -07:00
|
|
|
printk(KERN_DEFAULT "Pid: %d, comm: %.20s %s %s %.*s",
|
2009-12-08 01:29:42 -07:00
|
|
|
current->pid, current->comm, print_tainted(),
|
|
|
|
init_utsname()->release,
|
|
|
|
(int)strcspn(init_utsname()->version, " "),
|
2011-02-14 15:47:17 -07:00
|
|
|
init_utsname()->version);
|
2011-02-17 08:56:58 -07:00
|
|
|
printk(KERN_CONT " %s %s", vendor, product);
|
|
|
|
if (board)
|
|
|
|
printk(KERN_CONT "/%s", board);
|
2011-02-14 15:47:17 -07:00
|
|
|
printk(KERN_CONT "\n");
|
2009-12-08 01:29:42 -07:00
|
|
|
}
|
|
|
|
|
2009-02-27 14:25:28 -07:00
|
|
|
void flush_thread(void)
|
|
|
|
{
|
|
|
|
struct task_struct *tsk = current;
|
|
|
|
|
2009-09-09 11:22:48 -06:00
|
|
|
flush_ptrace_hw_breakpoint(tsk);
|
2009-02-27 14:25:28 -07:00
|
|
|
memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
|
|
|
|
/*
|
|
|
|
* Forget coprocessor state..
|
|
|
|
*/
|
|
|
|
tsk->fpu_counter = 0;
|
|
|
|
clear_fpu(tsk);
|
|
|
|
clear_used_math();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hard_disable_TSC(void)
|
|
|
|
{
|
|
|
|
write_cr4(read_cr4() | X86_CR4_TSD);
|
|
|
|
}
|
|
|
|
|
|
|
|
void disable_TSC(void)
|
|
|
|
{
|
|
|
|
preempt_disable();
|
|
|
|
if (!test_and_set_thread_flag(TIF_NOTSC))
|
|
|
|
/*
|
|
|
|
* Must flip the CPU state synchronously with
|
|
|
|
* TIF_NOTSC in the current running context.
|
|
|
|
*/
|
|
|
|
hard_disable_TSC();
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hard_enable_TSC(void)
|
|
|
|
{
|
|
|
|
write_cr4(read_cr4() & ~X86_CR4_TSD);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void enable_TSC(void)
|
|
|
|
{
|
|
|
|
preempt_disable();
|
|
|
|
if (test_and_clear_thread_flag(TIF_NOTSC))
|
|
|
|
/*
|
|
|
|
* Must flip the CPU state synchronously with
|
|
|
|
* TIF_NOTSC in the current running context.
|
|
|
|
*/
|
|
|
|
hard_enable_TSC();
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
|
|
|
|
int get_tsc_mode(unsigned long adr)
|
|
|
|
{
|
|
|
|
unsigned int val;
|
|
|
|
|
|
|
|
if (test_thread_flag(TIF_NOTSC))
|
|
|
|
val = PR_TSC_SIGSEGV;
|
|
|
|
else
|
|
|
|
val = PR_TSC_ENABLE;
|
|
|
|
|
|
|
|
return put_user(val, (unsigned int __user *)adr);
|
|
|
|
}
|
|
|
|
|
|
|
|
int set_tsc_mode(unsigned int val)
|
|
|
|
{
|
|
|
|
if (val == PR_TSC_SIGSEGV)
|
|
|
|
disable_TSC();
|
|
|
|
else if (val == PR_TSC_ENABLE)
|
|
|
|
enable_TSC();
|
|
|
|
else
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
|
|
|
|
struct tss_struct *tss)
|
|
|
|
{
|
|
|
|
struct thread_struct *prev, *next;
|
|
|
|
|
|
|
|
prev = &prev_p->thread;
|
|
|
|
next = &next_p->thread;
|
|
|
|
|
2010-03-25 07:51:51 -06:00
|
|
|
if (test_tsk_thread_flag(prev_p, TIF_BLOCKSTEP) ^
|
|
|
|
test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) {
|
|
|
|
unsigned long debugctl = get_debugctlmsr();
|
|
|
|
|
|
|
|
debugctl &= ~DEBUGCTLMSR_BTF;
|
|
|
|
if (test_tsk_thread_flag(next_p, TIF_BLOCKSTEP))
|
|
|
|
debugctl |= DEBUGCTLMSR_BTF;
|
|
|
|
|
|
|
|
update_debugctlmsr(debugctl);
|
|
|
|
}
|
2009-02-27 14:25:28 -07:00
|
|
|
|
|
|
|
if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
|
|
|
|
test_tsk_thread_flag(next_p, TIF_NOTSC)) {
|
|
|
|
/* prev and next are different */
|
|
|
|
if (test_tsk_thread_flag(next_p, TIF_NOTSC))
|
|
|
|
hard_disable_TSC();
|
|
|
|
else
|
|
|
|
hard_enable_TSC();
|
|
|
|
}
|
|
|
|
|
|
|
|
if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
|
|
|
|
/*
|
|
|
|
* Copy the relevant range of the IO bitmap.
|
|
|
|
* Normally this is 128 bytes or less:
|
|
|
|
*/
|
|
|
|
memcpy(tss->io_bitmap, next->io_bitmap_ptr,
|
|
|
|
max(prev->io_bitmap_max, next->io_bitmap_max));
|
|
|
|
} else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
|
|
|
|
/*
|
|
|
|
* Clear any possible leftover bits:
|
|
|
|
*/
|
|
|
|
memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
|
|
|
|
}
|
2009-09-19 00:40:22 -06:00
|
|
|
propagate_user_return_notify(prev_p, next_p);
|
2009-02-27 14:25:28 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
int sys_fork(struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
return do_fork(SIGCHLD, regs->sp, regs, 0, NULL, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This is trivial, and on the face of it looks like it
|
|
|
|
* could equally well be done in user mode.
|
|
|
|
*
|
|
|
|
* Not so, for quite unobvious reasons - register pressure.
|
|
|
|
* In user mode vfork() cannot have a stack frame, and if
|
|
|
|
* done by calling the "clone()" system call directly, you
|
|
|
|
* do not have enough call-clobbered registers to hold all
|
|
|
|
* the information you need.
|
|
|
|
*/
|
|
|
|
int sys_vfork(struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->sp, regs, 0,
|
|
|
|
NULL, NULL);
|
|
|
|
}
|
|
|
|
|
2009-12-09 17:01:56 -07:00
|
|
|
long
|
|
|
|
sys_clone(unsigned long clone_flags, unsigned long newsp,
|
|
|
|
void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
if (!newsp)
|
|
|
|
newsp = regs->sp;
|
|
|
|
return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
|
|
|
|
}
|
|
|
|
|
2009-12-09 10:34:44 -07:00
|
|
|
/*
|
|
|
|
* This gets run with %si containing the
|
|
|
|
* function to call, and %di containing
|
|
|
|
* the "args".
|
|
|
|
*/
|
|
|
|
extern void kernel_thread_helper(void);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create a kernel thread
|
|
|
|
*/
|
|
|
|
int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
|
|
|
|
{
|
|
|
|
struct pt_regs regs;
|
|
|
|
|
|
|
|
memset(®s, 0, sizeof(regs));
|
|
|
|
|
|
|
|
regs.si = (unsigned long) fn;
|
|
|
|
regs.di = (unsigned long) arg;
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
regs.ds = __USER_DS;
|
|
|
|
regs.es = __USER_DS;
|
|
|
|
regs.fs = __KERNEL_PERCPU;
|
|
|
|
regs.gs = __KERNEL_STACK_CANARY;
|
2010-01-13 03:16:07 -07:00
|
|
|
#else
|
|
|
|
regs.ss = __KERNEL_DS;
|
2009-12-09 10:34:44 -07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
regs.orig_ax = -1;
|
|
|
|
regs.ip = (unsigned long) kernel_thread_helper;
|
|
|
|
regs.cs = __KERNEL_CS | get_kernel_rpl();
|
|
|
|
regs.flags = X86_EFLAGS_IF | 0x2;
|
|
|
|
|
|
|
|
/* Ok, create the new process.. */
|
|
|
|
return do_fork(flags | CLONE_VM | CLONE_UNTRACED, 0, ®s, 0, NULL, NULL);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kernel_thread);
|
2009-02-27 14:25:28 -07:00
|
|
|
|
2009-12-09 17:01:53 -07:00
|
|
|
/*
|
|
|
|
* sys_execve() executes a new program.
|
|
|
|
*/
|
2010-08-17 16:52:56 -06:00
|
|
|
long sys_execve(const char __user *name,
|
|
|
|
const char __user *const __user *argv,
|
|
|
|
const char __user *const __user *envp, struct pt_regs *regs)
|
2009-12-09 17:01:53 -07:00
|
|
|
{
|
|
|
|
long error;
|
|
|
|
char *filename;
|
|
|
|
|
|
|
|
filename = getname(name);
|
|
|
|
error = PTR_ERR(filename);
|
|
|
|
if (IS_ERR(filename))
|
|
|
|
return error;
|
|
|
|
error = do_execve(filename, argv, envp, regs);
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
if (error == 0) {
|
|
|
|
/* Make sure we don't return using sysenter.. */
|
|
|
|
set_thread_flag(TIF_IRET);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
putname(filename);
|
|
|
|
return error;
|
|
|
|
}
|
2009-02-27 14:25:28 -07:00
|
|
|
|
2008-06-09 10:35:28 -06:00
|
|
|
/*
|
|
|
|
* Idle related variables and functions
|
|
|
|
*/
|
2010-11-03 10:06:14 -06:00
|
|
|
unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
|
2008-06-09 10:35:28 -06:00
|
|
|
EXPORT_SYMBOL(boot_option_idle_override);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Powermanagement idle function, if any..
|
|
|
|
*/
|
|
|
|
void (*pm_idle)(void);
|
2011-04-01 13:28:09 -06:00
|
|
|
#if defined(CONFIG_APM_MODULE) && defined(CONFIG_APM_CPU_IDLE)
|
2008-06-09 10:35:28 -06:00
|
|
|
EXPORT_SYMBOL(pm_idle);
|
2011-04-01 13:28:09 -06:00
|
|
|
#endif
|
2008-06-09 10:35:28 -06:00
|
|
|
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
/*
|
|
|
|
* This halt magic was a workaround for ancient floppy DMA
|
|
|
|
* wreckage. It should be safe to remove.
|
|
|
|
*/
|
|
|
|
static int hlt_counter;
|
|
|
|
void disable_hlt(void)
|
|
|
|
{
|
|
|
|
hlt_counter++;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(disable_hlt);
|
|
|
|
|
|
|
|
void enable_hlt(void)
|
|
|
|
{
|
|
|
|
hlt_counter--;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(enable_hlt);
|
|
|
|
|
|
|
|
static inline int hlt_use_halt(void)
|
|
|
|
{
|
|
|
|
return (!hlt_counter && boot_cpu_data.hlt_works_ok);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline int hlt_use_halt(void)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We use this if we don't have any better
|
|
|
|
* idle routine..
|
|
|
|
*/
|
|
|
|
void default_idle(void)
|
|
|
|
{
|
|
|
|
if (hlt_use_halt()) {
|
2010-04-20 05:17:36 -06:00
|
|
|
trace_power_start(POWER_CSTATE, 1, smp_processor_id());
|
2011-01-03 09:50:44 -07:00
|
|
|
trace_cpu_idle(1, smp_processor_id());
|
2008-06-09 10:35:28 -06:00
|
|
|
current_thread_info()->status &= ~TS_POLLING;
|
|
|
|
/*
|
|
|
|
* TS_POLLING-cleared state must be visible before we
|
|
|
|
* test NEED_RESCHED:
|
|
|
|
*/
|
|
|
|
smp_mb();
|
|
|
|
|
|
|
|
if (!need_resched())
|
|
|
|
safe_halt(); /* enables interrupts racelessly */
|
|
|
|
else
|
|
|
|
local_irq_enable();
|
|
|
|
current_thread_info()->status |= TS_POLLING;
|
cpuidle/x86/perf: fix power:cpu_idle double end events and throw cpu_idle events from the cpuidle layer
Currently intel_idle and acpi_idle driver show double cpu_idle "exit idle"
events -> this patch fixes it and makes cpu_idle events throwing less complex.
It also introduces cpu_idle events for all architectures which use
the cpuidle subsystem, namely:
- arch/arm/mach-at91/cpuidle.c
- arch/arm/mach-davinci/cpuidle.c
- arch/arm/mach-kirkwood/cpuidle.c
- arch/arm/mach-omap2/cpuidle34xx.c
- arch/drivers/acpi/processor_idle.c (for all cases, not only mwait)
- arch/x86/kernel/process.c (did throw events before, but was a mess)
- drivers/idle/intel_idle.c (did throw events before)
Convention should be:
Fire cpu_idle events inside the current pm_idle function (not somewhere
down the the callee tree) to keep things easy.
Current possible pm_idle functions in X86:
c1e_idle, poll_idle, cpuidle_idle_call, mwait_idle, default_idle
-> this is really easy is now.
This affects userspace:
The type field of the cpu_idle power event can now direclty get
mapped to:
/sys/devices/system/cpu/cpuX/cpuidle/stateX/{name,desc,usage,time,...}
instead of throwing very CPU/mwait specific values.
This change is not visible for the intel_idle driver.
For the acpi_idle driver it should only be visible if the vendor
misses out C-states in his BIOS.
Another (perf timechart) patch reads out cpuidle info of cpu_idle
events from:
/sys/.../cpuidle/stateX/*, then the cpuidle events are mapped
to the correct C-/cpuidle state again, even if e.g. vendors miss
out C-states in their BIOS and for example only export C1 and C3.
-> everything is fine.
Signed-off-by: Thomas Renninger <trenn@suse.de>
CC: Robert Schoene <robert.schoene@tu-dresden.de>
CC: Jean Pihet <j-pihet@ti.com>
CC: Arjan van de Ven <arjan@linux.intel.com>
CC: Ingo Molnar <mingo@elte.hu>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: linux-pm@lists.linux-foundation.org
CC: linux-acpi@vger.kernel.org
CC: linux-kernel@vger.kernel.org
CC: linux-perf-users@vger.kernel.org
CC: linux-omap@vger.kernel.org
Signed-off-by: Len Brown <len.brown@intel.com>
2011-01-07 03:29:44 -07:00
|
|
|
trace_power_end(smp_processor_id());
|
|
|
|
trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
|
2008-06-09 10:35:28 -06:00
|
|
|
} else {
|
|
|
|
local_irq_enable();
|
|
|
|
/* loop is done by the caller */
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
}
|
2011-04-01 13:28:09 -06:00
|
|
|
#if defined(CONFIG_APM_MODULE) && defined(CONFIG_APM_CPU_IDLE)
|
2008-06-09 10:35:28 -06:00
|
|
|
EXPORT_SYMBOL(default_idle);
|
|
|
|
#endif
|
|
|
|
|
2008-11-11 06:33:44 -07:00
|
|
|
void stop_this_cpu(void *dummy)
|
|
|
|
{
|
|
|
|
local_irq_disable();
|
|
|
|
/*
|
|
|
|
* Remove this CPU:
|
|
|
|
*/
|
2009-03-12 22:19:54 -06:00
|
|
|
set_cpu_online(smp_processor_id(), false);
|
2008-11-11 06:33:44 -07:00
|
|
|
disable_local_APIC();
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
if (hlt_works(smp_processor_id()))
|
|
|
|
halt();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-04-25 09:39:01 -06:00
|
|
|
static void do_nothing(void *unused)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* cpu_idle_wait - Used to ensure that all the CPUs discard old value of
|
|
|
|
* pm_idle and update to new pm_idle value. Required while changing pm_idle
|
|
|
|
* handler on SMP systems.
|
|
|
|
*
|
|
|
|
* Caller must have changed pm_idle to the new value before the call. Old
|
|
|
|
* pm_idle value will not be used by any CPU after the return of this function.
|
|
|
|
*/
|
|
|
|
void cpu_idle_wait(void)
|
|
|
|
{
|
|
|
|
smp_mb();
|
|
|
|
/* kick all the CPUs so that they exit out of pm_idle */
|
2008-06-27 03:48:22 -06:00
|
|
|
smp_call_function(do_nothing, NULL, 1);
|
2008-04-25 09:39:01 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(cpu_idle_wait);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
|
|
|
|
* which can obviate IPI to trigger checking of need_resched.
|
|
|
|
* We execute MONITOR against need_resched and enter optimized wait state
|
|
|
|
* through MWAIT. Whenever someone changes need_resched, we would be woken
|
|
|
|
* up from MWAIT (without an IPI).
|
|
|
|
*
|
|
|
|
* New with Core Duo processors, MWAIT can take some hints based on CPU
|
|
|
|
* capability.
|
|
|
|
*/
|
|
|
|
void mwait_idle_with_hints(unsigned long ax, unsigned long cx)
|
|
|
|
{
|
|
|
|
if (!need_resched()) {
|
2011-03-12 04:50:10 -07:00
|
|
|
if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR))
|
2009-02-06 17:52:05 -07:00
|
|
|
clflush((void *)¤t_thread_info()->flags);
|
|
|
|
|
2008-04-25 09:39:01 -06:00
|
|
|
__monitor((void *)¤t_thread_info()->flags, 0, 0);
|
|
|
|
smp_mb();
|
|
|
|
if (!need_resched())
|
|
|
|
__mwait(ax, cx);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Default MONITOR/MWAIT with no hints, used for default C1 state */
|
|
|
|
static void mwait_idle(void)
|
|
|
|
{
|
|
|
|
if (!need_resched()) {
|
2010-04-20 05:17:36 -06:00
|
|
|
trace_power_start(POWER_CSTATE, 1, smp_processor_id());
|
2011-01-03 09:50:44 -07:00
|
|
|
trace_cpu_idle(1, smp_processor_id());
|
2011-03-12 04:50:10 -07:00
|
|
|
if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR))
|
2009-02-06 17:52:05 -07:00
|
|
|
clflush((void *)¤t_thread_info()->flags);
|
|
|
|
|
2008-04-25 09:39:01 -06:00
|
|
|
__monitor((void *)¤t_thread_info()->flags, 0, 0);
|
|
|
|
smp_mb();
|
|
|
|
if (!need_resched())
|
|
|
|
__sti_mwait(0, 0);
|
|
|
|
else
|
|
|
|
local_irq_enable();
|
cpuidle/x86/perf: fix power:cpu_idle double end events and throw cpu_idle events from the cpuidle layer
Currently intel_idle and acpi_idle driver show double cpu_idle "exit idle"
events -> this patch fixes it and makes cpu_idle events throwing less complex.
It also introduces cpu_idle events for all architectures which use
the cpuidle subsystem, namely:
- arch/arm/mach-at91/cpuidle.c
- arch/arm/mach-davinci/cpuidle.c
- arch/arm/mach-kirkwood/cpuidle.c
- arch/arm/mach-omap2/cpuidle34xx.c
- arch/drivers/acpi/processor_idle.c (for all cases, not only mwait)
- arch/x86/kernel/process.c (did throw events before, but was a mess)
- drivers/idle/intel_idle.c (did throw events before)
Convention should be:
Fire cpu_idle events inside the current pm_idle function (not somewhere
down the the callee tree) to keep things easy.
Current possible pm_idle functions in X86:
c1e_idle, poll_idle, cpuidle_idle_call, mwait_idle, default_idle
-> this is really easy is now.
This affects userspace:
The type field of the cpu_idle power event can now direclty get
mapped to:
/sys/devices/system/cpu/cpuX/cpuidle/stateX/{name,desc,usage,time,...}
instead of throwing very CPU/mwait specific values.
This change is not visible for the intel_idle driver.
For the acpi_idle driver it should only be visible if the vendor
misses out C-states in his BIOS.
Another (perf timechart) patch reads out cpuidle info of cpu_idle
events from:
/sys/.../cpuidle/stateX/*, then the cpuidle events are mapped
to the correct C-/cpuidle state again, even if e.g. vendors miss
out C-states in their BIOS and for example only export C1 and C3.
-> everything is fine.
Signed-off-by: Thomas Renninger <trenn@suse.de>
CC: Robert Schoene <robert.schoene@tu-dresden.de>
CC: Jean Pihet <j-pihet@ti.com>
CC: Arjan van de Ven <arjan@linux.intel.com>
CC: Ingo Molnar <mingo@elte.hu>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: linux-pm@lists.linux-foundation.org
CC: linux-acpi@vger.kernel.org
CC: linux-kernel@vger.kernel.org
CC: linux-perf-users@vger.kernel.org
CC: linux-omap@vger.kernel.org
Signed-off-by: Len Brown <len.brown@intel.com>
2011-01-07 03:29:44 -07:00
|
|
|
trace_power_end(smp_processor_id());
|
|
|
|
trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
|
2008-04-25 09:39:01 -06:00
|
|
|
} else
|
|
|
|
local_irq_enable();
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* On SMP it's slightly faster (but much more power-consuming!)
|
|
|
|
* to poll the ->work.need_resched flag instead of waiting for the
|
|
|
|
* cross-CPU IPI to arrive. Use this option with caution.
|
|
|
|
*/
|
|
|
|
static void poll_idle(void)
|
|
|
|
{
|
2010-04-20 05:17:36 -06:00
|
|
|
trace_power_start(POWER_CSTATE, 0, smp_processor_id());
|
2011-01-03 09:50:44 -07:00
|
|
|
trace_cpu_idle(0, smp_processor_id());
|
2008-04-25 09:39:01 -06:00
|
|
|
local_irq_enable();
|
2008-08-27 08:35:06 -06:00
|
|
|
while (!need_resched())
|
|
|
|
cpu_relax();
|
2011-01-03 09:50:44 -07:00
|
|
|
trace_power_end(smp_processor_id());
|
|
|
|
trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
|
2008-04-25 09:39:01 -06:00
|
|
|
}
|
|
|
|
|
2008-05-16 14:55:26 -06:00
|
|
|
/*
|
|
|
|
* mwait selection logic:
|
|
|
|
*
|
|
|
|
* It depends on the CPU. For AMD CPUs that support MWAIT this is
|
|
|
|
* wrong. Family 0x10 and 0x11 CPUs will enter C1 on HLT. Powersavings
|
|
|
|
* then depend on a clock divisor and current Pstate of the core. If
|
|
|
|
* all cores of a processor are in halt state (C1) the processor can
|
|
|
|
* enter the C1E (C1 enhanced) state. If mwait is used this will never
|
|
|
|
* happen.
|
|
|
|
*
|
|
|
|
* idle=mwait overrides this decision and forces the usage of mwait.
|
|
|
|
*/
|
2008-06-09 10:04:27 -06:00
|
|
|
|
|
|
|
#define MWAIT_INFO 0x05
|
|
|
|
#define MWAIT_ECX_EXTENDED_INFO 0x01
|
|
|
|
#define MWAIT_EDX_C1 0xf0
|
|
|
|
|
2011-02-11 10:17:54 -07:00
|
|
|
int mwait_usable(const struct cpuinfo_x86 *c)
|
2008-05-16 14:55:26 -06:00
|
|
|
{
|
2008-06-09 10:04:27 -06:00
|
|
|
u32 eax, ebx, ecx, edx;
|
|
|
|
|
2010-11-03 10:06:14 -06:00
|
|
|
if (boot_option_idle_override == IDLE_FORCE_MWAIT)
|
2008-05-16 14:55:26 -06:00
|
|
|
return 1;
|
|
|
|
|
2008-06-09 10:04:27 -06:00
|
|
|
if (c->cpuid_level < MWAIT_INFO)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
cpuid(MWAIT_INFO, &eax, &ebx, &ecx, &edx);
|
|
|
|
/* Check, whether EDX has extended info about MWAIT */
|
|
|
|
if (!(ecx & MWAIT_ECX_EXTENDED_INFO))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* edx enumeratios MONITOR/MWAIT extensions. Check, whether
|
|
|
|
* C1 supports MWAIT
|
|
|
|
*/
|
|
|
|
return (edx & MWAIT_EDX_C1);
|
2008-05-16 14:55:26 -06:00
|
|
|
}
|
|
|
|
|
2011-04-01 14:59:53 -06:00
|
|
|
bool amd_e400_c1e_detected;
|
|
|
|
EXPORT_SYMBOL(amd_e400_c1e_detected);
|
2008-06-09 11:15:00 -06:00
|
|
|
|
2011-04-01 14:59:53 -06:00
|
|
|
static cpumask_var_t amd_e400_c1e_mask;
|
2008-09-22 10:54:29 -06:00
|
|
|
|
2011-04-01 14:59:53 -06:00
|
|
|
void amd_e400_remove_cpu(int cpu)
|
2008-09-22 10:54:29 -06:00
|
|
|
{
|
2011-04-01 14:59:53 -06:00
|
|
|
if (amd_e400_c1e_mask != NULL)
|
|
|
|
cpumask_clear_cpu(cpu, amd_e400_c1e_mask);
|
2008-09-22 10:54:29 -06:00
|
|
|
}
|
|
|
|
|
2008-06-09 11:15:00 -06:00
|
|
|
/*
|
2011-04-01 14:59:53 -06:00
|
|
|
* AMD Erratum 400 aware idle routine. We check for C1E active in the interrupt
|
2008-06-09 11:15:00 -06:00
|
|
|
* pending message MSR. If we detect C1E, then we handle it the same
|
|
|
|
* way as C3 power states (local apic timer and TSC stop)
|
|
|
|
*/
|
2011-04-01 14:59:53 -06:00
|
|
|
static void amd_e400_idle(void)
|
2008-06-09 11:15:00 -06:00
|
|
|
{
|
|
|
|
if (need_resched())
|
|
|
|
return;
|
|
|
|
|
2011-04-01 14:59:53 -06:00
|
|
|
if (!amd_e400_c1e_detected) {
|
2008-06-09 11:15:00 -06:00
|
|
|
u32 lo, hi;
|
|
|
|
|
|
|
|
rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
|
2010-07-27 10:53:35 -06:00
|
|
|
|
2008-06-09 11:15:00 -06:00
|
|
|
if (lo & K8_INTP_C1E_ACTIVE_MASK) {
|
2011-04-01 14:59:53 -06:00
|
|
|
amd_e400_c1e_detected = true;
|
2008-11-17 17:11:37 -07:00
|
|
|
if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
|
2008-09-18 13:12:10 -06:00
|
|
|
mark_tsc_unstable("TSC halt in AMD C1E");
|
|
|
|
printk(KERN_INFO "System has AMD C1E enabled\n");
|
2008-06-09 11:15:00 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-04-01 14:59:53 -06:00
|
|
|
if (amd_e400_c1e_detected) {
|
2008-06-09 11:15:00 -06:00
|
|
|
int cpu = smp_processor_id();
|
|
|
|
|
2011-04-01 14:59:53 -06:00
|
|
|
if (!cpumask_test_cpu(cpu, amd_e400_c1e_mask)) {
|
|
|
|
cpumask_set_cpu(cpu, amd_e400_c1e_mask);
|
2008-06-17 01:12:03 -06:00
|
|
|
/*
|
2009-08-17 15:34:59 -06:00
|
|
|
* Force broadcast so ACPI can not interfere.
|
2008-06-17 01:12:03 -06:00
|
|
|
*/
|
2008-06-09 11:15:00 -06:00
|
|
|
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_FORCE,
|
|
|
|
&cpu);
|
|
|
|
printk(KERN_INFO "Switch to broadcast mode on CPU%d\n",
|
|
|
|
cpu);
|
|
|
|
}
|
|
|
|
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
|
2008-06-17 01:12:03 -06:00
|
|
|
|
2008-06-09 11:15:00 -06:00
|
|
|
default_idle();
|
2008-06-17 01:12:03 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The switch back from broadcast mode needs to be
|
|
|
|
* called with interrupts disabled.
|
|
|
|
*/
|
|
|
|
local_irq_disable();
|
|
|
|
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
|
|
|
|
local_irq_enable();
|
2008-06-09 11:15:00 -06:00
|
|
|
} else
|
|
|
|
default_idle();
|
|
|
|
}
|
|
|
|
|
2008-04-25 09:39:01 -06:00
|
|
|
void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
|
|
|
|
{
|
2009-01-27 09:07:08 -07:00
|
|
|
#ifdef CONFIG_SMP
|
2008-04-25 09:39:01 -06:00
|
|
|
if (pm_idle == poll_idle && smp_num_siblings > 1) {
|
2010-03-05 12:10:38 -07:00
|
|
|
printk_once(KERN_WARNING "WARNING: polling idle and HT enabled,"
|
2008-04-25 09:39:01 -06:00
|
|
|
" performance may degrade.\n");
|
|
|
|
}
|
|
|
|
#endif
|
2008-06-09 08:59:53 -06:00
|
|
|
if (pm_idle)
|
|
|
|
return;
|
|
|
|
|
2008-05-16 14:55:26 -06:00
|
|
|
if (cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) {
|
2008-04-25 09:39:01 -06:00
|
|
|
/*
|
|
|
|
* One CPU supports mwait => All CPUs supports mwait
|
|
|
|
*/
|
2008-06-09 08:59:53 -06:00
|
|
|
printk(KERN_INFO "using mwait in idle threads.\n");
|
|
|
|
pm_idle = mwait_idle;
|
2010-07-28 11:09:31 -06:00
|
|
|
} else if (cpu_has_amd_erratum(amd_erratum_400)) {
|
|
|
|
/* E400: APIC timer interrupt does not wake up CPU from C1e */
|
2011-04-01 14:59:53 -06:00
|
|
|
printk(KERN_INFO "using AMD E400 aware idle routine\n");
|
|
|
|
pm_idle = amd_e400_idle;
|
2008-06-09 08:59:53 -06:00
|
|
|
} else
|
|
|
|
pm_idle = default_idle;
|
2008-04-25 09:39:01 -06:00
|
|
|
}
|
|
|
|
|
2011-04-01 14:59:53 -06:00
|
|
|
void __init init_amd_e400_c1e_mask(void)
|
2009-03-16 22:20:34 -06:00
|
|
|
{
|
2011-04-01 14:59:53 -06:00
|
|
|
/* If we're using amd_e400_idle, we need to allocate amd_e400_c1e_mask. */
|
|
|
|
if (pm_idle == amd_e400_idle)
|
|
|
|
zalloc_cpumask_var(&amd_e400_c1e_mask, GFP_KERNEL);
|
2009-03-16 22:20:34 -06:00
|
|
|
}
|
|
|
|
|
2008-04-25 09:39:01 -06:00
|
|
|
static int __init idle_setup(char *str)
|
|
|
|
{
|
2008-07-05 05:53:36 -06:00
|
|
|
if (!str)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2008-04-25 09:39:01 -06:00
|
|
|
if (!strcmp(str, "poll")) {
|
|
|
|
printk("using polling idle threads.\n");
|
|
|
|
pm_idle = poll_idle;
|
2010-11-03 10:06:14 -06:00
|
|
|
boot_option_idle_override = IDLE_POLL;
|
|
|
|
} else if (!strcmp(str, "mwait")) {
|
|
|
|
boot_option_idle_override = IDLE_FORCE_MWAIT;
|
2011-05-31 11:07:22 -06:00
|
|
|
WARN_ONCE(1, "\"idle=mwait\" will be removed in 2012\n");
|
2010-11-03 10:06:14 -06:00
|
|
|
} else if (!strcmp(str, "halt")) {
|
2008-06-24 03:58:53 -06:00
|
|
|
/*
|
|
|
|
* When the boot option of idle=halt is added, halt is
|
|
|
|
* forced to be used for CPU idle. In such case CPU C2/C3
|
|
|
|
* won't be used again.
|
|
|
|
* To continue to load the CPU idle driver, don't touch
|
|
|
|
* the boot_option_idle_override.
|
|
|
|
*/
|
|
|
|
pm_idle = default_idle;
|
2010-11-03 10:06:14 -06:00
|
|
|
boot_option_idle_override = IDLE_HALT;
|
2008-06-24 04:01:09 -06:00
|
|
|
} else if (!strcmp(str, "nomwait")) {
|
|
|
|
/*
|
|
|
|
* If the boot option of "idle=nomwait" is added,
|
|
|
|
* it means that mwait will be disabled for CPU C2/C3
|
|
|
|
* states. In such case it won't touch the variable
|
|
|
|
* of boot_option_idle_override.
|
|
|
|
*/
|
2010-11-03 10:06:14 -06:00
|
|
|
boot_option_idle_override = IDLE_NOMWAIT;
|
2008-06-24 03:58:53 -06:00
|
|
|
} else
|
2008-04-25 09:39:01 -06:00
|
|
|
return -1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("idle", idle_setup);
|
|
|
|
|
2009-05-11 20:05:28 -06:00
|
|
|
unsigned long arch_align_stack(unsigned long sp)
|
|
|
|
{
|
|
|
|
if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
|
|
|
|
sp -= get_random_int() % 8192;
|
|
|
|
return sp & ~0xf;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long arch_randomize_brk(struct mm_struct *mm)
|
|
|
|
{
|
|
|
|
unsigned long range_end = mm->brk + 0x02000000;
|
|
|
|
return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
|
|
|
|
}
|
|
|
|
|