sched: fix cpu clock
David Miller pointed it out that nothing in cpu_clock() sets prev_cpu_time. This caused __sync_cpu_clock() to be called all the time - against the intention of this code. The result was that in practice we hit a global spinlock every time cpu_clock() is called - which - even though cpu_clock() is used for tracing and debugging, is suboptimal. While at it, also: - move the irq disabling to the outest layer, this should make cpu_clock() warp-free when called with irqs enabled. - use long long instead of cycles_t - for platforms where cycles_t is 32-bit. Reported-by: David Miller <davem@davemloft.net> Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
parent
cb4ad1ffc7
commit
dfbf4a1bc3
|
@ -910,11 +910,14 @@ static DEFINE_PER_CPU(unsigned long long, prev_cpu_time);
|
||||||
static DEFINE_SPINLOCK(time_sync_lock);
|
static DEFINE_SPINLOCK(time_sync_lock);
|
||||||
static unsigned long long prev_global_time;
|
static unsigned long long prev_global_time;
|
||||||
|
|
||||||
static unsigned long long __sync_cpu_clock(cycles_t time, int cpu)
|
static unsigned long long __sync_cpu_clock(unsigned long long time, int cpu)
|
||||||
{
|
{
|
||||||
unsigned long flags;
|
/*
|
||||||
|
* We want this inlined, to not get tracer function calls
|
||||||
spin_lock_irqsave(&time_sync_lock, flags);
|
* in this critical section:
|
||||||
|
*/
|
||||||
|
spin_acquire(&time_sync_lock.dep_map, 0, 0, _THIS_IP_);
|
||||||
|
__raw_spin_lock(&time_sync_lock.raw_lock);
|
||||||
|
|
||||||
if (time < prev_global_time) {
|
if (time < prev_global_time) {
|
||||||
per_cpu(time_offset, cpu) += prev_global_time - time;
|
per_cpu(time_offset, cpu) += prev_global_time - time;
|
||||||
|
@ -923,7 +926,8 @@ static unsigned long long __sync_cpu_clock(cycles_t time, int cpu)
|
||||||
prev_global_time = time;
|
prev_global_time = time;
|
||||||
}
|
}
|
||||||
|
|
||||||
spin_unlock_irqrestore(&time_sync_lock, flags);
|
__raw_spin_unlock(&time_sync_lock.raw_lock);
|
||||||
|
spin_release(&time_sync_lock.dep_map, 1, _THIS_IP_);
|
||||||
|
|
||||||
return time;
|
return time;
|
||||||
}
|
}
|
||||||
|
@ -931,7 +935,6 @@ static unsigned long long __sync_cpu_clock(cycles_t time, int cpu)
|
||||||
static unsigned long long __cpu_clock(int cpu)
|
static unsigned long long __cpu_clock(int cpu)
|
||||||
{
|
{
|
||||||
unsigned long long now;
|
unsigned long long now;
|
||||||
unsigned long flags;
|
|
||||||
struct rq *rq;
|
struct rq *rq;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -941,11 +944,9 @@ static unsigned long long __cpu_clock(int cpu)
|
||||||
if (unlikely(!scheduler_running))
|
if (unlikely(!scheduler_running))
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
local_irq_save(flags);
|
|
||||||
rq = cpu_rq(cpu);
|
rq = cpu_rq(cpu);
|
||||||
update_rq_clock(rq);
|
update_rq_clock(rq);
|
||||||
now = rq->clock;
|
now = rq->clock;
|
||||||
local_irq_restore(flags);
|
|
||||||
|
|
||||||
return now;
|
return now;
|
||||||
}
|
}
|
||||||
|
@ -957,13 +958,18 @@ static unsigned long long __cpu_clock(int cpu)
|
||||||
unsigned long long cpu_clock(int cpu)
|
unsigned long long cpu_clock(int cpu)
|
||||||
{
|
{
|
||||||
unsigned long long prev_cpu_time, time, delta_time;
|
unsigned long long prev_cpu_time, time, delta_time;
|
||||||
|
unsigned long flags;
|
||||||
|
|
||||||
|
local_irq_save(flags);
|
||||||
prev_cpu_time = per_cpu(prev_cpu_time, cpu);
|
prev_cpu_time = per_cpu(prev_cpu_time, cpu);
|
||||||
time = __cpu_clock(cpu) + per_cpu(time_offset, cpu);
|
time = __cpu_clock(cpu) + per_cpu(time_offset, cpu);
|
||||||
delta_time = time-prev_cpu_time;
|
delta_time = time-prev_cpu_time;
|
||||||
|
|
||||||
if (unlikely(delta_time > time_sync_thresh))
|
if (unlikely(delta_time > time_sync_thresh)) {
|
||||||
time = __sync_cpu_clock(time, cpu);
|
time = __sync_cpu_clock(time, cpu);
|
||||||
|
per_cpu(prev_cpu_time, cpu) = time;
|
||||||
|
}
|
||||||
|
local_irq_restore(flags);
|
||||||
|
|
||||||
return time;
|
return time;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue