![Martin Schwidefsky](/assets/img/avatar_default.png)
Use the ACCESS_ONCE macro for both accesses to idle->sequence in the loops to calculate the idle time. If only one access uses the macro, the compiler is free to cache the value for the second access which can cause endless loops. Cc: stable@vger.kernel.org # 3.6+ Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
403 lines
10 KiB
C
403 lines
10 KiB
C
/*
|
|
* Virtual cpu timer based timer functions.
|
|
*
|
|
* Copyright IBM Corp. 2004, 2012
|
|
* Author(s): Jan Glauber <jan.glauber@de.ibm.com>
|
|
*/
|
|
|
|
#include <linux/kernel_stat.h>
|
|
#include <linux/notifier.h>
|
|
#include <linux/kprobes.h>
|
|
#include <linux/export.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/timex.h>
|
|
#include <linux/types.h>
|
|
#include <linux/time.h>
|
|
#include <linux/cpu.h>
|
|
#include <linux/smp.h>
|
|
|
|
#include <asm/irq_regs.h>
|
|
#include <asm/cputime.h>
|
|
#include <asm/vtimer.h>
|
|
#include <asm/vtime.h>
|
|
#include <asm/irq.h>
|
|
#include "entry.h"
|
|
|
|
static void virt_timer_expire(void);
|
|
|
|
DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
|
|
|
|
static LIST_HEAD(virt_timer_list);
|
|
static DEFINE_SPINLOCK(virt_timer_lock);
|
|
static atomic64_t virt_timer_current;
|
|
static atomic64_t virt_timer_elapsed;
|
|
|
|
static inline u64 get_vtimer(void)
|
|
{
|
|
u64 timer;
|
|
|
|
asm volatile("stpt %0" : "=m" (timer));
|
|
return timer;
|
|
}
|
|
|
|
static inline void set_vtimer(u64 expires)
|
|
{
|
|
u64 timer;
|
|
|
|
asm volatile(
|
|
" stpt %0\n" /* Store current cpu timer value */
|
|
" spt %1" /* Set new value imm. afterwards */
|
|
: "=m" (timer) : "m" (expires));
|
|
S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
|
|
S390_lowcore.last_update_timer = expires;
|
|
}
|
|
|
|
static inline int virt_timer_forward(u64 elapsed)
|
|
{
|
|
BUG_ON(!irqs_disabled());
|
|
|
|
if (list_empty(&virt_timer_list))
|
|
return 0;
|
|
elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
|
|
return elapsed >= atomic64_read(&virt_timer_current);
|
|
}
|
|
|
|
/*
|
|
* Update process times based on virtual cpu times stored by entry.S
|
|
* to the lowcore fields user_timer, system_timer & steal_clock.
|
|
*/
|
|
static int do_account_vtime(struct task_struct *tsk, int hardirq_offset)
|
|
{
|
|
struct thread_info *ti = task_thread_info(tsk);
|
|
u64 timer, clock, user, system, steal;
|
|
|
|
timer = S390_lowcore.last_update_timer;
|
|
clock = S390_lowcore.last_update_clock;
|
|
asm volatile(
|
|
" stpt %0\n" /* Store current cpu timer value */
|
|
" stck %1" /* Store current tod clock value */
|
|
: "=m" (S390_lowcore.last_update_timer),
|
|
"=m" (S390_lowcore.last_update_clock));
|
|
S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
|
|
S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
|
|
|
|
user = S390_lowcore.user_timer - ti->user_timer;
|
|
S390_lowcore.steal_timer -= user;
|
|
ti->user_timer = S390_lowcore.user_timer;
|
|
account_user_time(tsk, user, user);
|
|
|
|
system = S390_lowcore.system_timer - ti->system_timer;
|
|
S390_lowcore.steal_timer -= system;
|
|
ti->system_timer = S390_lowcore.system_timer;
|
|
account_system_time(tsk, hardirq_offset, system, system);
|
|
|
|
steal = S390_lowcore.steal_timer;
|
|
if ((s64) steal > 0) {
|
|
S390_lowcore.steal_timer = 0;
|
|
account_steal_time(steal);
|
|
}
|
|
|
|
return virt_timer_forward(user + system);
|
|
}
|
|
|
|
void vtime_task_switch(struct task_struct *prev)
|
|
{
|
|
struct thread_info *ti;
|
|
|
|
do_account_vtime(prev, 0);
|
|
ti = task_thread_info(prev);
|
|
ti->user_timer = S390_lowcore.user_timer;
|
|
ti->system_timer = S390_lowcore.system_timer;
|
|
ti = task_thread_info(current);
|
|
S390_lowcore.user_timer = ti->user_timer;
|
|
S390_lowcore.system_timer = ti->system_timer;
|
|
}
|
|
|
|
/*
|
|
* In s390, accounting pending user time also implies
|
|
* accounting system time in order to correctly compute
|
|
* the stolen time accounting.
|
|
*/
|
|
void vtime_account_user(struct task_struct *tsk)
|
|
{
|
|
if (do_account_vtime(tsk, HARDIRQ_OFFSET))
|
|
virt_timer_expire();
|
|
}
|
|
|
|
/*
|
|
* Update process times based on virtual cpu times stored by entry.S
|
|
* to the lowcore fields user_timer, system_timer & steal_clock.
|
|
*/
|
|
void vtime_account_irq_enter(struct task_struct *tsk)
|
|
{
|
|
struct thread_info *ti = task_thread_info(tsk);
|
|
u64 timer, system;
|
|
|
|
WARN_ON_ONCE(!irqs_disabled());
|
|
|
|
timer = S390_lowcore.last_update_timer;
|
|
S390_lowcore.last_update_timer = get_vtimer();
|
|
S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
|
|
|
|
system = S390_lowcore.system_timer - ti->system_timer;
|
|
S390_lowcore.steal_timer -= system;
|
|
ti->system_timer = S390_lowcore.system_timer;
|
|
account_system_time(tsk, 0, system, system);
|
|
|
|
virt_timer_forward(system);
|
|
}
|
|
EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
|
|
|
|
void vtime_account_system(struct task_struct *tsk)
|
|
__attribute__((alias("vtime_account_irq_enter")));
|
|
EXPORT_SYMBOL_GPL(vtime_account_system);
|
|
|
|
void __kprobes vtime_stop_cpu(void)
|
|
{
|
|
struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
|
|
unsigned long long idle_time;
|
|
unsigned long psw_mask;
|
|
|
|
trace_hardirqs_on();
|
|
|
|
/* Wait for external, I/O or machine check interrupt. */
|
|
psw_mask = PSW_KERNEL_BITS | PSW_MASK_WAIT | PSW_MASK_DAT |
|
|
PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
|
|
idle->nohz_delay = 0;
|
|
|
|
/* Call the assembler magic in entry.S */
|
|
psw_idle(idle, psw_mask);
|
|
|
|
/* Account time spent with enabled wait psw loaded as idle time. */
|
|
idle->sequence++;
|
|
smp_wmb();
|
|
idle_time = idle->clock_idle_exit - idle->clock_idle_enter;
|
|
idle->clock_idle_enter = idle->clock_idle_exit = 0ULL;
|
|
idle->idle_time += idle_time;
|
|
idle->idle_count++;
|
|
account_idle_time(idle_time);
|
|
smp_wmb();
|
|
idle->sequence++;
|
|
}
|
|
|
|
cputime64_t s390_get_idle_time(int cpu)
|
|
{
|
|
struct s390_idle_data *idle = &per_cpu(s390_idle, cpu);
|
|
unsigned long long now, idle_enter, idle_exit;
|
|
unsigned int sequence;
|
|
|
|
do {
|
|
now = get_tod_clock();
|
|
sequence = ACCESS_ONCE(idle->sequence);
|
|
idle_enter = ACCESS_ONCE(idle->clock_idle_enter);
|
|
idle_exit = ACCESS_ONCE(idle->clock_idle_exit);
|
|
} while ((sequence & 1) || (ACCESS_ONCE(idle->sequence) != sequence));
|
|
return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0;
|
|
}
|
|
|
|
/*
|
|
* Sorted add to a list. List is linear searched until first bigger
|
|
* element is found.
|
|
*/
|
|
static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
|
|
{
|
|
struct vtimer_list *tmp;
|
|
|
|
list_for_each_entry(tmp, head, entry) {
|
|
if (tmp->expires > timer->expires) {
|
|
list_add_tail(&timer->entry, &tmp->entry);
|
|
return;
|
|
}
|
|
}
|
|
list_add_tail(&timer->entry, head);
|
|
}
|
|
|
|
/*
|
|
* Handler for expired virtual CPU timer.
|
|
*/
|
|
static void virt_timer_expire(void)
|
|
{
|
|
struct vtimer_list *timer, *tmp;
|
|
unsigned long elapsed;
|
|
LIST_HEAD(cb_list);
|
|
|
|
/* walk timer list, fire all expired timers */
|
|
spin_lock(&virt_timer_lock);
|
|
elapsed = atomic64_read(&virt_timer_elapsed);
|
|
list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
|
|
if (timer->expires < elapsed)
|
|
/* move expired timer to the callback queue */
|
|
list_move_tail(&timer->entry, &cb_list);
|
|
else
|
|
timer->expires -= elapsed;
|
|
}
|
|
if (!list_empty(&virt_timer_list)) {
|
|
timer = list_first_entry(&virt_timer_list,
|
|
struct vtimer_list, entry);
|
|
atomic64_set(&virt_timer_current, timer->expires);
|
|
}
|
|
atomic64_sub(elapsed, &virt_timer_elapsed);
|
|
spin_unlock(&virt_timer_lock);
|
|
|
|
/* Do callbacks and recharge periodic timers */
|
|
list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
|
|
list_del_init(&timer->entry);
|
|
timer->function(timer->data);
|
|
if (timer->interval) {
|
|
/* Recharge interval timer */
|
|
timer->expires = timer->interval +
|
|
atomic64_read(&virt_timer_elapsed);
|
|
spin_lock(&virt_timer_lock);
|
|
list_add_sorted(timer, &virt_timer_list);
|
|
spin_unlock(&virt_timer_lock);
|
|
}
|
|
}
|
|
}
|
|
|
|
void init_virt_timer(struct vtimer_list *timer)
|
|
{
|
|
timer->function = NULL;
|
|
INIT_LIST_HEAD(&timer->entry);
|
|
}
|
|
EXPORT_SYMBOL(init_virt_timer);
|
|
|
|
static inline int vtimer_pending(struct vtimer_list *timer)
|
|
{
|
|
return !list_empty(&timer->entry);
|
|
}
|
|
|
|
static void internal_add_vtimer(struct vtimer_list *timer)
|
|
{
|
|
if (list_empty(&virt_timer_list)) {
|
|
/* First timer, just program it. */
|
|
atomic64_set(&virt_timer_current, timer->expires);
|
|
atomic64_set(&virt_timer_elapsed, 0);
|
|
list_add(&timer->entry, &virt_timer_list);
|
|
} else {
|
|
/* Update timer against current base. */
|
|
timer->expires += atomic64_read(&virt_timer_elapsed);
|
|
if (likely((s64) timer->expires <
|
|
(s64) atomic64_read(&virt_timer_current)))
|
|
/* The new timer expires before the current timer. */
|
|
atomic64_set(&virt_timer_current, timer->expires);
|
|
/* Insert new timer into the list. */
|
|
list_add_sorted(timer, &virt_timer_list);
|
|
}
|
|
}
|
|
|
|
static void __add_vtimer(struct vtimer_list *timer, int periodic)
|
|
{
|
|
unsigned long flags;
|
|
|
|
timer->interval = periodic ? timer->expires : 0;
|
|
spin_lock_irqsave(&virt_timer_lock, flags);
|
|
internal_add_vtimer(timer);
|
|
spin_unlock_irqrestore(&virt_timer_lock, flags);
|
|
}
|
|
|
|
/*
|
|
* add_virt_timer - add an oneshot virtual CPU timer
|
|
*/
|
|
void add_virt_timer(struct vtimer_list *timer)
|
|
{
|
|
__add_vtimer(timer, 0);
|
|
}
|
|
EXPORT_SYMBOL(add_virt_timer);
|
|
|
|
/*
|
|
* add_virt_timer_int - add an interval virtual CPU timer
|
|
*/
|
|
void add_virt_timer_periodic(struct vtimer_list *timer)
|
|
{
|
|
__add_vtimer(timer, 1);
|
|
}
|
|
EXPORT_SYMBOL(add_virt_timer_periodic);
|
|
|
|
static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
|
|
{
|
|
unsigned long flags;
|
|
int rc;
|
|
|
|
BUG_ON(!timer->function);
|
|
|
|
if (timer->expires == expires && vtimer_pending(timer))
|
|
return 1;
|
|
spin_lock_irqsave(&virt_timer_lock, flags);
|
|
rc = vtimer_pending(timer);
|
|
if (rc)
|
|
list_del_init(&timer->entry);
|
|
timer->interval = periodic ? expires : 0;
|
|
timer->expires = expires;
|
|
internal_add_vtimer(timer);
|
|
spin_unlock_irqrestore(&virt_timer_lock, flags);
|
|
return rc;
|
|
}
|
|
|
|
/*
|
|
* returns whether it has modified a pending timer (1) or not (0)
|
|
*/
|
|
int mod_virt_timer(struct vtimer_list *timer, u64 expires)
|
|
{
|
|
return __mod_vtimer(timer, expires, 0);
|
|
}
|
|
EXPORT_SYMBOL(mod_virt_timer);
|
|
|
|
/*
|
|
* returns whether it has modified a pending timer (1) or not (0)
|
|
*/
|
|
int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
|
|
{
|
|
return __mod_vtimer(timer, expires, 1);
|
|
}
|
|
EXPORT_SYMBOL(mod_virt_timer_periodic);
|
|
|
|
/*
|
|
* Delete a virtual timer.
|
|
*
|
|
* returns whether the deleted timer was pending (1) or not (0)
|
|
*/
|
|
int del_virt_timer(struct vtimer_list *timer)
|
|
{
|
|
unsigned long flags;
|
|
|
|
if (!vtimer_pending(timer))
|
|
return 0;
|
|
spin_lock_irqsave(&virt_timer_lock, flags);
|
|
list_del_init(&timer->entry);
|
|
spin_unlock_irqrestore(&virt_timer_lock, flags);
|
|
return 1;
|
|
}
|
|
EXPORT_SYMBOL(del_virt_timer);
|
|
|
|
/*
|
|
* Start the virtual CPU timer on the current CPU.
|
|
*/
|
|
void init_cpu_vtimer(void)
|
|
{
|
|
/* set initial cpu timer */
|
|
set_vtimer(VTIMER_MAX_SLICE);
|
|
}
|
|
|
|
static int s390_nohz_notify(struct notifier_block *self, unsigned long action,
|
|
void *hcpu)
|
|
{
|
|
struct s390_idle_data *idle;
|
|
long cpu = (long) hcpu;
|
|
|
|
idle = &per_cpu(s390_idle, cpu);
|
|
switch (action & ~CPU_TASKS_FROZEN) {
|
|
case CPU_DYING:
|
|
idle->nohz_delay = 0;
|
|
default:
|
|
break;
|
|
}
|
|
return NOTIFY_OK;
|
|
}
|
|
|
|
void __init vtime_init(void)
|
|
{
|
|
/* Enable cpu timer interrupts on the boot cpu. */
|
|
init_cpu_vtimer();
|
|
cpu_notifier(s390_nohz_notify, 0);
|
|
}
|