2006-01-09 21:52:32 -07:00
|
|
|
/*
|
|
|
|
* linux/kernel/hrtimer.c
|
|
|
|
*
|
2007-02-16 02:27:50 -07:00
|
|
|
* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
|
2007-02-16 02:28:03 -07:00
|
|
|
* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
|
2007-02-16 02:28:11 -07:00
|
|
|
* Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
|
2006-01-09 21:52:32 -07:00
|
|
|
*
|
|
|
|
* High-resolution kernel timers
|
|
|
|
*
|
|
|
|
* In contrast to the low-resolution timeout API implemented in
|
|
|
|
* kernel/timer.c, hrtimers provide finer resolution and accuracy
|
|
|
|
* depending on system configuration and capabilities.
|
|
|
|
*
|
|
|
|
* These timers are currently used for:
|
|
|
|
* - itimers
|
|
|
|
* - POSIX timers
|
|
|
|
* - nanosleep
|
|
|
|
* - precise in-kernel timing
|
|
|
|
*
|
|
|
|
* Started by: Thomas Gleixner and Ingo Molnar
|
|
|
|
*
|
|
|
|
* Credits:
|
|
|
|
* based on kernel/timer.c
|
|
|
|
*
|
2006-02-01 04:05:13 -07:00
|
|
|
* Help, testing, suggestions, bugfixes, improvements were
|
|
|
|
* provided by:
|
|
|
|
*
|
|
|
|
* George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
|
|
|
|
* et. al.
|
|
|
|
*
|
2006-01-09 21:52:32 -07:00
|
|
|
* For licencing details see kernel-base/COPYING
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/hrtimer.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/syscalls.h>
|
2007-02-16 02:28:11 -07:00
|
|
|
#include <linux/kallsyms.h>
|
2006-01-09 21:52:32 -07:00
|
|
|
#include <linux/interrupt.h>
|
2007-02-16 02:28:03 -07:00
|
|
|
#include <linux/tick.h>
|
2007-02-16 02:28:11 -07:00
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/err.h>
|
2008-04-30 01:55:04 -06:00
|
|
|
#include <linux/debugobjects.h>
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
#include <asm/uaccess.h>
|
|
|
|
|
|
|
|
/**
|
|
|
|
* ktime_get - get the monotonic time in ktime_t format
|
|
|
|
*
|
|
|
|
* returns the time in ktime_t format
|
|
|
|
*/
|
2007-02-16 02:28:00 -07:00
|
|
|
ktime_t ktime_get(void)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
|
|
|
struct timespec now;
|
|
|
|
|
|
|
|
ktime_get_ts(&now);
|
|
|
|
|
|
|
|
return timespec_to_ktime(now);
|
|
|
|
}
|
2007-03-16 02:18:42 -06:00
|
|
|
EXPORT_SYMBOL_GPL(ktime_get);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
/**
|
|
|
|
* ktime_get_real - get the real (wall-) time in ktime_t format
|
|
|
|
*
|
|
|
|
* returns the time in ktime_t format
|
|
|
|
*/
|
2007-02-16 02:28:00 -07:00
|
|
|
ktime_t ktime_get_real(void)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
|
|
|
struct timespec now;
|
|
|
|
|
|
|
|
getnstimeofday(&now);
|
|
|
|
|
|
|
|
return timespec_to_ktime(now);
|
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL_GPL(ktime_get_real);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The timer bases:
|
2006-02-01 04:05:11 -07:00
|
|
|
*
|
|
|
|
* Note: If we want to add new timer bases, we have to skip the two
|
|
|
|
* clock ids captured by the cpu-timers. We do this by holding empty
|
|
|
|
* entries rather than doing math adjustment of the clock ids.
|
|
|
|
* This ensures that we capture erroneous accesses to these clock ids
|
|
|
|
* rather than moving them into the range of valid clock id's.
|
2006-01-09 21:52:32 -07:00
|
|
|
*/
|
2007-02-16 02:28:11 -07:00
|
|
|
DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
|
|
|
|
.clock_base =
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
{
|
|
|
|
.index = CLOCK_REALTIME,
|
|
|
|
.get_time = &ktime_get_real,
|
2007-02-16 02:28:11 -07:00
|
|
|
.resolution = KTIME_LOW_RES,
|
2007-02-16 02:27:50 -07:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.index = CLOCK_MONOTONIC,
|
|
|
|
.get_time = &ktime_get,
|
2007-02-16 02:28:11 -07:00
|
|
|
.resolution = KTIME_LOW_RES,
|
2007-02-16 02:27:50 -07:00
|
|
|
},
|
|
|
|
}
|
2006-01-09 21:52:32 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* ktime_get_ts - get the monotonic clock in timespec format
|
|
|
|
* @ts: pointer to timespec variable
|
|
|
|
*
|
|
|
|
* The function calculates the monotonic clock from the realtime
|
|
|
|
* clock and the wall_to_monotonic offset and stores the result
|
2007-02-10 02:45:59 -07:00
|
|
|
* in normalized timespec format in the variable pointed to by @ts.
|
2006-01-09 21:52:32 -07:00
|
|
|
*/
|
|
|
|
void ktime_get_ts(struct timespec *ts)
|
|
|
|
{
|
|
|
|
struct timespec tomono;
|
|
|
|
unsigned long seq;
|
|
|
|
|
|
|
|
do {
|
|
|
|
seq = read_seqbegin(&xtime_lock);
|
|
|
|
getnstimeofday(ts);
|
|
|
|
tomono = wall_to_monotonic;
|
|
|
|
|
|
|
|
} while (read_seqretry(&xtime_lock, seq));
|
|
|
|
|
|
|
|
set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
|
|
|
|
ts->tv_nsec + tomono.tv_nsec);
|
|
|
|
}
|
2006-01-09 21:52:39 -07:00
|
|
|
EXPORT_SYMBOL_GPL(ktime_get_ts);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2006-03-26 02:38:05 -07:00
|
|
|
/*
|
|
|
|
* Get the coarse grained time at the softirq based on xtime and
|
|
|
|
* wall_to_monotonic.
|
|
|
|
*/
|
2007-02-16 02:27:50 -07:00
|
|
|
static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base)
|
2006-03-26 02:38:05 -07:00
|
|
|
{
|
|
|
|
ktime_t xtim, tomono;
|
2007-03-16 15:38:21 -06:00
|
|
|
struct timespec xts, tom;
|
2006-03-26 02:38:05 -07:00
|
|
|
unsigned long seq;
|
|
|
|
|
|
|
|
do {
|
|
|
|
seq = read_seqbegin(&xtime_lock);
|
2007-07-24 18:47:43 -06:00
|
|
|
xts = current_kernel_time();
|
2007-03-16 15:38:21 -06:00
|
|
|
tom = wall_to_monotonic;
|
2006-03-26 02:38:05 -07:00
|
|
|
} while (read_seqretry(&xtime_lock, seq));
|
|
|
|
|
2007-02-16 02:27:26 -07:00
|
|
|
xtim = timespec_to_ktime(xts);
|
2007-03-16 15:38:21 -06:00
|
|
|
tomono = timespec_to_ktime(tom);
|
2007-02-16 02:27:50 -07:00
|
|
|
base->clock_base[CLOCK_REALTIME].softirq_time = xtim;
|
|
|
|
base->clock_base[CLOCK_MONOTONIC].softirq_time =
|
|
|
|
ktime_add(xtim, tomono);
|
2006-03-26 02:38:05 -07:00
|
|
|
}
|
|
|
|
|
2006-01-09 21:52:32 -07:00
|
|
|
/*
|
|
|
|
* Functions and macros which are different for UP/SMP systems are kept in a
|
|
|
|
* single place
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
|
|
|
|
* means that all timers which are tied to this base via timer->base are
|
|
|
|
* locked, and the base itself is locked too.
|
|
|
|
*
|
|
|
|
* So __run_timers/migrate_timers can safely modify all timers which could
|
|
|
|
* be found on the lists/queues.
|
|
|
|
*
|
|
|
|
* When the timer's base is locked, and the timer removed from list, it is
|
|
|
|
* possible to set timer->base = NULL and drop the lock: the timer remains
|
|
|
|
* locked.
|
|
|
|
*/
|
2007-02-16 02:27:50 -07:00
|
|
|
static
|
|
|
|
struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
|
|
|
|
unsigned long *flags)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_clock_base *base;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
base = timer->base;
|
|
|
|
if (likely(base != NULL)) {
|
2007-02-16 02:27:50 -07:00
|
|
|
spin_lock_irqsave(&base->cpu_base->lock, *flags);
|
2006-01-09 21:52:32 -07:00
|
|
|
if (likely(base == timer->base))
|
|
|
|
return base;
|
|
|
|
/* The timer has migrated to another CPU: */
|
2007-02-16 02:27:50 -07:00
|
|
|
spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Switch the timer base to the current CPU when possible.
|
|
|
|
*/
|
2007-02-16 02:27:50 -07:00
|
|
|
static inline struct hrtimer_clock_base *
|
|
|
|
switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_clock_base *new_base;
|
|
|
|
struct hrtimer_cpu_base *new_cpu_base;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2007-02-16 02:27:50 -07:00
|
|
|
new_cpu_base = &__get_cpu_var(hrtimer_bases);
|
|
|
|
new_base = &new_cpu_base->clock_base[base->index];
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
if (base != new_base) {
|
|
|
|
/*
|
|
|
|
* We are trying to schedule the timer on the local CPU.
|
|
|
|
* However we can't change timer's base while it is running,
|
|
|
|
* so we keep it on the same CPU. No hassle vs. reprogramming
|
|
|
|
* the event source in the high resolution case. The softirq
|
|
|
|
* code will take care of this when the timer function has
|
|
|
|
* completed. There is no conflict as we hold the lock until
|
|
|
|
* the timer is enqueued.
|
|
|
|
*/
|
2007-02-16 02:28:11 -07:00
|
|
|
if (unlikely(hrtimer_callback_running(timer)))
|
2006-01-09 21:52:32 -07:00
|
|
|
return base;
|
|
|
|
|
|
|
|
/* See the comment in lock_timer_base() */
|
|
|
|
timer->base = NULL;
|
2007-02-16 02:27:50 -07:00
|
|
|
spin_unlock(&base->cpu_base->lock);
|
|
|
|
spin_lock(&new_base->cpu_base->lock);
|
2006-01-09 21:52:32 -07:00
|
|
|
timer->base = new_base;
|
|
|
|
}
|
|
|
|
return new_base;
|
|
|
|
}
|
|
|
|
|
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
|
2007-02-16 02:27:50 -07:00
|
|
|
static inline struct hrtimer_clock_base *
|
2006-01-09 21:52:32 -07:00
|
|
|
lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
|
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_clock_base *base = timer->base;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2007-02-16 02:27:50 -07:00
|
|
|
spin_lock_irqsave(&base->cpu_base->lock, *flags);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
return base;
|
|
|
|
}
|
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
# define switch_hrtimer_base(t, b) (b)
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
#endif /* !CONFIG_SMP */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Functions for the union type storage format of ktime_t which are
|
|
|
|
* too large for inlining:
|
|
|
|
*/
|
|
|
|
#if BITS_PER_LONG < 64
|
|
|
|
# ifndef CONFIG_KTIME_SCALAR
|
|
|
|
/**
|
|
|
|
* ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable
|
|
|
|
* @kt: addend
|
|
|
|
* @nsec: the scalar nsec value to add
|
|
|
|
*
|
|
|
|
* Returns the sum of kt and nsec in ktime_t format
|
|
|
|
*/
|
|
|
|
ktime_t ktime_add_ns(const ktime_t kt, u64 nsec)
|
|
|
|
{
|
|
|
|
ktime_t tmp;
|
|
|
|
|
|
|
|
if (likely(nsec < NSEC_PER_SEC)) {
|
|
|
|
tmp.tv64 = nsec;
|
|
|
|
} else {
|
|
|
|
unsigned long rem = do_div(nsec, NSEC_PER_SEC);
|
|
|
|
|
|
|
|
tmp = ktime_set((long)nsec, rem);
|
|
|
|
}
|
|
|
|
|
|
|
|
return ktime_add(kt, tmp);
|
|
|
|
}
|
2007-04-27 16:31:24 -06:00
|
|
|
|
|
|
|
EXPORT_SYMBOL_GPL(ktime_add_ns);
|
2007-08-19 18:16:05 -06:00
|
|
|
|
|
|
|
/**
|
|
|
|
* ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable
|
|
|
|
* @kt: minuend
|
|
|
|
* @nsec: the scalar nsec value to subtract
|
|
|
|
*
|
|
|
|
* Returns the subtraction of @nsec from @kt in ktime_t format
|
|
|
|
*/
|
|
|
|
ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec)
|
|
|
|
{
|
|
|
|
ktime_t tmp;
|
|
|
|
|
|
|
|
if (likely(nsec < NSEC_PER_SEC)) {
|
|
|
|
tmp.tv64 = nsec;
|
|
|
|
} else {
|
|
|
|
unsigned long rem = do_div(nsec, NSEC_PER_SEC);
|
|
|
|
|
|
|
|
tmp = ktime_set((long)nsec, rem);
|
|
|
|
}
|
|
|
|
|
|
|
|
return ktime_sub(kt, tmp);
|
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL_GPL(ktime_sub_ns);
|
2006-01-09 21:52:32 -07:00
|
|
|
# endif /* !CONFIG_KTIME_SCALAR */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Divide a ktime value by a nanosecond value
|
|
|
|
*/
|
timerfd: new timerfd API
This is the new timerfd API as it is implemented by the following patch:
int timerfd_create(int clockid, int flags);
int timerfd_settime(int ufd, int flags,
const struct itimerspec *utmr,
struct itimerspec *otmr);
int timerfd_gettime(int ufd, struct itimerspec *otmr);
The timerfd_create() API creates an un-programmed timerfd fd. The "clockid"
parameter can be either CLOCK_MONOTONIC or CLOCK_REALTIME.
The timerfd_settime() API give new settings by the timerfd fd, by optionally
retrieving the previous expiration time (in case the "otmr" parameter is not
NULL).
The time value specified in "utmr" is absolute, if the TFD_TIMER_ABSTIME bit
is set in the "flags" parameter. Otherwise it's a relative time.
The timerfd_gettime() API returns the next expiration time of the timer, or
{0, 0} if the timerfd has not been set yet.
Like the previous timerfd API implementation, read(2) and poll(2) are
supported (with the same interface). Here's a simple test program I used to
exercise the new timerfd APIs:
http://www.xmailserver.org/timerfd-test2.c
[akpm@linux-foundation.org: coding-style cleanups]
[akpm@linux-foundation.org: fix ia64 build]
[akpm@linux-foundation.org: fix m68k build]
[akpm@linux-foundation.org: fix mips build]
[akpm@linux-foundation.org: fix alpha, arm, blackfin, cris, m68k, s390, sparc and sparc64 builds]
[heiko.carstens@de.ibm.com: fix s390]
[akpm@linux-foundation.org: fix powerpc build]
[akpm@linux-foundation.org: fix sparc64 more]
Signed-off-by: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Davide Libenzi <davidel@xmailserver.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-04 23:27:26 -07:00
|
|
|
u64 ktime_divns(const ktime_t kt, s64 div)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2008-05-22 16:25:11 -06:00
|
|
|
u64 dclc;
|
2006-01-09 21:52:32 -07:00
|
|
|
int sft = 0;
|
|
|
|
|
2008-05-22 16:25:11 -06:00
|
|
|
dclc = ktime_to_ns(kt);
|
2006-01-09 21:52:32 -07:00
|
|
|
/* Make sure the divisor is less than 2^32: */
|
|
|
|
while (div >> 32) {
|
|
|
|
sft++;
|
|
|
|
div >>= 1;
|
|
|
|
}
|
|
|
|
dclc >>= sft;
|
|
|
|
do_div(dclc, (unsigned long) div);
|
|
|
|
|
timerfd: new timerfd API
This is the new timerfd API as it is implemented by the following patch:
int timerfd_create(int clockid, int flags);
int timerfd_settime(int ufd, int flags,
const struct itimerspec *utmr,
struct itimerspec *otmr);
int timerfd_gettime(int ufd, struct itimerspec *otmr);
The timerfd_create() API creates an un-programmed timerfd fd. The "clockid"
parameter can be either CLOCK_MONOTONIC or CLOCK_REALTIME.
The timerfd_settime() API give new settings by the timerfd fd, by optionally
retrieving the previous expiration time (in case the "otmr" parameter is not
NULL).
The time value specified in "utmr" is absolute, if the TFD_TIMER_ABSTIME bit
is set in the "flags" parameter. Otherwise it's a relative time.
The timerfd_gettime() API returns the next expiration time of the timer, or
{0, 0} if the timerfd has not been set yet.
Like the previous timerfd API implementation, read(2) and poll(2) are
supported (with the same interface). Here's a simple test program I used to
exercise the new timerfd APIs:
http://www.xmailserver.org/timerfd-test2.c
[akpm@linux-foundation.org: coding-style cleanups]
[akpm@linux-foundation.org: fix ia64 build]
[akpm@linux-foundation.org: fix m68k build]
[akpm@linux-foundation.org: fix mips build]
[akpm@linux-foundation.org: fix alpha, arm, blackfin, cris, m68k, s390, sparc and sparc64 builds]
[heiko.carstens@de.ibm.com: fix s390]
[akpm@linux-foundation.org: fix powerpc build]
[akpm@linux-foundation.org: fix sparc64 more]
Signed-off-by: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Davide Libenzi <davidel@xmailserver.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-04 23:27:26 -07:00
|
|
|
return dclc;
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
|
|
|
#endif /* BITS_PER_LONG >= 64 */
|
|
|
|
|
2008-02-13 01:20:43 -07:00
|
|
|
/*
|
|
|
|
* Add two ktime values and do a safety check for overflow:
|
|
|
|
*/
|
|
|
|
ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
|
|
|
|
{
|
|
|
|
ktime_t res = ktime_add(lhs, rhs);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We use KTIME_SEC_MAX here, the maximum timeout which we can
|
|
|
|
* return to user space in a timespec:
|
|
|
|
*/
|
|
|
|
if (res.tv64 < 0 || res.tv64 < lhs.tv64 || res.tv64 < rhs.tv64)
|
|
|
|
res = ktime_set(KTIME_SEC_MAX, 0);
|
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2008-04-30 01:55:04 -06:00
|
|
|
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
|
|
|
|
|
|
|
|
static struct debug_obj_descr hrtimer_debug_descr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fixup_init is called when:
|
|
|
|
* - an active object is initialized
|
|
|
|
*/
|
|
|
|
static int hrtimer_fixup_init(void *addr, enum debug_obj_state state)
|
|
|
|
{
|
|
|
|
struct hrtimer *timer = addr;
|
|
|
|
|
|
|
|
switch (state) {
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
hrtimer_cancel(timer);
|
|
|
|
debug_object_init(timer, &hrtimer_debug_descr);
|
|
|
|
return 1;
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fixup_activate is called when:
|
|
|
|
* - an active object is activated
|
|
|
|
* - an unknown object is activated (might be a statically initialized object)
|
|
|
|
*/
|
|
|
|
static int hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
|
|
|
|
{
|
|
|
|
switch (state) {
|
|
|
|
|
|
|
|
case ODEBUG_STATE_NOTAVAILABLE:
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
WARN_ON(1);
|
|
|
|
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fixup_free is called when:
|
|
|
|
* - an active object is freed
|
|
|
|
*/
|
|
|
|
static int hrtimer_fixup_free(void *addr, enum debug_obj_state state)
|
|
|
|
{
|
|
|
|
struct hrtimer *timer = addr;
|
|
|
|
|
|
|
|
switch (state) {
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
hrtimer_cancel(timer);
|
|
|
|
debug_object_free(timer, &hrtimer_debug_descr);
|
|
|
|
return 1;
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct debug_obj_descr hrtimer_debug_descr = {
|
|
|
|
.name = "hrtimer",
|
|
|
|
.fixup_init = hrtimer_fixup_init,
|
|
|
|
.fixup_activate = hrtimer_fixup_activate,
|
|
|
|
.fixup_free = hrtimer_fixup_free,
|
|
|
|
};
|
|
|
|
|
|
|
|
static inline void debug_hrtimer_init(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_init(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_hrtimer_activate(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_activate(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_deactivate(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_hrtimer_free(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_free(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode);
|
|
|
|
|
|
|
|
void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
debug_object_init_on_stack(timer, &hrtimer_debug_descr);
|
|
|
|
__hrtimer_init(timer, clock_id, mode);
|
|
|
|
}
|
|
|
|
|
|
|
|
void destroy_hrtimer_on_stack(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_free(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
static inline void debug_hrtimer_init(struct hrtimer *timer) { }
|
|
|
|
static inline void debug_hrtimer_activate(struct hrtimer *timer) { }
|
|
|
|
static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
|
|
|
|
#endif
|
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
/* High resolution timer related functions */
|
|
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
|
|
|
|
|
|
|
/*
|
|
|
|
* High resolution timer enabled ?
|
|
|
|
*/
|
|
|
|
static int hrtimer_hres_enabled __read_mostly = 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Enable / Disable high resolution mode
|
|
|
|
*/
|
|
|
|
static int __init setup_hrtimer_hres(char *str)
|
|
|
|
{
|
|
|
|
if (!strcmp(str, "off"))
|
|
|
|
hrtimer_hres_enabled = 0;
|
|
|
|
else if (!strcmp(str, "on"))
|
|
|
|
hrtimer_hres_enabled = 1;
|
|
|
|
else
|
|
|
|
return 0;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
__setup("highres=", setup_hrtimer_hres);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* hrtimer_high_res_enabled - query, if the highres mode is enabled
|
|
|
|
*/
|
|
|
|
static inline int hrtimer_is_hres_enabled(void)
|
|
|
|
{
|
|
|
|
return hrtimer_hres_enabled;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Is the high resolution mode active ?
|
|
|
|
*/
|
|
|
|
static inline int hrtimer_hres_active(void)
|
|
|
|
{
|
|
|
|
return __get_cpu_var(hrtimer_bases).hres_active;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Reprogram the event source with checking both queues for the
|
|
|
|
* next event
|
|
|
|
* Called with interrupts disabled and base->lock held
|
|
|
|
*/
|
|
|
|
static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct hrtimer_clock_base *base = cpu_base->clock_base;
|
|
|
|
ktime_t expires;
|
|
|
|
|
|
|
|
cpu_base->expires_next.tv64 = KTIME_MAX;
|
|
|
|
|
|
|
|
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) {
|
|
|
|
struct hrtimer *timer;
|
|
|
|
|
|
|
|
if (!base->first)
|
|
|
|
continue;
|
|
|
|
timer = rb_entry(base->first, struct hrtimer, node);
|
2008-09-01 16:02:30 -06:00
|
|
|
expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
|
2007-02-16 02:28:11 -07:00
|
|
|
if (expires.tv64 < cpu_base->expires_next.tv64)
|
|
|
|
cpu_base->expires_next = expires;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (cpu_base->expires_next.tv64 != KTIME_MAX)
|
|
|
|
tick_program_event(cpu_base->expires_next, 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Shared reprogramming for clock_realtime and clock_monotonic
|
|
|
|
*
|
|
|
|
* When a timer is enqueued and expires earlier than the already enqueued
|
|
|
|
* timers, we have to check, whether it expires earlier than the timer for
|
|
|
|
* which the clock event device was armed.
|
|
|
|
*
|
|
|
|
* Called with interrupts disabled and base->cpu_base.lock held
|
|
|
|
*/
|
|
|
|
static int hrtimer_reprogram(struct hrtimer *timer,
|
|
|
|
struct hrtimer_clock_base *base)
|
|
|
|
{
|
|
|
|
ktime_t *expires_next = &__get_cpu_var(hrtimer_bases).expires_next;
|
2008-09-01 16:02:30 -06:00
|
|
|
ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
|
2007-02-16 02:28:11 -07:00
|
|
|
int res;
|
|
|
|
|
2008-09-01 16:02:30 -06:00
|
|
|
WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
|
2008-02-13 16:58:36 -07:00
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
/*
|
|
|
|
* When the callback is running, we do not reprogram the clock event
|
|
|
|
* device. The timer callback is either running on a different CPU or
|
2007-10-19 15:10:43 -06:00
|
|
|
* the callback is executed in the hrtimer_interrupt context. The
|
2007-02-16 02:28:11 -07:00
|
|
|
* reprogramming is handled either by the softirq, which called the
|
|
|
|
* callback or at the end of the hrtimer_interrupt.
|
|
|
|
*/
|
|
|
|
if (hrtimer_callback_running(timer))
|
|
|
|
return 0;
|
|
|
|
|
2008-02-13 16:58:36 -07:00
|
|
|
/*
|
|
|
|
* CLOCK_REALTIME timer might be requested with an absolute
|
|
|
|
* expiry time which is less than base->offset. Nothing wrong
|
|
|
|
* about that, just avoid to call into the tick code, which
|
|
|
|
* has now objections against negative expiry values.
|
|
|
|
*/
|
|
|
|
if (expires.tv64 < 0)
|
|
|
|
return -ETIME;
|
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
if (expires.tv64 >= expires_next->tv64)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Clockevents returns -ETIME, when the event was in the past.
|
|
|
|
*/
|
|
|
|
res = tick_program_event(expires, 0);
|
|
|
|
if (!IS_ERR_VALUE(res))
|
|
|
|
*expires_next = expires;
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Retrigger next event is called after clock was set
|
|
|
|
*
|
|
|
|
* Called with interrupts disabled via on_each_cpu()
|
|
|
|
*/
|
|
|
|
static void retrigger_next_event(void *arg)
|
|
|
|
{
|
|
|
|
struct hrtimer_cpu_base *base;
|
|
|
|
struct timespec realtime_offset;
|
|
|
|
unsigned long seq;
|
|
|
|
|
|
|
|
if (!hrtimer_hres_active())
|
|
|
|
return;
|
|
|
|
|
|
|
|
do {
|
|
|
|
seq = read_seqbegin(&xtime_lock);
|
|
|
|
set_normalized_timespec(&realtime_offset,
|
|
|
|
-wall_to_monotonic.tv_sec,
|
|
|
|
-wall_to_monotonic.tv_nsec);
|
|
|
|
} while (read_seqretry(&xtime_lock, seq));
|
|
|
|
|
|
|
|
base = &__get_cpu_var(hrtimer_bases);
|
|
|
|
|
|
|
|
/* Adjust CLOCK_REALTIME offset */
|
|
|
|
spin_lock(&base->lock);
|
|
|
|
base->clock_base[CLOCK_REALTIME].offset =
|
|
|
|
timespec_to_ktime(realtime_offset);
|
|
|
|
|
|
|
|
hrtimer_force_reprogram(base);
|
|
|
|
spin_unlock(&base->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Clock realtime was set
|
|
|
|
*
|
|
|
|
* Change the offset of the realtime clock vs. the monotonic
|
|
|
|
* clock.
|
|
|
|
*
|
|
|
|
* We might have to reprogram the high resolution timer interrupt. On
|
|
|
|
* SMP we call the architecture specific code to retrigger _all_ high
|
|
|
|
* resolution timer interrupts. On UP we just disable interrupts and
|
|
|
|
* call the high resolution interrupt code.
|
|
|
|
*/
|
|
|
|
void clock_was_set(void)
|
|
|
|
{
|
|
|
|
/* Retrigger the CPU local events everywhere */
|
2008-05-09 01:39:44 -06:00
|
|
|
on_each_cpu(retrigger_next_event, NULL, 1);
|
2007-02-16 02:28:11 -07:00
|
|
|
}
|
|
|
|
|
2007-04-07 04:05:00 -06:00
|
|
|
/*
|
|
|
|
* During resume we might have to reprogram the high resolution timer
|
|
|
|
* interrupt (on the local CPU):
|
|
|
|
*/
|
|
|
|
void hres_timers_resume(void)
|
|
|
|
{
|
|
|
|
/* Retrigger the CPU local events: */
|
|
|
|
retrigger_next_event(NULL);
|
|
|
|
}
|
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
/*
|
|
|
|
* Initialize the high resolution related parts of cpu_base
|
|
|
|
*/
|
|
|
|
static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base)
|
|
|
|
{
|
|
|
|
base->expires_next.tv64 = KTIME_MAX;
|
|
|
|
base->hres_active = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize the high resolution related parts of a hrtimer
|
|
|
|
*/
|
|
|
|
static inline void hrtimer_init_timer_hres(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2008-11-25 04:43:51 -07:00
|
|
|
static void __run_hrtimer(struct hrtimer *timer);
|
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
/*
|
|
|
|
* When High resolution timers are active, try to reprogram. Note, that in case
|
|
|
|
* the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry
|
|
|
|
* check happens. The timer gets enqueued into the rbtree. The reprogramming
|
|
|
|
* and expiry check is done in the hrtimer_interrupt or in the softirq.
|
|
|
|
*/
|
|
|
|
static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
|
|
|
|
struct hrtimer_clock_base *base)
|
|
|
|
{
|
|
|
|
if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) {
|
2008-11-25 04:43:51 -07:00
|
|
|
/*
|
|
|
|
* XXX: recursion check?
|
|
|
|
* hrtimer_forward() should round up with timer granularity
|
|
|
|
* so that we never get into inf recursion here,
|
|
|
|
* it doesn't do that though
|
|
|
|
*/
|
|
|
|
__run_hrtimer(timer);
|
|
|
|
return 1;
|
2007-02-16 02:28:11 -07:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Switch to high resolution mode
|
|
|
|
*/
|
2007-03-06 02:42:08 -07:00
|
|
|
static int hrtimer_switch_to_hres(void)
|
2007-02-16 02:28:11 -07:00
|
|
|
{
|
2007-07-21 05:37:36 -06:00
|
|
|
int cpu = smp_processor_id();
|
|
|
|
struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu);
|
2007-02-16 02:28:11 -07:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
if (base->hres_active)
|
2007-03-06 02:42:08 -07:00
|
|
|
return 1;
|
2007-02-16 02:28:11 -07:00
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
|
|
|
if (tick_init_highres()) {
|
|
|
|
local_irq_restore(flags);
|
2007-07-21 05:37:36 -06:00
|
|
|
printk(KERN_WARNING "Could not switch to high resolution "
|
|
|
|
"mode on CPU %d\n", cpu);
|
2007-03-06 02:42:08 -07:00
|
|
|
return 0;
|
2007-02-16 02:28:11 -07:00
|
|
|
}
|
|
|
|
base->hres_active = 1;
|
|
|
|
base->clock_base[CLOCK_REALTIME].resolution = KTIME_HIGH_RES;
|
|
|
|
base->clock_base[CLOCK_MONOTONIC].resolution = KTIME_HIGH_RES;
|
|
|
|
|
|
|
|
tick_setup_sched_timer();
|
|
|
|
|
|
|
|
/* "Retrigger" the interrupt to get things going */
|
|
|
|
retrigger_next_event(NULL);
|
|
|
|
local_irq_restore(flags);
|
2007-10-28 23:35:29 -06:00
|
|
|
printk(KERN_DEBUG "Switched to high resolution mode on CPU %d\n",
|
2007-02-16 02:28:11 -07:00
|
|
|
smp_processor_id());
|
2007-03-06 02:42:08 -07:00
|
|
|
return 1;
|
2007-02-16 02:28:11 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
static inline int hrtimer_hres_active(void) { return 0; }
|
|
|
|
static inline int hrtimer_is_hres_enabled(void) { return 0; }
|
2007-03-06 02:42:08 -07:00
|
|
|
static inline int hrtimer_switch_to_hres(void) { return 0; }
|
2007-02-16 02:28:11 -07:00
|
|
|
static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base) { }
|
|
|
|
static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
|
|
|
|
struct hrtimer_clock_base *base)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }
|
|
|
|
static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { }
|
2008-01-25 13:08:31 -07:00
|
|
|
static inline int hrtimer_reprogram(struct hrtimer *timer,
|
|
|
|
struct hrtimer_clock_base *base)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
2007-02-16 02:28:11 -07:00
|
|
|
|
|
|
|
#endif /* CONFIG_HIGH_RES_TIMERS */
|
|
|
|
|
[PATCH] Add debugging feature /proc/timer_stat
Add /proc/timer_stats support: debugging feature to profile timer expiration.
Both the starting site, process/PID and the expiration function is captured.
This allows the quick identification of timer event sources in a system.
Sample output:
# echo 1 > /proc/timer_stats
# cat /proc/timer_stats
Timer Stats Version: v0.1
Sample period: 4.010 s
24, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
11, 0 swapper sk_reset_timer (tcp_delack_timer)
6, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
17, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
4, 2050 pcscd do_nanosleep (hrtimer_wakeup)
5, 4179 sshd sk_reset_timer (tcp_write_timer)
4, 2248 yum-updatesd schedule_timeout (process_timeout)
18, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
3, 0 swapper sk_reset_timer (tcp_delack_timer)
1, 1 swapper neigh_table_init_no_netlink (neigh_periodic_timer)
2, 1 swapper e1000_up (e1000_watchdog)
1, 1 init schedule_timeout (process_timeout)
100 total events, 25.24 events/sec
[ cleanups and hrtimers support from Thomas Gleixner <tglx@linutronix.de> ]
[bunk@stusta.de: nr_entries can become static]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-16 02:28:13 -07:00
|
|
|
#ifdef CONFIG_TIMER_STATS
|
|
|
|
void __timer_stats_hrtimer_set_start_info(struct hrtimer *timer, void *addr)
|
|
|
|
{
|
|
|
|
if (timer->start_site)
|
|
|
|
return;
|
|
|
|
|
|
|
|
timer->start_site = addr;
|
|
|
|
memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);
|
|
|
|
timer->start_pid = current->pid;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2006-01-09 21:52:32 -07:00
|
|
|
/*
|
2007-10-19 17:56:53 -06:00
|
|
|
* Counterpart to lock_hrtimer_base above:
|
2006-01-09 21:52:32 -07:00
|
|
|
*/
|
|
|
|
static inline
|
|
|
|
void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
|
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_forward - forward the timer expiry
|
|
|
|
* @timer: hrtimer to forward
|
2006-03-26 02:38:06 -07:00
|
|
|
* @now: forward past this time
|
2006-01-09 21:52:32 -07:00
|
|
|
* @interval: the interval to forward
|
|
|
|
*
|
|
|
|
* Forward the timer expiry so it will expire in the future.
|
2006-01-16 15:58:55 -07:00
|
|
|
* Returns the number of overruns.
|
2006-01-09 21:52:32 -07:00
|
|
|
*/
|
timerfd: new timerfd API
This is the new timerfd API as it is implemented by the following patch:
int timerfd_create(int clockid, int flags);
int timerfd_settime(int ufd, int flags,
const struct itimerspec *utmr,
struct itimerspec *otmr);
int timerfd_gettime(int ufd, struct itimerspec *otmr);
The timerfd_create() API creates an un-programmed timerfd fd. The "clockid"
parameter can be either CLOCK_MONOTONIC or CLOCK_REALTIME.
The timerfd_settime() API give new settings by the timerfd fd, by optionally
retrieving the previous expiration time (in case the "otmr" parameter is not
NULL).
The time value specified in "utmr" is absolute, if the TFD_TIMER_ABSTIME bit
is set in the "flags" parameter. Otherwise it's a relative time.
The timerfd_gettime() API returns the next expiration time of the timer, or
{0, 0} if the timerfd has not been set yet.
Like the previous timerfd API implementation, read(2) and poll(2) are
supported (with the same interface). Here's a simple test program I used to
exercise the new timerfd APIs:
http://www.xmailserver.org/timerfd-test2.c
[akpm@linux-foundation.org: coding-style cleanups]
[akpm@linux-foundation.org: fix ia64 build]
[akpm@linux-foundation.org: fix m68k build]
[akpm@linux-foundation.org: fix mips build]
[akpm@linux-foundation.org: fix alpha, arm, blackfin, cris, m68k, s390, sparc and sparc64 builds]
[heiko.carstens@de.ibm.com: fix s390]
[akpm@linux-foundation.org: fix powerpc build]
[akpm@linux-foundation.org: fix sparc64 more]
Signed-off-by: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Davide Libenzi <davidel@xmailserver.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-04 23:27:26 -07:00
|
|
|
u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
timerfd: new timerfd API
This is the new timerfd API as it is implemented by the following patch:
int timerfd_create(int clockid, int flags);
int timerfd_settime(int ufd, int flags,
const struct itimerspec *utmr,
struct itimerspec *otmr);
int timerfd_gettime(int ufd, struct itimerspec *otmr);
The timerfd_create() API creates an un-programmed timerfd fd. The "clockid"
parameter can be either CLOCK_MONOTONIC or CLOCK_REALTIME.
The timerfd_settime() API give new settings by the timerfd fd, by optionally
retrieving the previous expiration time (in case the "otmr" parameter is not
NULL).
The time value specified in "utmr" is absolute, if the TFD_TIMER_ABSTIME bit
is set in the "flags" parameter. Otherwise it's a relative time.
The timerfd_gettime() API returns the next expiration time of the timer, or
{0, 0} if the timerfd has not been set yet.
Like the previous timerfd API implementation, read(2) and poll(2) are
supported (with the same interface). Here's a simple test program I used to
exercise the new timerfd APIs:
http://www.xmailserver.org/timerfd-test2.c
[akpm@linux-foundation.org: coding-style cleanups]
[akpm@linux-foundation.org: fix ia64 build]
[akpm@linux-foundation.org: fix m68k build]
[akpm@linux-foundation.org: fix mips build]
[akpm@linux-foundation.org: fix alpha, arm, blackfin, cris, m68k, s390, sparc and sparc64 builds]
[heiko.carstens@de.ibm.com: fix s390]
[akpm@linux-foundation.org: fix powerpc build]
[akpm@linux-foundation.org: fix sparc64 more]
Signed-off-by: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Davide Libenzi <davidel@xmailserver.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-04 23:27:26 -07:00
|
|
|
u64 orun = 1;
|
2006-03-26 02:38:06 -07:00
|
|
|
ktime_t delta;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2008-09-01 16:02:30 -06:00
|
|
|
delta = ktime_sub(now, hrtimer_get_expires(timer));
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
if (delta.tv64 < 0)
|
|
|
|
return 0;
|
|
|
|
|
2006-01-12 03:47:34 -07:00
|
|
|
if (interval.tv64 < timer->base->resolution.tv64)
|
|
|
|
interval.tv64 = timer->base->resolution.tv64;
|
|
|
|
|
2006-01-09 21:52:32 -07:00
|
|
|
if (unlikely(delta.tv64 >= interval.tv64)) {
|
2006-03-26 02:38:11 -07:00
|
|
|
s64 incr = ktime_to_ns(interval);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
orun = ktime_divns(delta, incr);
|
2008-09-01 16:02:30 -06:00
|
|
|
hrtimer_add_expires_ns(timer, incr * orun);
|
|
|
|
if (hrtimer_get_expires_tv64(timer) > now.tv64)
|
2006-01-09 21:52:32 -07:00
|
|
|
return orun;
|
|
|
|
/*
|
|
|
|
* This (and the ktime_add() below) is the
|
|
|
|
* correction for exact:
|
|
|
|
*/
|
|
|
|
orun++;
|
|
|
|
}
|
2008-09-01 16:02:30 -06:00
|
|
|
hrtimer_add_expires(timer, interval);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
return orun;
|
|
|
|
}
|
2007-05-08 01:31:58 -06:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_forward);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* enqueue_hrtimer - internal function to (re)start a timer
|
|
|
|
*
|
|
|
|
* The timer is inserted in expiry order. Insertion into the
|
|
|
|
* red black tree is O(log(n)). Must hold the base lock.
|
|
|
|
*/
|
2007-02-16 02:27:50 -07:00
|
|
|
static void enqueue_hrtimer(struct hrtimer *timer,
|
2007-02-16 02:28:11 -07:00
|
|
|
struct hrtimer_clock_base *base, int reprogram)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
|
|
|
struct rb_node **link = &base->active.rb_node;
|
|
|
|
struct rb_node *parent = NULL;
|
|
|
|
struct hrtimer *entry;
|
2007-07-21 05:37:36 -06:00
|
|
|
int leftmost = 1;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2008-04-30 01:55:04 -06:00
|
|
|
debug_hrtimer_activate(timer);
|
|
|
|
|
2006-01-09 21:52:32 -07:00
|
|
|
/*
|
|
|
|
* Find the right place in the rbtree:
|
|
|
|
*/
|
|
|
|
while (*link) {
|
|
|
|
parent = *link;
|
|
|
|
entry = rb_entry(parent, struct hrtimer, node);
|
|
|
|
/*
|
|
|
|
* We dont care about collisions. Nodes with
|
|
|
|
* the same expiry time stay together.
|
|
|
|
*/
|
2008-09-01 16:02:30 -06:00
|
|
|
if (hrtimer_get_expires_tv64(timer) <
|
|
|
|
hrtimer_get_expires_tv64(entry)) {
|
2006-01-09 21:52:32 -07:00
|
|
|
link = &(*link)->rb_left;
|
2007-07-21 05:37:36 -06:00
|
|
|
} else {
|
2006-01-09 21:52:32 -07:00
|
|
|
link = &(*link)->rb_right;
|
2007-07-21 05:37:36 -06:00
|
|
|
leftmost = 0;
|
|
|
|
}
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2006-01-12 03:25:54 -07:00
|
|
|
* Insert the timer to the rbtree and check whether it
|
|
|
|
* replaces the first pending timer
|
2006-01-09 21:52:32 -07:00
|
|
|
*/
|
2007-07-21 05:37:36 -06:00
|
|
|
if (leftmost) {
|
2007-02-16 02:28:11 -07:00
|
|
|
/*
|
|
|
|
* Reprogram the clock event device. When the timer is already
|
|
|
|
* expired hrtimer_enqueue_reprogram has either called the
|
|
|
|
* callback or added it to the pending list and raised the
|
|
|
|
* softirq.
|
|
|
|
*
|
|
|
|
* This is a NOP for !HIGHRES
|
|
|
|
*/
|
|
|
|
if (reprogram && hrtimer_enqueue_reprogram(timer, base))
|
|
|
|
return;
|
|
|
|
|
|
|
|
base->first = &timer->node;
|
|
|
|
}
|
|
|
|
|
2006-01-09 21:52:32 -07:00
|
|
|
rb_link_node(&timer->node, parent, link);
|
|
|
|
rb_insert_color(&timer->node, &base->active);
|
2007-02-16 02:27:51 -07:00
|
|
|
/*
|
|
|
|
* HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the
|
|
|
|
* state of a possibly running callback.
|
|
|
|
*/
|
|
|
|
timer->state |= HRTIMER_STATE_ENQUEUED;
|
2006-01-12 03:25:54 -07:00
|
|
|
}
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* __remove_hrtimer - internal function to remove a timer
|
|
|
|
*
|
|
|
|
* Caller must hold the base lock.
|
2007-02-16 02:28:11 -07:00
|
|
|
*
|
|
|
|
* High resolution timer mode reprograms the clock event device when the
|
|
|
|
* timer is the one which expires next. The caller can disable this by setting
|
|
|
|
* reprogram to zero. This is useful, when the context does a reprogramming
|
|
|
|
* anyway (e.g. timer interrupt)
|
2006-01-09 21:52:32 -07:00
|
|
|
*/
|
2007-02-16 02:27:50 -07:00
|
|
|
static void __remove_hrtimer(struct hrtimer *timer,
|
2007-02-16 02:27:51 -07:00
|
|
|
struct hrtimer_clock_base *base,
|
2007-02-16 02:28:11 -07:00
|
|
|
unsigned long newstate, int reprogram)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2008-11-25 04:43:51 -07:00
|
|
|
if (timer->state & HRTIMER_STATE_ENQUEUED) {
|
2007-02-16 02:28:11 -07:00
|
|
|
/*
|
|
|
|
* Remove the timer from the rbtree and replace the
|
|
|
|
* first entry pointer if necessary.
|
|
|
|
*/
|
|
|
|
if (base->first == &timer->node) {
|
|
|
|
base->first = rb_next(&timer->node);
|
|
|
|
/* Reprogram the clock event device. if enabled */
|
|
|
|
if (reprogram && hrtimer_hres_active())
|
|
|
|
hrtimer_force_reprogram(base->cpu_base);
|
|
|
|
}
|
|
|
|
rb_erase(&timer->node, &base->active);
|
|
|
|
}
|
2007-02-16 02:27:51 -07:00
|
|
|
timer->state = newstate;
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* remove hrtimer, called with base lock held
|
|
|
|
*/
|
|
|
|
static inline int
|
2007-02-16 02:27:50 -07:00
|
|
|
remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2007-02-16 02:27:51 -07:00
|
|
|
if (hrtimer_is_queued(timer)) {
|
2007-02-16 02:28:11 -07:00
|
|
|
int reprogram;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove the timer and force reprogramming when high
|
|
|
|
* resolution mode is active and the timer is on the current
|
|
|
|
* CPU. If we remove a timer on another CPU, reprogramming is
|
|
|
|
* skipped. The interrupt event on this CPU is fired and
|
|
|
|
* reprogramming happens in the interrupt handler. This is a
|
|
|
|
* rare case and less expensive than a smp call.
|
|
|
|
*/
|
2008-04-30 01:55:04 -06:00
|
|
|
debug_hrtimer_deactivate(timer);
|
[PATCH] Add debugging feature /proc/timer_stat
Add /proc/timer_stats support: debugging feature to profile timer expiration.
Both the starting site, process/PID and the expiration function is captured.
This allows the quick identification of timer event sources in a system.
Sample output:
# echo 1 > /proc/timer_stats
# cat /proc/timer_stats
Timer Stats Version: v0.1
Sample period: 4.010 s
24, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
11, 0 swapper sk_reset_timer (tcp_delack_timer)
6, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
17, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
4, 2050 pcscd do_nanosleep (hrtimer_wakeup)
5, 4179 sshd sk_reset_timer (tcp_write_timer)
4, 2248 yum-updatesd schedule_timeout (process_timeout)
18, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
3, 0 swapper sk_reset_timer (tcp_delack_timer)
1, 1 swapper neigh_table_init_no_netlink (neigh_periodic_timer)
2, 1 swapper e1000_up (e1000_watchdog)
1, 1 init schedule_timeout (process_timeout)
100 total events, 25.24 events/sec
[ cleanups and hrtimers support from Thomas Gleixner <tglx@linutronix.de> ]
[bunk@stusta.de: nr_entries can become static]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-16 02:28:13 -07:00
|
|
|
timer_stats_hrtimer_clear_start_info(timer);
|
2007-02-16 02:28:11 -07:00
|
|
|
reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases);
|
|
|
|
__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE,
|
|
|
|
reprogram);
|
2006-01-09 21:52:32 -07:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2008-10-20 05:33:36 -06:00
|
|
|
* hrtimer_start_range_ns - (re)start an hrtimer on the current CPU
|
2006-01-09 21:52:32 -07:00
|
|
|
* @timer: the timer to be added
|
|
|
|
* @tim: expiry time
|
2008-09-07 11:47:46 -06:00
|
|
|
* @delta_ns: "slack" range for the timer
|
2006-01-09 21:52:32 -07:00
|
|
|
* @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* 0 on success
|
|
|
|
* 1 when the timer was active
|
|
|
|
*/
|
|
|
|
int
|
2008-09-07 11:47:46 -06:00
|
|
|
hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns,
|
|
|
|
const enum hrtimer_mode mode)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_clock_base *base, *new_base;
|
2006-01-09 21:52:32 -07:00
|
|
|
unsigned long flags;
|
2008-11-25 04:43:51 -07:00
|
|
|
int ret;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
base = lock_hrtimer_base(timer, &flags);
|
|
|
|
|
|
|
|
/* Remove an active timer from the queue: */
|
|
|
|
ret = remove_hrtimer(timer, base);
|
|
|
|
|
|
|
|
/* Switch the timer base, if necessary: */
|
|
|
|
new_base = switch_hrtimer_base(timer, base);
|
|
|
|
|
2007-02-16 02:27:49 -07:00
|
|
|
if (mode == HRTIMER_MODE_REL) {
|
2008-02-13 01:20:43 -07:00
|
|
|
tim = ktime_add_safe(tim, new_base->get_time());
|
2006-02-14 14:53:15 -07:00
|
|
|
/*
|
|
|
|
* CONFIG_TIME_LOW_RES is a temporary way for architectures
|
|
|
|
* to signal that they simply return xtime in
|
|
|
|
* do_gettimeoffset(). In this case we want to round up by
|
|
|
|
* resolution when starting a relative timer, to avoid short
|
|
|
|
* timeouts. This will go away with the GTOD framework.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_TIME_LOW_RES
|
2008-02-13 01:20:43 -07:00
|
|
|
tim = ktime_add_safe(tim, base->resolution);
|
2006-02-14 14:53:15 -07:00
|
|
|
#endif
|
|
|
|
}
|
2008-04-30 01:55:04 -06:00
|
|
|
|
2008-09-07 11:47:46 -06:00
|
|
|
hrtimer_set_expires_range_ns(timer, tim, delta_ns);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
[PATCH] Add debugging feature /proc/timer_stat
Add /proc/timer_stats support: debugging feature to profile timer expiration.
Both the starting site, process/PID and the expiration function is captured.
This allows the quick identification of timer event sources in a system.
Sample output:
# echo 1 > /proc/timer_stats
# cat /proc/timer_stats
Timer Stats Version: v0.1
Sample period: 4.010 s
24, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
11, 0 swapper sk_reset_timer (tcp_delack_timer)
6, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
17, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
4, 2050 pcscd do_nanosleep (hrtimer_wakeup)
5, 4179 sshd sk_reset_timer (tcp_write_timer)
4, 2248 yum-updatesd schedule_timeout (process_timeout)
18, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
3, 0 swapper sk_reset_timer (tcp_delack_timer)
1, 1 swapper neigh_table_init_no_netlink (neigh_periodic_timer)
2, 1 swapper e1000_up (e1000_watchdog)
1, 1 init schedule_timeout (process_timeout)
100 total events, 25.24 events/sec
[ cleanups and hrtimers support from Thomas Gleixner <tglx@linutronix.de> ]
[bunk@stusta.de: nr_entries can become static]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-16 02:28:13 -07:00
|
|
|
timer_stats_hrtimer_set_start_info(timer);
|
|
|
|
|
2007-03-28 05:17:18 -06:00
|
|
|
/*
|
|
|
|
* Only allow reprogramming if the new base is on this CPU.
|
|
|
|
* (it might still be on another CPU if the timer was pending)
|
|
|
|
*/
|
|
|
|
enqueue_hrtimer(timer, new_base,
|
|
|
|
new_base->cpu_base == &__get_cpu_var(hrtimer_bases));
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
unlock_hrtimer_base(timer, &flags);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
2008-09-07 11:47:46 -06:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
|
|
|
|
|
|
|
|
/**
|
2008-10-20 05:33:36 -06:00
|
|
|
* hrtimer_start - (re)start an hrtimer on the current CPU
|
2008-09-07 11:47:46 -06:00
|
|
|
* @timer: the timer to be added
|
|
|
|
* @tim: expiry time
|
|
|
|
* @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* 0 on success
|
|
|
|
* 1 when the timer was active
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
return hrtimer_start_range_ns(timer, tim, 0, mode);
|
|
|
|
}
|
2006-05-30 22:26:09 -06:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_start);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2008-09-07 11:47:46 -06:00
|
|
|
|
2006-01-09 21:52:32 -07:00
|
|
|
/**
|
|
|
|
* hrtimer_try_to_cancel - try to deactivate a timer
|
|
|
|
* @timer: hrtimer to stop
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* 0 when the timer was not active
|
|
|
|
* 1 when the timer was active
|
|
|
|
* -1 when the timer is currently excuting the callback function and
|
2006-06-25 06:49:15 -06:00
|
|
|
* cannot be stopped
|
2006-01-09 21:52:32 -07:00
|
|
|
*/
|
|
|
|
int hrtimer_try_to_cancel(struct hrtimer *timer)
|
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_clock_base *base;
|
2006-01-09 21:52:32 -07:00
|
|
|
unsigned long flags;
|
|
|
|
int ret = -1;
|
|
|
|
|
|
|
|
base = lock_hrtimer_base(timer, &flags);
|
|
|
|
|
2007-02-16 02:27:51 -07:00
|
|
|
if (!hrtimer_callback_running(timer))
|
2006-01-09 21:52:32 -07:00
|
|
|
ret = remove_hrtimer(timer, base);
|
|
|
|
|
|
|
|
unlock_hrtimer_base(timer, &flags);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
}
|
2006-05-30 22:26:09 -06:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_cancel - cancel a timer and wait for the handler to finish.
|
|
|
|
* @timer: the timer to be cancelled
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* 0 when the timer was not active
|
|
|
|
* 1 when the timer was active
|
|
|
|
*/
|
|
|
|
int hrtimer_cancel(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
for (;;) {
|
|
|
|
int ret = hrtimer_try_to_cancel(timer);
|
|
|
|
|
|
|
|
if (ret >= 0)
|
|
|
|
return ret;
|
2006-04-10 23:54:13 -06:00
|
|
|
cpu_relax();
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
|
|
|
}
|
2006-05-30 22:26:09 -06:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_cancel);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_get_remaining - get remaining time for the timer
|
|
|
|
* @timer: the timer to read
|
|
|
|
*/
|
|
|
|
ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
|
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_clock_base *base;
|
2006-01-09 21:52:32 -07:00
|
|
|
unsigned long flags;
|
|
|
|
ktime_t rem;
|
|
|
|
|
|
|
|
base = lock_hrtimer_base(timer, &flags);
|
2008-09-01 16:02:30 -06:00
|
|
|
rem = hrtimer_expires_remaining(timer);
|
2006-01-09 21:52:32 -07:00
|
|
|
unlock_hrtimer_base(timer, &flags);
|
|
|
|
|
|
|
|
return rem;
|
|
|
|
}
|
2006-05-30 22:26:09 -06:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_get_remaining);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2008-04-20 06:59:33 -06:00
|
|
|
#ifdef CONFIG_NO_HZ
|
2006-03-06 16:42:45 -07:00
|
|
|
/**
|
|
|
|
* hrtimer_get_next_event - get the time until next expiry event
|
|
|
|
*
|
|
|
|
* Returns the delta to the next expiry event or KTIME_MAX if no timer
|
|
|
|
* is pending.
|
|
|
|
*/
|
|
|
|
ktime_t hrtimer_get_next_event(void)
|
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
|
|
|
|
struct hrtimer_clock_base *base = cpu_base->clock_base;
|
2006-03-06 16:42:45 -07:00
|
|
|
ktime_t delta, mindelta = { .tv64 = KTIME_MAX };
|
|
|
|
unsigned long flags;
|
|
|
|
int i;
|
|
|
|
|
2007-02-16 02:27:50 -07:00
|
|
|
spin_lock_irqsave(&cpu_base->lock, flags);
|
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
if (!hrtimer_hres_active()) {
|
|
|
|
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) {
|
|
|
|
struct hrtimer *timer;
|
2006-03-06 16:42:45 -07:00
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
if (!base->first)
|
|
|
|
continue;
|
2007-02-16 02:27:50 -07:00
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
timer = rb_entry(base->first, struct hrtimer, node);
|
2008-09-01 16:02:30 -06:00
|
|
|
delta.tv64 = hrtimer_get_expires_tv64(timer);
|
2007-02-16 02:28:11 -07:00
|
|
|
delta = ktime_sub(delta, base->get_time());
|
|
|
|
if (delta.tv64 < mindelta.tv64)
|
|
|
|
mindelta.tv64 = delta.tv64;
|
|
|
|
}
|
2006-03-06 16:42:45 -07:00
|
|
|
}
|
2007-02-16 02:27:50 -07:00
|
|
|
|
|
|
|
spin_unlock_irqrestore(&cpu_base->lock, flags);
|
|
|
|
|
2006-03-06 16:42:45 -07:00
|
|
|
if (mindelta.tv64 < 0)
|
|
|
|
mindelta.tv64 = 0;
|
|
|
|
return mindelta;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2008-04-30 01:55:04 -06:00
|
|
|
static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_cpu_base *cpu_base;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2006-02-01 04:05:11 -07:00
|
|
|
memset(timer, 0, sizeof(struct hrtimer));
|
|
|
|
|
2007-02-16 02:27:50 -07:00
|
|
|
cpu_base = &__raw_get_cpu_var(hrtimer_bases);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2007-02-16 02:27:49 -07:00
|
|
|
if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS)
|
2006-02-01 04:05:11 -07:00
|
|
|
clock_id = CLOCK_MONOTONIC;
|
|
|
|
|
2007-02-16 02:27:50 -07:00
|
|
|
timer->base = &cpu_base->clock_base[clock_id];
|
2008-01-25 13:08:31 -07:00
|
|
|
INIT_LIST_HEAD(&timer->cb_entry);
|
2007-02-16 02:28:11 -07:00
|
|
|
hrtimer_init_timer_hres(timer);
|
[PATCH] Add debugging feature /proc/timer_stat
Add /proc/timer_stats support: debugging feature to profile timer expiration.
Both the starting site, process/PID and the expiration function is captured.
This allows the quick identification of timer event sources in a system.
Sample output:
# echo 1 > /proc/timer_stats
# cat /proc/timer_stats
Timer Stats Version: v0.1
Sample period: 4.010 s
24, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
11, 0 swapper sk_reset_timer (tcp_delack_timer)
6, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
17, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
4, 2050 pcscd do_nanosleep (hrtimer_wakeup)
5, 4179 sshd sk_reset_timer (tcp_write_timer)
4, 2248 yum-updatesd schedule_timeout (process_timeout)
18, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
3, 0 swapper sk_reset_timer (tcp_delack_timer)
1, 1 swapper neigh_table_init_no_netlink (neigh_periodic_timer)
2, 1 swapper e1000_up (e1000_watchdog)
1, 1 init schedule_timeout (process_timeout)
100 total events, 25.24 events/sec
[ cleanups and hrtimers support from Thomas Gleixner <tglx@linutronix.de> ]
[bunk@stusta.de: nr_entries can become static]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-16 02:28:13 -07:00
|
|
|
|
|
|
|
#ifdef CONFIG_TIMER_STATS
|
|
|
|
timer->start_site = NULL;
|
|
|
|
timer->start_pid = -1;
|
|
|
|
memset(timer->start_comm, 0, TASK_COMM_LEN);
|
|
|
|
#endif
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
2008-04-30 01:55:04 -06:00
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_init - initialize a timer to the given clock
|
|
|
|
* @timer: the timer to be initialized
|
|
|
|
* @clock_id: the clock to be used
|
|
|
|
* @mode: timer mode abs/rel
|
|
|
|
*/
|
|
|
|
void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
debug_hrtimer_init(timer);
|
|
|
|
__hrtimer_init(timer, clock_id, mode);
|
|
|
|
}
|
2006-05-30 22:26:09 -06:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_init);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_get_res - get the timer resolution for a clock
|
|
|
|
* @which_clock: which clock to query
|
|
|
|
* @tp: pointer to timespec variable to store the resolution
|
|
|
|
*
|
2007-02-10 02:45:59 -07:00
|
|
|
* Store the resolution of the clock selected by @which_clock in the
|
|
|
|
* variable pointed to by @tp.
|
2006-01-09 21:52:32 -07:00
|
|
|
*/
|
|
|
|
int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
|
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_cpu_base *cpu_base;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2007-02-16 02:27:50 -07:00
|
|
|
cpu_base = &__raw_get_cpu_var(hrtimer_bases);
|
|
|
|
*tp = ktime_to_timespec(cpu_base->clock_base[which_clock].resolution);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2006-05-30 22:26:09 -06:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_get_res);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2008-01-25 13:08:31 -07:00
|
|
|
static void __run_hrtimer(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
struct hrtimer_clock_base *base = timer->base;
|
|
|
|
struct hrtimer_cpu_base *cpu_base = base->cpu_base;
|
|
|
|
enum hrtimer_restart (*fn)(struct hrtimer *);
|
|
|
|
int restart;
|
|
|
|
|
2008-11-25 04:43:51 -07:00
|
|
|
WARN_ON(!irqs_disabled());
|
|
|
|
|
2008-04-30 01:55:04 -06:00
|
|
|
debug_hrtimer_deactivate(timer);
|
2008-01-25 13:08:31 -07:00
|
|
|
__remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);
|
|
|
|
timer_stats_account_hrtimer(timer);
|
|
|
|
fn = timer->function;
|
2008-11-25 04:43:51 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Because we run timers from hardirq context, there is no chance
|
|
|
|
* they get migrated to another cpu, therefore its safe to unlock
|
|
|
|
* the timer base.
|
|
|
|
*/
|
|
|
|
spin_unlock(&cpu_base->lock);
|
|
|
|
restart = fn(timer);
|
|
|
|
spin_lock(&cpu_base->lock);
|
2008-01-25 13:08:31 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Note: We clear the CALLBACK bit after enqueue_hrtimer to avoid
|
|
|
|
* reprogramming of the event hardware. This happens at the end of this
|
|
|
|
* function anyway.
|
|
|
|
*/
|
|
|
|
if (restart != HRTIMER_NORESTART) {
|
|
|
|
BUG_ON(timer->state != HRTIMER_STATE_CALLBACK);
|
|
|
|
enqueue_hrtimer(timer, base, 0);
|
|
|
|
}
|
|
|
|
timer->state &= ~HRTIMER_STATE_CALLBACK;
|
|
|
|
}
|
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
|
|
|
|
|
|
|
/*
|
|
|
|
* High resolution timer interrupt
|
|
|
|
* Called with interrupts disabled
|
|
|
|
*/
|
|
|
|
void hrtimer_interrupt(struct clock_event_device *dev)
|
|
|
|
{
|
|
|
|
struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
|
|
|
|
struct hrtimer_clock_base *base;
|
|
|
|
ktime_t expires_next, now;
|
2008-11-25 04:43:51 -07:00
|
|
|
int i;
|
2007-02-16 02:28:11 -07:00
|
|
|
|
|
|
|
BUG_ON(!cpu_base->hres_active);
|
|
|
|
cpu_base->nr_events++;
|
|
|
|
dev->next_event.tv64 = KTIME_MAX;
|
|
|
|
|
|
|
|
retry:
|
|
|
|
now = ktime_get();
|
|
|
|
|
|
|
|
expires_next.tv64 = KTIME_MAX;
|
|
|
|
|
|
|
|
base = cpu_base->clock_base;
|
|
|
|
|
|
|
|
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
|
|
|
|
ktime_t basenow;
|
|
|
|
struct rb_node *node;
|
|
|
|
|
|
|
|
spin_lock(&cpu_base->lock);
|
|
|
|
|
|
|
|
basenow = ktime_add(now, base->offset);
|
|
|
|
|
|
|
|
while ((node = base->first)) {
|
|
|
|
struct hrtimer *timer;
|
|
|
|
|
|
|
|
timer = rb_entry(node, struct hrtimer, node);
|
|
|
|
|
2008-09-01 16:47:08 -06:00
|
|
|
/*
|
|
|
|
* The immediate goal for using the softexpires is
|
|
|
|
* minimizing wakeups, not running timers at the
|
|
|
|
* earliest interrupt after their soft expiration.
|
|
|
|
* This allows us to avoid using a Priority Search
|
|
|
|
* Tree, which can answer a stabbing querry for
|
|
|
|
* overlapping intervals and instead use the simple
|
|
|
|
* BST we already have.
|
|
|
|
* We don't add extra wakeups by delaying timers that
|
|
|
|
* are right-of a not yet expired timer, because that
|
|
|
|
* timer will have to trigger a wakeup anyway.
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) {
|
2007-02-16 02:28:11 -07:00
|
|
|
ktime_t expires;
|
|
|
|
|
2008-09-01 16:02:30 -06:00
|
|
|
expires = ktime_sub(hrtimer_get_expires(timer),
|
2007-02-16 02:28:11 -07:00
|
|
|
base->offset);
|
|
|
|
if (expires.tv64 < expires_next.tv64)
|
|
|
|
expires_next = expires;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2008-01-25 13:08:31 -07:00
|
|
|
__run_hrtimer(timer);
|
2007-02-16 02:28:11 -07:00
|
|
|
}
|
|
|
|
spin_unlock(&cpu_base->lock);
|
|
|
|
base++;
|
|
|
|
}
|
|
|
|
|
|
|
|
cpu_base->expires_next = expires_next;
|
|
|
|
|
|
|
|
/* Reprogramming necessary ? */
|
|
|
|
if (expires_next.tv64 != KTIME_MAX) {
|
|
|
|
if (tick_program_event(expires_next, 0))
|
|
|
|
goto retry;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-01-05 03:28:19 -07:00
|
|
|
/*
|
|
|
|
* local version of hrtimer_peek_ahead_timers() called with interrupts
|
|
|
|
* disabled.
|
|
|
|
*/
|
|
|
|
static void __hrtimer_peek_ahead_timers(void)
|
|
|
|
{
|
|
|
|
struct tick_device *td;
|
|
|
|
|
|
|
|
if (!hrtimer_hres_active())
|
|
|
|
return;
|
|
|
|
|
|
|
|
td = &__get_cpu_var(tick_cpu_device);
|
|
|
|
if (td && td->evtdev)
|
|
|
|
hrtimer_interrupt(td->evtdev);
|
|
|
|
}
|
|
|
|
|
2008-09-10 17:06:00 -06:00
|
|
|
/**
|
|
|
|
* hrtimer_peek_ahead_timers -- run soft-expired timers now
|
|
|
|
*
|
|
|
|
* hrtimer_peek_ahead_timers will peek at the timer queue of
|
|
|
|
* the current cpu and check if there are any timers for which
|
|
|
|
* the soft expires time has passed. If any such timers exist,
|
|
|
|
* they are run immediately and then removed from the timer queue.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
void hrtimer_peek_ahead_timers(void)
|
|
|
|
{
|
2008-10-20 05:38:11 -06:00
|
|
|
unsigned long flags;
|
2008-10-13 08:32:15 -06:00
|
|
|
|
2008-09-10 17:06:00 -06:00
|
|
|
local_irq_save(flags);
|
2009-01-05 03:28:19 -07:00
|
|
|
__hrtimer_peek_ahead_timers();
|
2008-09-10 17:06:00 -06:00
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
2008-01-25 13:08:31 -07:00
|
|
|
#endif /* CONFIG_HIGH_RES_TIMERS */
|
[PATCH] Add debugging feature /proc/timer_stat
Add /proc/timer_stats support: debugging feature to profile timer expiration.
Both the starting site, process/PID and the expiration function is captured.
This allows the quick identification of timer event sources in a system.
Sample output:
# echo 1 > /proc/timer_stats
# cat /proc/timer_stats
Timer Stats Version: v0.1
Sample period: 4.010 s
24, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
11, 0 swapper sk_reset_timer (tcp_delack_timer)
6, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
17, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
4, 2050 pcscd do_nanosleep (hrtimer_wakeup)
5, 4179 sshd sk_reset_timer (tcp_write_timer)
4, 2248 yum-updatesd schedule_timeout (process_timeout)
18, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
3, 0 swapper sk_reset_timer (tcp_delack_timer)
1, 1 swapper neigh_table_init_no_netlink (neigh_periodic_timer)
2, 1 swapper e1000_up (e1000_watchdog)
1, 1 init schedule_timeout (process_timeout)
100 total events, 25.24 events/sec
[ cleanups and hrtimers support from Thomas Gleixner <tglx@linutronix.de> ]
[bunk@stusta.de: nr_entries can become static]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-16 02:28:13 -07:00
|
|
|
|
2008-01-25 13:08:31 -07:00
|
|
|
/*
|
|
|
|
* Called from timer softirq every jiffy, expire hrtimers:
|
|
|
|
*
|
|
|
|
* For HRT its the fall back code to run the softirq in the timer
|
|
|
|
* softirq context in case the hrtimer initialization failed or has
|
|
|
|
* not been done yet.
|
|
|
|
*/
|
|
|
|
void hrtimer_run_pending(void)
|
|
|
|
{
|
|
|
|
if (hrtimer_hres_active())
|
|
|
|
return;
|
2007-02-16 02:28:11 -07:00
|
|
|
|
2008-01-25 13:08:31 -07:00
|
|
|
/*
|
|
|
|
* This _is_ ugly: We have to check in the softirq context,
|
|
|
|
* whether we can switch to highres and / or nohz mode. The
|
|
|
|
* clocksource switch happens in the timer interrupt with
|
|
|
|
* xtime_lock held. Notification from there only sets the
|
|
|
|
* check bit in the tick_oneshot code, otherwise we might
|
|
|
|
* deadlock vs. xtime_lock.
|
|
|
|
*/
|
|
|
|
if (tick_check_oneshot_change(!hrtimer_is_hres_enabled()))
|
|
|
|
hrtimer_switch_to_hres();
|
2007-02-16 02:28:11 -07:00
|
|
|
}
|
|
|
|
|
2006-01-09 21:52:32 -07:00
|
|
|
/*
|
2008-01-25 13:08:31 -07:00
|
|
|
* Called from hardirq context every jiffy
|
2006-01-09 21:52:32 -07:00
|
|
|
*/
|
2008-04-18 14:39:00 -06:00
|
|
|
void hrtimer_run_queues(void)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2006-01-12 03:25:54 -07:00
|
|
|
struct rb_node *node;
|
2008-04-18 14:39:00 -06:00
|
|
|
struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
|
|
|
|
struct hrtimer_clock_base *base;
|
|
|
|
int index, gettime = 1;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2008-04-18 14:39:00 -06:00
|
|
|
if (hrtimer_hres_active())
|
2006-03-31 03:31:20 -07:00
|
|
|
return;
|
|
|
|
|
2008-04-18 14:39:00 -06:00
|
|
|
for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) {
|
|
|
|
base = &cpu_base->clock_base[index];
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2008-04-18 14:39:00 -06:00
|
|
|
if (!base->first)
|
2008-01-25 13:08:31 -07:00
|
|
|
continue;
|
2008-04-18 14:39:00 -06:00
|
|
|
|
2008-09-19 06:13:44 -06:00
|
|
|
if (gettime) {
|
2008-04-18 14:39:00 -06:00
|
|
|
hrtimer_get_softirq_time(cpu_base);
|
|
|
|
gettime = 0;
|
2006-03-26 02:38:09 -07:00
|
|
|
}
|
2008-01-25 13:08:31 -07:00
|
|
|
|
2008-04-18 14:39:00 -06:00
|
|
|
spin_lock(&cpu_base->lock);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2008-04-18 14:39:00 -06:00
|
|
|
while ((node = base->first)) {
|
|
|
|
struct hrtimer *timer;
|
2007-02-16 02:28:11 -07:00
|
|
|
|
2008-04-18 14:39:00 -06:00
|
|
|
timer = rb_entry(node, struct hrtimer, node);
|
2008-09-01 16:02:30 -06:00
|
|
|
if (base->softirq_time.tv64 <=
|
|
|
|
hrtimer_get_expires_tv64(timer))
|
2008-04-18 14:39:00 -06:00
|
|
|
break;
|
|
|
|
|
|
|
|
__run_hrtimer(timer);
|
|
|
|
}
|
|
|
|
spin_unlock(&cpu_base->lock);
|
|
|
|
}
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
|
|
|
|
2006-01-09 21:52:35 -07:00
|
|
|
/*
|
|
|
|
* Sleep related functions:
|
|
|
|
*/
|
2007-02-16 02:27:49 -07:00
|
|
|
static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
|
2006-03-31 03:31:17 -07:00
|
|
|
{
|
|
|
|
struct hrtimer_sleeper *t =
|
|
|
|
container_of(timer, struct hrtimer_sleeper, timer);
|
|
|
|
struct task_struct *task = t->task;
|
|
|
|
|
|
|
|
t->task = NULL;
|
|
|
|
if (task)
|
|
|
|
wake_up_process(task);
|
|
|
|
|
|
|
|
return HRTIMER_NORESTART;
|
|
|
|
}
|
|
|
|
|
2006-07-03 01:25:41 -06:00
|
|
|
void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
|
2006-03-31 03:31:17 -07:00
|
|
|
{
|
|
|
|
sl->timer.function = hrtimer_wakeup;
|
|
|
|
sl->task = task;
|
|
|
|
}
|
|
|
|
|
2006-03-31 03:31:19 -07:00
|
|
|
static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
|
2006-03-26 02:38:08 -07:00
|
|
|
{
|
2006-03-31 03:31:19 -07:00
|
|
|
hrtimer_init_sleeper(t, current);
|
2006-01-09 21:52:35 -07:00
|
|
|
|
2006-03-26 02:38:08 -07:00
|
|
|
do {
|
|
|
|
set_current_state(TASK_INTERRUPTIBLE);
|
2008-09-01 16:02:30 -06:00
|
|
|
hrtimer_start_expires(&t->timer, mode);
|
2008-01-25 13:08:32 -07:00
|
|
|
if (!hrtimer_active(&t->timer))
|
|
|
|
t->task = NULL;
|
2006-03-26 02:38:08 -07:00
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
if (likely(t->task))
|
|
|
|
schedule();
|
2006-03-26 02:38:08 -07:00
|
|
|
|
2006-03-31 03:31:19 -07:00
|
|
|
hrtimer_cancel(&t->timer);
|
2007-02-16 02:27:49 -07:00
|
|
|
mode = HRTIMER_MODE_ABS;
|
2006-03-31 03:31:19 -07:00
|
|
|
|
|
|
|
} while (t->task && !signal_pending(current));
|
2006-03-26 02:38:08 -07:00
|
|
|
|
2008-02-01 09:45:13 -07:00
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
|
2006-03-31 03:31:19 -07:00
|
|
|
return t->task == NULL;
|
2006-01-09 21:52:35 -07:00
|
|
|
}
|
|
|
|
|
2008-02-01 07:29:05 -07:00
|
|
|
static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp)
|
|
|
|
{
|
|
|
|
struct timespec rmt;
|
|
|
|
ktime_t rem;
|
|
|
|
|
2008-09-01 16:02:30 -06:00
|
|
|
rem = hrtimer_expires_remaining(timer);
|
2008-02-01 07:29:05 -07:00
|
|
|
if (rem.tv64 <= 0)
|
|
|
|
return 0;
|
|
|
|
rmt = ktime_to_timespec(rem);
|
|
|
|
|
|
|
|
if (copy_to_user(rmtp, &rmt, sizeof(*rmtp)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2006-09-29 03:00:28 -06:00
|
|
|
long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
|
2006-01-09 21:52:35 -07:00
|
|
|
{
|
2006-03-31 03:31:19 -07:00
|
|
|
struct hrtimer_sleeper t;
|
2008-02-01 07:29:05 -07:00
|
|
|
struct timespec __user *rmtp;
|
2008-04-30 01:55:04 -06:00
|
|
|
int ret = 0;
|
2006-01-09 21:52:35 -07:00
|
|
|
|
2008-04-30 01:55:04 -06:00
|
|
|
hrtimer_init_on_stack(&t.timer, restart->nanosleep.index,
|
|
|
|
HRTIMER_MODE_ABS);
|
2008-09-01 16:02:30 -06:00
|
|
|
hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
|
2006-01-09 21:52:35 -07:00
|
|
|
|
2007-02-16 02:27:49 -07:00
|
|
|
if (do_nanosleep(&t, HRTIMER_MODE_ABS))
|
2008-04-30 01:55:04 -06:00
|
|
|
goto out;
|
2006-01-09 21:52:35 -07:00
|
|
|
|
2008-02-10 01:17:43 -07:00
|
|
|
rmtp = restart->nanosleep.rmtp;
|
2006-03-26 02:38:08 -07:00
|
|
|
if (rmtp) {
|
2008-04-30 01:55:04 -06:00
|
|
|
ret = update_rmtp(&t.timer, rmtp);
|
2008-02-01 07:29:05 -07:00
|
|
|
if (ret <= 0)
|
2008-04-30 01:55:04 -06:00
|
|
|
goto out;
|
2006-03-26 02:38:08 -07:00
|
|
|
}
|
2006-01-09 21:52:35 -07:00
|
|
|
|
|
|
|
/* The other values in restart are already filled in */
|
2008-04-30 01:55:04 -06:00
|
|
|
ret = -ERESTART_RESTARTBLOCK;
|
|
|
|
out:
|
|
|
|
destroy_hrtimer_on_stack(&t.timer);
|
|
|
|
return ret;
|
2006-01-09 21:52:35 -07:00
|
|
|
}
|
|
|
|
|
2008-02-01 07:29:05 -07:00
|
|
|
long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
|
2006-01-09 21:52:35 -07:00
|
|
|
const enum hrtimer_mode mode, const clockid_t clockid)
|
|
|
|
{
|
|
|
|
struct restart_block *restart;
|
2006-03-31 03:31:19 -07:00
|
|
|
struct hrtimer_sleeper t;
|
2008-04-30 01:55:04 -06:00
|
|
|
int ret = 0;
|
2008-09-08 09:58:59 -06:00
|
|
|
unsigned long slack;
|
|
|
|
|
|
|
|
slack = current->timer_slack_ns;
|
|
|
|
if (rt_task(current))
|
|
|
|
slack = 0;
|
2006-01-09 21:52:35 -07:00
|
|
|
|
2008-04-30 01:55:04 -06:00
|
|
|
hrtimer_init_on_stack(&t.timer, clockid, mode);
|
2008-09-08 09:58:59 -06:00
|
|
|
hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack);
|
2006-03-26 02:38:08 -07:00
|
|
|
if (do_nanosleep(&t, mode))
|
2008-04-30 01:55:04 -06:00
|
|
|
goto out;
|
2006-01-09 21:52:35 -07:00
|
|
|
|
2006-02-01 04:05:11 -07:00
|
|
|
/* Absolute timers do not update the rmtp value and restart: */
|
2008-04-30 01:55:04 -06:00
|
|
|
if (mode == HRTIMER_MODE_ABS) {
|
|
|
|
ret = -ERESTARTNOHAND;
|
|
|
|
goto out;
|
|
|
|
}
|
2006-01-09 21:52:35 -07:00
|
|
|
|
2006-03-26 02:38:08 -07:00
|
|
|
if (rmtp) {
|
2008-04-30 01:55:04 -06:00
|
|
|
ret = update_rmtp(&t.timer, rmtp);
|
2008-02-01 07:29:05 -07:00
|
|
|
if (ret <= 0)
|
2008-04-30 01:55:04 -06:00
|
|
|
goto out;
|
2006-03-26 02:38:08 -07:00
|
|
|
}
|
2006-01-09 21:52:35 -07:00
|
|
|
|
|
|
|
restart = ¤t_thread_info()->restart_block;
|
2006-09-29 03:00:28 -06:00
|
|
|
restart->fn = hrtimer_nanosleep_restart;
|
2008-02-10 01:17:43 -07:00
|
|
|
restart->nanosleep.index = t.timer.base->index;
|
|
|
|
restart->nanosleep.rmtp = rmtp;
|
2008-09-01 16:02:30 -06:00
|
|
|
restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
|
2006-01-09 21:52:35 -07:00
|
|
|
|
2008-04-30 01:55:04 -06:00
|
|
|
ret = -ERESTART_RESTARTBLOCK;
|
|
|
|
out:
|
|
|
|
destroy_hrtimer_on_stack(&t.timer);
|
|
|
|
return ret;
|
2006-01-09 21:52:35 -07:00
|
|
|
}
|
|
|
|
|
2006-01-09 21:52:36 -07:00
|
|
|
asmlinkage long
|
|
|
|
sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp)
|
|
|
|
{
|
2008-02-01 07:29:05 -07:00
|
|
|
struct timespec tu;
|
2006-01-09 21:52:36 -07:00
|
|
|
|
|
|
|
if (copy_from_user(&tu, rqtp, sizeof(tu)))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
if (!timespec_valid(&tu))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2008-02-01 07:29:05 -07:00
|
|
|
return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
|
2006-01-09 21:52:36 -07:00
|
|
|
}
|
|
|
|
|
2006-01-09 21:52:32 -07:00
|
|
|
/*
|
|
|
|
* Functions related to boot-time initialization:
|
|
|
|
*/
|
2008-01-21 18:18:24 -07:00
|
|
|
static void __cpuinit init_hrtimers_cpu(int cpu)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
|
2006-01-09 21:52:32 -07:00
|
|
|
int i;
|
|
|
|
|
2007-02-16 02:27:50 -07:00
|
|
|
spin_lock_init(&cpu_base->lock);
|
|
|
|
|
|
|
|
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
|
|
|
|
cpu_base->clock_base[i].cpu_base = cpu_base;
|
|
|
|
|
2007-02-16 02:28:11 -07:00
|
|
|
hrtimer_init_hres(cpu_base);
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
|
2008-11-25 04:43:51 -07:00
|
|
|
static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
|
2008-12-04 03:17:10 -07:00
|
|
|
struct hrtimer_clock_base *new_base)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
|
|
|
struct hrtimer *timer;
|
|
|
|
struct rb_node *node;
|
|
|
|
|
|
|
|
while ((node = rb_first(&old_base->active))) {
|
|
|
|
timer = rb_entry(node, struct hrtimer, node);
|
2007-02-16 02:28:11 -07:00
|
|
|
BUG_ON(hrtimer_callback_running(timer));
|
2008-04-30 01:55:04 -06:00
|
|
|
debug_hrtimer_deactivate(timer);
|
2008-09-29 07:44:46 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Mark it as STATE_MIGRATE not INACTIVE otherwise the
|
|
|
|
* timer could be seen as !active and just vanish away
|
|
|
|
* under us on another CPU
|
|
|
|
*/
|
|
|
|
__remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0);
|
2006-01-09 21:52:32 -07:00
|
|
|
timer->base = new_base;
|
2007-02-16 02:28:11 -07:00
|
|
|
/*
|
2008-12-04 03:17:10 -07:00
|
|
|
* Enqueue the timers on the new cpu, but do not reprogram
|
|
|
|
* the timer as that would enable a deadlock between
|
|
|
|
* hrtimer_enqueue_reprogramm() running the timer and us still
|
|
|
|
* holding a nested base lock.
|
|
|
|
*
|
|
|
|
* Instead we tickle the hrtimer interrupt after the migration
|
|
|
|
* is done, which will run all expired timers and re-programm
|
|
|
|
* the timer device.
|
2007-02-16 02:28:11 -07:00
|
|
|
*/
|
2008-12-04 03:17:10 -07:00
|
|
|
enqueue_hrtimer(timer, new_base, 0);
|
2008-09-29 06:09:39 -06:00
|
|
|
|
2008-09-29 07:44:46 -06:00
|
|
|
/* Clear the migration state bit */
|
|
|
|
timer->state &= ~HRTIMER_STATE_MIGRATE;
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-01-05 03:28:20 -07:00
|
|
|
static void migrate_hrtimers(int scpu)
|
2006-01-09 21:52:32 -07:00
|
|
|
{
|
2007-02-16 02:27:50 -07:00
|
|
|
struct hrtimer_cpu_base *old_base, *new_base;
|
2009-01-05 03:28:21 -07:00
|
|
|
int i;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2008-12-04 03:17:10 -07:00
|
|
|
BUG_ON(cpu_online(scpu));
|
|
|
|
tick_cancel_sched_timer(scpu);
|
2009-01-05 03:28:21 -07:00
|
|
|
|
|
|
|
local_irq_disable();
|
|
|
|
old_base = &per_cpu(hrtimer_bases, scpu);
|
|
|
|
new_base = &__get_cpu_var(hrtimer_bases);
|
2008-08-20 17:46:04 -06:00
|
|
|
/*
|
|
|
|
* The caller is globally serialized and nobody else
|
|
|
|
* takes two locks at once, deadlock is not possible.
|
|
|
|
*/
|
2009-01-05 03:28:21 -07:00
|
|
|
spin_lock(&new_base->lock);
|
2008-04-04 12:54:10 -06:00
|
|
|
spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
|
2006-01-09 21:52:32 -07:00
|
|
|
|
2007-02-16 02:27:50 -07:00
|
|
|
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
|
2008-11-25 04:43:51 -07:00
|
|
|
migrate_hrtimer_list(&old_base->clock_base[i],
|
2008-12-04 03:17:10 -07:00
|
|
|
&new_base->clock_base[i]);
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
|
|
|
|
2008-04-04 12:54:10 -06:00
|
|
|
spin_unlock(&old_base->lock);
|
2009-01-05 03:28:21 -07:00
|
|
|
spin_unlock(&new_base->lock);
|
2008-12-04 03:17:10 -07:00
|
|
|
|
2009-01-05 03:28:21 -07:00
|
|
|
/* Check, if we got expired work to do */
|
|
|
|
__hrtimer_peek_ahead_timers();
|
|
|
|
local_irq_enable();
|
2006-01-09 21:52:32 -07:00
|
|
|
}
|
2008-12-04 03:17:10 -07:00
|
|
|
|
2006-01-09 21:52:32 -07:00
|
|
|
#endif /* CONFIG_HOTPLUG_CPU */
|
|
|
|
|
2006-07-30 04:03:35 -06:00
|
|
|
static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self,
|
2006-01-09 21:52:32 -07:00
|
|
|
unsigned long action, void *hcpu)
|
|
|
|
{
|
2008-12-18 16:48:27 -07:00
|
|
|
int scpu = (long)hcpu;
|
2006-01-09 21:52:32 -07:00
|
|
|
|
|
|
|
switch (action) {
|
|
|
|
|
|
|
|
case CPU_UP_PREPARE:
|
2007-05-09 03:35:10 -06:00
|
|
|
case CPU_UP_PREPARE_FROZEN:
|
2008-12-04 03:17:10 -07:00
|
|
|
init_hrtimers_cpu(scpu);
|
2006-01-09 21:52:32 -07:00
|
|
|
break;
|
|
|
|
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
case CPU_DEAD:
|
2007-05-09 03:35:10 -06:00
|
|
|
case CPU_DEAD_FROZEN:
|
2008-12-18 16:48:27 -07:00
|
|
|
{
|
2008-12-04 03:17:10 -07:00
|
|
|
clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &scpu);
|
2009-01-05 03:28:20 -07:00
|
|
|
migrate_hrtimers(scpu);
|
2006-01-09 21:52:32 -07:00
|
|
|
break;
|
2008-12-18 16:48:27 -07:00
|
|
|
}
|
2006-01-09 21:52:32 -07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NOTIFY_OK;
|
|
|
|
}
|
|
|
|
|
2006-07-30 04:03:35 -06:00
|
|
|
static struct notifier_block __cpuinitdata hrtimers_nb = {
|
2006-01-09 21:52:32 -07:00
|
|
|
.notifier_call = hrtimer_cpu_notify,
|
|
|
|
};
|
|
|
|
|
|
|
|
void __init hrtimers_init(void)
|
|
|
|
{
|
|
|
|
hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE,
|
|
|
|
(void *)(long)smp_processor_id());
|
|
|
|
register_cpu_notifier(&hrtimers_nb);
|
|
|
|
}
|
|
|
|
|
2008-08-31 09:05:58 -06:00
|
|
|
/**
|
2008-09-01 16:47:08 -06:00
|
|
|
* schedule_hrtimeout_range - sleep until timeout
|
2008-08-31 09:05:58 -06:00
|
|
|
* @expires: timeout value (ktime_t)
|
2008-09-01 16:47:08 -06:00
|
|
|
* @delta: slack in expires timeout (ktime_t)
|
2008-08-31 09:05:58 -06:00
|
|
|
* @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
|
|
|
|
*
|
|
|
|
* Make the current task sleep until the given expiry time has
|
|
|
|
* elapsed. The routine will return immediately unless
|
|
|
|
* the current task state has been set (see set_current_state()).
|
|
|
|
*
|
2008-09-01 16:47:08 -06:00
|
|
|
* The @delta argument gives the kernel the freedom to schedule the
|
|
|
|
* actual wakeup to a time that is both power and performance friendly.
|
|
|
|
* The kernel give the normal best effort behavior for "@expires+@delta",
|
|
|
|
* but may decide to fire the timer earlier, but no earlier than @expires.
|
|
|
|
*
|
2008-08-31 09:05:58 -06:00
|
|
|
* You can set the task state as follows -
|
|
|
|
*
|
|
|
|
* %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
|
|
|
|
* pass before the routine returns.
|
|
|
|
*
|
|
|
|
* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
|
|
|
|
* delivered to the current task.
|
|
|
|
*
|
|
|
|
* The current task state is guaranteed to be TASK_RUNNING when this
|
|
|
|
* routine returns.
|
|
|
|
*
|
|
|
|
* Returns 0 when the timer has expired otherwise -EINTR
|
|
|
|
*/
|
2008-09-01 16:47:08 -06:00
|
|
|
int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta,
|
2008-08-31 09:05:58 -06:00
|
|
|
const enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
struct hrtimer_sleeper t;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Optimize when a zero timeout value is given. It does not
|
|
|
|
* matter whether this is an absolute or a relative time.
|
|
|
|
*/
|
|
|
|
if (expires && !expires->tv64) {
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A NULL parameter means "inifinte"
|
|
|
|
*/
|
|
|
|
if (!expires) {
|
|
|
|
schedule();
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
return -EINTR;
|
|
|
|
}
|
|
|
|
|
|
|
|
hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, mode);
|
2008-09-01 16:47:08 -06:00
|
|
|
hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
|
2008-08-31 09:05:58 -06:00
|
|
|
|
|
|
|
hrtimer_init_sleeper(&t, current);
|
|
|
|
|
2008-09-01 16:02:30 -06:00
|
|
|
hrtimer_start_expires(&t.timer, mode);
|
2008-08-31 09:05:58 -06:00
|
|
|
if (!hrtimer_active(&t.timer))
|
|
|
|
t.task = NULL;
|
|
|
|
|
|
|
|
if (likely(t.task))
|
|
|
|
schedule();
|
|
|
|
|
|
|
|
hrtimer_cancel(&t.timer);
|
|
|
|
destroy_hrtimer_on_stack(&t.timer);
|
|
|
|
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
|
|
|
|
return !t.task ? 0 : -EINTR;
|
|
|
|
}
|
2008-09-01 16:47:08 -06:00
|
|
|
EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* schedule_hrtimeout - sleep until timeout
|
|
|
|
* @expires: timeout value (ktime_t)
|
|
|
|
* @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
|
|
|
|
*
|
|
|
|
* Make the current task sleep until the given expiry time has
|
|
|
|
* elapsed. The routine will return immediately unless
|
|
|
|
* the current task state has been set (see set_current_state()).
|
|
|
|
*
|
|
|
|
* You can set the task state as follows -
|
|
|
|
*
|
|
|
|
* %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
|
|
|
|
* pass before the routine returns.
|
|
|
|
*
|
|
|
|
* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
|
|
|
|
* delivered to the current task.
|
|
|
|
*
|
|
|
|
* The current task state is guaranteed to be TASK_RUNNING when this
|
|
|
|
* routine returns.
|
|
|
|
*
|
|
|
|
* Returns 0 when the timer has expired otherwise -EINTR
|
|
|
|
*/
|
|
|
|
int __sched schedule_hrtimeout(ktime_t *expires,
|
|
|
|
const enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
return schedule_hrtimeout_range(expires, 0, mode);
|
|
|
|
}
|
2008-08-31 09:05:58 -06:00
|
|
|
EXPORT_SYMBOL_GPL(schedule_hrtimeout);
|