redonkable
/
alistair23-linux
1
0
Fork 0
Code Releases Activity

Merge branch 'rcu/urgent' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu into core/urgent 

Merge RCU fixes from Paul E. McKenney:

 " This series has four patches, the major point of which is to eliminate
   some slowdowns (including boot-time slowdowns) resulting from some
   RCU_FAST_NO_HZ changes.  The issue with the changes is that posting timers
   from the idle loop has no effect if the CPU has entered dyntick-idle
   mode because the CPU has already computed its wakeup time, and posting
   a timer does not cause it to be recomputed.  The short-term fix is for
   RCU to precompute the timeout value so that the CPU's calculation is
   correct. "

Signed-off-by: Ingo Molnar <mingo@kernel.org>
hifive-unleashed-5.1
Ingo Molnar 2012-06-11 10:30:23 +02:00
parent eab309494a aa9b16306e
commit 4a1e001d2b
7 changed files with 116 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/linux/rcutiny.h
View File

@ -87,8 +87,9 @@ static inline void kfree_call_rcu(struct rcu_head *head,
#ifdef CONFIG_TINY_RCU #ifdef CONFIG_TINY_RCU
static inline int rcu_needs_cpu(int cpu) static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
{ {
*delta_jiffies = ULONG_MAX;
return 0; return 0;
} }
@ -96,8 +97,9 @@ static inline int rcu_needs_cpu(int cpu)
int rcu_preempt_needs_cpu(void); int rcu_preempt_needs_cpu(void);
static inline int rcu_needs_cpu(int cpu) static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
{ {
*delta_jiffies = ULONG_MAX;
return rcu_preempt_needs_cpu(); return rcu_preempt_needs_cpu();
} }

2
include/linux/rcutree.h
View File

@ -32,7 +32,7 @@
extern void rcu_init(void); extern void rcu_init(void);
extern void rcu_note_context_switch(int cpu); extern void rcu_note_context_switch(int cpu);
extern int rcu_needs_cpu(int cpu); extern int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies);
extern void rcu_cpu_stall_reset(void); extern void rcu_cpu_stall_reset(void);
/* /*

1
include/trace/events/rcu.h
View File

@ -289,6 +289,7 @@ TRACE_EVENT(rcu_dyntick,
* "In holdoff": Nothing to do, holding off after unsuccessful attempt. * "In holdoff": Nothing to do, holding off after unsuccessful attempt.
* "Begin holdoff": Attempt failed, don't retry until next jiffy. * "Begin holdoff": Attempt failed, don't retry until next jiffy.
* "Dyntick with callbacks": Entering dyntick-idle despite callbacks. * "Dyntick with callbacks": Entering dyntick-idle despite callbacks.
* "Dyntick with lazy callbacks": Entering dyntick-idle w/lazy callbacks.
* "More callbacks": Still more callbacks, try again to clear them out. * "More callbacks": Still more callbacks, try again to clear them out.
* "Callbacks drained": All callbacks processed, off to dyntick idle! * "Callbacks drained": All callbacks processed, off to dyntick idle!
* "Timer": Timer fired to cause CPU to continue processing callbacks. * "Timer": Timer fired to cause CPU to continue processing callbacks.

2
kernel/rcutree.c
View File

@ -1397,6 +1397,8 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
rdp->qlen_lazy += rsp->qlen_lazy; rdp->qlen_lazy += rsp->qlen_lazy;
rdp->qlen += rsp->qlen; rdp->qlen += rsp->qlen;
rdp->n_cbs_adopted += rsp->qlen; rdp->n_cbs_adopted += rsp->qlen;
if (rsp->qlen_lazy != rsp->qlen)
rcu_idle_count_callbacks_posted();
rsp->qlen_lazy = 0; rsp->qlen_lazy = 0;
rsp->qlen = 0; rsp->qlen = 0;

14
kernel/rcutree.h
View File

@ -84,6 +84,20 @@ struct rcu_dynticks {
/* Process level is worth LLONG_MAX/2. */ /* Process level is worth LLONG_MAX/2. */
int dynticks_nmi_nesting; /* Track NMI nesting level. */ int dynticks_nmi_nesting; /* Track NMI nesting level. */
atomic_t dynticks; /* Even value for idle, else odd. */ atomic_t dynticks; /* Even value for idle, else odd. */
#ifdef CONFIG_RCU_FAST_NO_HZ
int dyntick_drain; /* Prepare-for-idle state variable. */
unsigned long dyntick_holdoff;
/* No retries for the jiffy of failure. */
struct timer_list idle_gp_timer;
/* Wake up CPU sleeping with callbacks. */
unsigned long idle_gp_timer_expires;
/* When to wake up CPU (for repost). */
bool idle_first_pass; /* First pass of attempt to go idle? */
unsigned long nonlazy_posted;
/* # times non-lazy CBs posted to CPU. */
unsigned long nonlazy_posted_snap;
/* idle-period nonlazy_posted snapshot. */
#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
}; };
/* RCU's kthread states for tracing. */ /* RCU's kthread states for tracing. */

163
kernel/rcutree_plugin.h
View File

@ -1886,8 +1886,9 @@ static void __cpuinit rcu_prepare_kthreads(int cpu)
* Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
* any flavor of RCU. * any flavor of RCU.
*/ */
int rcu_needs_cpu(int cpu) int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
{ {
*delta_jiffies = ULONG_MAX;
return rcu_cpu_has_callbacks(cpu); return rcu_cpu_has_callbacks(cpu);
} }
@ -1962,41 +1963,6 @@ static void rcu_idle_count_callbacks_posted(void)
#define RCU_IDLE_GP_DELAY 6 /* Roughly one grace period. */ #define RCU_IDLE_GP_DELAY 6 /* Roughly one grace period. */
#define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */ #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
/* Loop counter for rcu_prepare_for_idle(). */
static DEFINE_PER_CPU(int, rcu_dyntick_drain);
/* If rcu_dyntick_holdoff==jiffies, don't try to enter dyntick-idle mode. */
static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
/* Timer to awaken the CPU if it enters dyntick-idle mode with callbacks. */
static DEFINE_PER_CPU(struct timer_list, rcu_idle_gp_timer);
/* Scheduled expiry time for rcu_idle_gp_timer to allow reposting. */
static DEFINE_PER_CPU(unsigned long, rcu_idle_gp_timer_expires);
/* Enable special processing on first attempt to enter dyntick-idle mode. */
static DEFINE_PER_CPU(bool, rcu_idle_first_pass);
/* Running count of non-lazy callbacks posted, never decremented. */
static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted);
/* Snapshot of rcu_nonlazy_posted to detect meaningful exits from idle. */
static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted_snap);
/*
* Allow the CPU to enter dyntick-idle mode if either: (1) There are no
* callbacks on this CPU, (2) this CPU has not yet attempted to enter
* dyntick-idle mode, or (3) this CPU is in the process of attempting to
* enter dyntick-idle mode. Otherwise, if we have recently tried and failed
* to enter dyntick-idle mode, we refuse to try to enter it. After all,
* it is better to incur scheduling-clock interrupts than to spin
* continuously for the same time duration!
*/
int rcu_needs_cpu(int cpu)
{
/* Flag a new idle sojourn to the idle-entry state machine. */
per_cpu(rcu_idle_first_pass, cpu) = 1;
/* If no callbacks, RCU doesn't need the CPU. */
if (!rcu_cpu_has_callbacks(cpu))
return 0;
/* Otherwise, RCU needs the CPU only if it recently tried and failed. */
return per_cpu(rcu_dyntick_holdoff, cpu) == jiffies;
}
/* /*
* Does the specified flavor of RCU have non-lazy callbacks pending on * Does the specified flavor of RCU have non-lazy callbacks pending on
* the specified CPU? Both RCU flavor and CPU are specified by the * the specified CPU? Both RCU flavor and CPU are specified by the
@ -2039,6 +2005,47 @@ static bool rcu_cpu_has_nonlazy_callbacks(int cpu)
rcu_preempt_cpu_has_nonlazy_callbacks(cpu); rcu_preempt_cpu_has_nonlazy_callbacks(cpu);
} }
/*
* Allow the CPU to enter dyntick-idle mode if either: (1) There are no
* callbacks on this CPU, (2) this CPU has not yet attempted to enter
* dyntick-idle mode, or (3) this CPU is in the process of attempting to
* enter dyntick-idle mode. Otherwise, if we have recently tried and failed
* to enter dyntick-idle mode, we refuse to try to enter it. After all,
* it is better to incur scheduling-clock interrupts than to spin
* continuously for the same time duration!
*
* The delta_jiffies argument is used to store the time when RCU is
* going to need the CPU again if it still has callbacks. The reason
* for this is that rcu_prepare_for_idle() might need to post a timer,
* but if so, it will do so after tick_nohz_stop_sched_tick() has set
* the wakeup time for this CPU. This means that RCU's timer can be
* delayed until the wakeup time, which defeats the purpose of posting
* a timer.
*/
int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
{
struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
/* Flag a new idle sojourn to the idle-entry state machine. */
rdtp->idle_first_pass = 1;
/* If no callbacks, RCU doesn't need the CPU. */
if (!rcu_cpu_has_callbacks(cpu)) {
*delta_jiffies = ULONG_MAX;
return 0;
}
if (rdtp->dyntick_holdoff == jiffies) {
/* RCU recently tried and failed, so don't try again. */
*delta_jiffies = 1;
return 1;
}
/* Set up for the possibility that RCU will post a timer. */
if (rcu_cpu_has_nonlazy_callbacks(cpu))
*delta_jiffies = RCU_IDLE_GP_DELAY;
else
*delta_jiffies = RCU_IDLE_LAZY_GP_DELAY;
return 0;
}
/* /*
* Handler for smp_call_function_single(). The only point of this * Handler for smp_call_function_single(). The only point of this
* handler is to wake the CPU up, so the handler does only tracing. * handler is to wake the CPU up, so the handler does only tracing.
@ -2075,21 +2082,24 @@ static void rcu_idle_gp_timer_func(unsigned long cpu_in)
*/ */
static void rcu_prepare_for_idle_init(int cpu) static void rcu_prepare_for_idle_init(int cpu)
{ {
per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1; struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
setup_timer(&per_cpu(rcu_idle_gp_timer, cpu),
rcu_idle_gp_timer_func, cpu); rdtp->dyntick_holdoff = jiffies - 1;
per_cpu(rcu_idle_gp_timer_expires, cpu) = jiffies - 1; setup_timer(&rdtp->idle_gp_timer, rcu_idle_gp_timer_func, cpu);
per_cpu(rcu_idle_first_pass, cpu) = 1; rdtp->idle_gp_timer_expires = jiffies - 1;
rdtp->idle_first_pass = 1;
} }
/* /*
* Clean up for exit from idle. Because we are exiting from idle, there * Clean up for exit from idle. Because we are exiting from idle, there
* is no longer any point to rcu_idle_gp_timer, so cancel it. This will * is no longer any point to ->idle_gp_timer, so cancel it. This will
* do nothing if this timer is not active, so just cancel it unconditionally. * do nothing if this timer is not active, so just cancel it unconditionally.
*/ */
static void rcu_cleanup_after_idle(int cpu) static void rcu_cleanup_after_idle(int cpu)
{ {
del_timer(&per_cpu(rcu_idle_gp_timer, cpu)); struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
del_timer(&rdtp->idle_gp_timer);
trace_rcu_prep_idle("Cleanup after idle"); trace_rcu_prep_idle("Cleanup after idle");
} }
@ -2108,42 +2118,41 @@ static void rcu_cleanup_after_idle(int cpu)
* Because it is not legal to invoke rcu_process_callbacks() with irqs * Because it is not legal to invoke rcu_process_callbacks() with irqs
* disabled, we do one pass of force_quiescent_state(), then do a * disabled, we do one pass of force_quiescent_state(), then do a
* invoke_rcu_core() to cause rcu_process_callbacks() to be invoked * invoke_rcu_core() to cause rcu_process_callbacks() to be invoked
* later. The per-cpu rcu_dyntick_drain variable controls the sequencing. * later. The ->dyntick_drain field controls the sequencing.
* *
* The caller must have disabled interrupts. * The caller must have disabled interrupts.
*/ */
static void rcu_prepare_for_idle(int cpu) static void rcu_prepare_for_idle(int cpu)
{ {
struct timer_list *tp; struct timer_list *tp;
struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
/* /*
* If this is an idle re-entry, for example, due to use of * If this is an idle re-entry, for example, due to use of
* RCU_NONIDLE() or the new idle-loop tracing API within the idle * RCU_NONIDLE() or the new idle-loop tracing API within the idle
* loop, then don't take any state-machine actions, unless the * loop, then don't take any state-machine actions, unless the
* momentary exit from idle queued additional non-lazy callbacks. * momentary exit from idle queued additional non-lazy callbacks.
* Instead, repost the rcu_idle_gp_timer if this CPU has callbacks * Instead, repost the ->idle_gp_timer if this CPU has callbacks
* pending. * pending.
*/ */
if (!per_cpu(rcu_idle_first_pass, cpu) && if (!rdtp->idle_first_pass &&
(per_cpu(rcu_nonlazy_posted, cpu) == (rdtp->nonlazy_posted == rdtp->nonlazy_posted_snap)) {
per_cpu(rcu_nonlazy_posted_snap, cpu))) {
if (rcu_cpu_has_callbacks(cpu)) { if (rcu_cpu_has_callbacks(cpu)) {
tp = &per_cpu(rcu_idle_gp_timer, cpu); tp = &rdtp->idle_gp_timer;
mod_timer_pinned(tp, per_cpu(rcu_idle_gp_timer_expires, cpu)); mod_timer_pinned(tp, rdtp->idle_gp_timer_expires);
} }
return; return;
} }
per_cpu(rcu_idle_first_pass, cpu) = 0; rdtp->idle_first_pass = 0;
per_cpu(rcu_nonlazy_posted_snap, cpu) = rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted - 1;
per_cpu(rcu_nonlazy_posted, cpu) - 1;
/* /*
* If there are no callbacks on this CPU, enter dyntick-idle mode. * If there are no callbacks on this CPU, enter dyntick-idle mode.
* Also reset state to avoid prejudicing later attempts. * Also reset state to avoid prejudicing later attempts.
*/ */
if (!rcu_cpu_has_callbacks(cpu)) { if (!rcu_cpu_has_callbacks(cpu)) {
per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1; rdtp->dyntick_holdoff = jiffies - 1;
per_cpu(rcu_dyntick_drain, cpu) = 0; rdtp->dyntick_drain = 0;
trace_rcu_prep_idle("No callbacks"); trace_rcu_prep_idle("No callbacks");
return; return;
} }
@ -2152,36 +2161,37 @@ static void rcu_prepare_for_idle(int cpu)
* If in holdoff mode, just return. We will presumably have * If in holdoff mode, just return. We will presumably have
* refrained from disabling the scheduling-clock tick. * refrained from disabling the scheduling-clock tick.
*/ */
if (per_cpu(rcu_dyntick_holdoff, cpu) == jiffies) { if (rdtp->dyntick_holdoff == jiffies) {
trace_rcu_prep_idle("In holdoff"); trace_rcu_prep_idle("In holdoff");
return; return;
} }
/* Check and update the rcu_dyntick_drain sequencing. */ /* Check and update the ->dyntick_drain sequencing. */
if (per_cpu(rcu_dyntick_drain, cpu) <= 0) { if (rdtp->dyntick_drain <= 0) {
/* First time through, initialize the counter. */ /* First time through, initialize the counter. */
per_cpu(rcu_dyntick_drain, cpu) = RCU_IDLE_FLUSHES; rdtp->dyntick_drain = RCU_IDLE_FLUSHES;
} else if (per_cpu(rcu_dyntick_drain, cpu) <= RCU_IDLE_OPT_FLUSHES && } else if (rdtp->dyntick_drain <= RCU_IDLE_OPT_FLUSHES &&
!rcu_pending(cpu) && !rcu_pending(cpu) &&
!local_softirq_pending()) { !local_softirq_pending()) {
/* Can we go dyntick-idle despite still having callbacks? */ /* Can we go dyntick-idle despite still having callbacks? */
rdtp->dyntick_drain = 0;
rdtp->dyntick_holdoff = jiffies;
if (rcu_cpu_has_nonlazy_callbacks(cpu)) {
trace_rcu_prep_idle("Dyntick with callbacks"); trace_rcu_prep_idle("Dyntick with callbacks");
per_cpu(rcu_dyntick_drain, cpu) = 0; rdtp->idle_gp_timer_expires =
per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
if (rcu_cpu_has_nonlazy_callbacks(cpu))
per_cpu(rcu_idle_gp_timer_expires, cpu) =
jiffies + RCU_IDLE_GP_DELAY; jiffies + RCU_IDLE_GP_DELAY;
else } else {
per_cpu(rcu_idle_gp_timer_expires, cpu) = rdtp->idle_gp_timer_expires =
jiffies + RCU_IDLE_LAZY_GP_DELAY; jiffies + RCU_IDLE_LAZY_GP_DELAY;
tp = &per_cpu(rcu_idle_gp_timer, cpu); trace_rcu_prep_idle("Dyntick with lazy callbacks");
mod_timer_pinned(tp, per_cpu(rcu_idle_gp_timer_expires, cpu)); }
per_cpu(rcu_nonlazy_posted_snap, cpu) = tp = &rdtp->idle_gp_timer;
per_cpu(rcu_nonlazy_posted, cpu); mod_timer_pinned(tp, rdtp->idle_gp_timer_expires);
rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
return; /* Nothing more to do immediately. */ return; /* Nothing more to do immediately. */
} else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) { } else if (--(rdtp->dyntick_drain) <= 0) {
/* We have hit the limit, so time to give up. */ /* We have hit the limit, so time to give up. */
per_cpu(rcu_dyntick_holdoff, cpu) = jiffies; rdtp->dyntick_holdoff = jiffies;
trace_rcu_prep_idle("Begin holdoff"); trace_rcu_prep_idle("Begin holdoff");
invoke_rcu_core(); /* Force the CPU out of dyntick-idle. */ invoke_rcu_core(); /* Force the CPU out of dyntick-idle. */
return; return;
@ -2227,7 +2237,7 @@ static void rcu_prepare_for_idle(int cpu)
*/ */
static void rcu_idle_count_callbacks_posted(void) static void rcu_idle_count_callbacks_posted(void)
{ {
__this_cpu_add(rcu_nonlazy_posted, 1); __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
} }
#endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */ #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
@ -2238,11 +2248,12 @@ static void rcu_idle_count_callbacks_posted(void)
static void print_cpu_stall_fast_no_hz(char *cp, int cpu) static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
{ {
struct timer_list *tltp = &per_cpu(rcu_idle_gp_timer, cpu); struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
struct timer_list *tltp = &rdtp->idle_gp_timer;
sprintf(cp, "drain=%d %c timer=%lu", sprintf(cp, "drain=%d %c timer=%lu",
per_cpu(rcu_dyntick_drain, cpu), rdtp->dyntick_drain,
per_cpu(rcu_dyntick_holdoff, cpu) == jiffies ? 'H' : '.', rdtp->dyntick_holdoff == jiffies ? 'H' : '.',
timer_pending(tltp) ? tltp->expires - jiffies : -1); timer_pending(tltp) ? tltp->expires - jiffies : -1);
} }

7
kernel/time/tick-sched.c
View File

@ -274,6 +274,7 @@ EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
static void tick_nohz_stop_sched_tick(struct tick_sched *ts) static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
{ {
unsigned long seq, last_jiffies, next_jiffies, delta_jiffies; unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;
unsigned long rcu_delta_jiffies;
ktime_t last_update, expires, now; ktime_t last_update, expires, now;
struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
u64 time_delta; u64 time_delta;
@ -322,7 +323,7 @@ static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
time_delta = timekeeping_max_deferment(); time_delta = timekeeping_max_deferment();
} while (read_seqretry(&xtime_lock, seq)); } while (read_seqretry(&xtime_lock, seq));
if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu) || if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || printk_needs_cpu(cpu) ||
arch_needs_cpu(cpu)) { arch_needs_cpu(cpu)) {
next_jiffies = last_jiffies + 1; next_jiffies = last_jiffies + 1;
delta_jiffies = 1; delta_jiffies = 1;
@ -330,6 +331,10 @@ static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
/* Get the next timer wheel timer */ /* Get the next timer wheel timer */
next_jiffies = get_next_timer_interrupt(last_jiffies); next_jiffies = get_next_timer_interrupt(last_jiffies);
delta_jiffies = next_jiffies - last_jiffies; delta_jiffies = next_jiffies - last_jiffies;
if (rcu_delta_jiffies < delta_jiffies) {
next_jiffies = last_jiffies + rcu_delta_jiffies;
delta_jiffies = rcu_delta_jiffies;
}
} }
/* /*
* Do not stop the tick, if we are only one off * Do not stop the tick, if we are only one off