rcu: Provide cond_resched_rcu_qs() to force quiescent states in long loops
RCU-tasks requires the occasional voluntary context switch from CPU-bound in-kernel tasks. In some cases, this requires instrumenting cond_resched(). However, there is some reluctance to countenance unconditionally instrumenting cond_resched() (see http://lwn.net/Articles/603252/), so this commit creates a separate cond_resched_rcu_qs() that may be used in place of cond_resched() in locations prone to long-duration in-kernel looping. This commit currently instruments only RCU-tasks. Future possibilities include also instrumenting RCU, RCU-bh, and RCU-sched in order to reduce IPI usage. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
parent
8315f42295
commit
bde6c3aa99
|
@ -367,7 +367,7 @@ static struct fdtable *close_files(struct files_struct * files)
|
||||||
struct file * file = xchg(&fdt->fd[i], NULL);
|
struct file * file = xchg(&fdt->fd[i], NULL);
|
||||||
if (file) {
|
if (file) {
|
||||||
filp_close(file, files);
|
filp_close(file, files);
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
i++;
|
i++;
|
||||||
|
|
|
@ -330,6 +330,19 @@ static inline void rcu_user_hooks_switch(struct task_struct *prev,
|
||||||
#define rcu_note_voluntary_context_switch(t) do { } while (0)
|
#define rcu_note_voluntary_context_switch(t) do { } while (0)
|
||||||
#endif /* #else #ifdef CONFIG_TASKS_RCU */
|
#endif /* #else #ifdef CONFIG_TASKS_RCU */
|
||||||
|
|
||||||
|
/**
|
||||||
|
* cond_resched_rcu_qs - Report potential quiescent states to RCU
|
||||||
|
*
|
||||||
|
* This macro resembles cond_resched(), except that it is defined to
|
||||||
|
* report potential quiescent states to RCU-tasks even if the cond_resched()
|
||||||
|
* machinery were to be shut off, as some advocate for PREEMPT kernels.
|
||||||
|
*/
|
||||||
|
#define cond_resched_rcu_qs() \
|
||||||
|
do { \
|
||||||
|
rcu_note_voluntary_context_switch(current); \
|
||||||
|
cond_resched(); \
|
||||||
|
} while (0)
|
||||||
|
|
||||||
#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP)
|
#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP)
|
||||||
bool __rcu_is_watching(void);
|
bool __rcu_is_watching(void);
|
||||||
#endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) */
|
#endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) */
|
||||||
|
|
|
@ -667,7 +667,7 @@ static int rcu_torture_boost(void *arg)
|
||||||
}
|
}
|
||||||
call_rcu_time = jiffies;
|
call_rcu_time = jiffies;
|
||||||
}
|
}
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
stutter_wait("rcu_torture_boost");
|
stutter_wait("rcu_torture_boost");
|
||||||
if (torture_must_stop())
|
if (torture_must_stop())
|
||||||
goto checkwait;
|
goto checkwait;
|
||||||
|
@ -1019,7 +1019,7 @@ rcu_torture_reader(void *arg)
|
||||||
__this_cpu_inc(rcu_torture_batch[completed]);
|
__this_cpu_inc(rcu_torture_batch[completed]);
|
||||||
preempt_enable();
|
preempt_enable();
|
||||||
cur_ops->readunlock(idx);
|
cur_ops->readunlock(idx);
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
stutter_wait("rcu_torture_reader");
|
stutter_wait("rcu_torture_reader");
|
||||||
} while (!torture_must_stop());
|
} while (!torture_must_stop());
|
||||||
if (irqreader && cur_ops->irq_capable) {
|
if (irqreader && cur_ops->irq_capable) {
|
||||||
|
|
|
@ -1647,7 +1647,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
|
||||||
rnp->level, rnp->grplo,
|
rnp->level, rnp->grplo,
|
||||||
rnp->grphi, rnp->qsmask);
|
rnp->grphi, rnp->qsmask);
|
||||||
raw_spin_unlock_irq(&rnp->lock);
|
raw_spin_unlock_irq(&rnp->lock);
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
}
|
}
|
||||||
|
|
||||||
mutex_unlock(&rsp->onoff_mutex);
|
mutex_unlock(&rsp->onoff_mutex);
|
||||||
|
@ -1736,7 +1736,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
||||||
/* smp_mb() provided by prior unlock-lock pair. */
|
/* smp_mb() provided by prior unlock-lock pair. */
|
||||||
nocb += rcu_future_gp_cleanup(rsp, rnp);
|
nocb += rcu_future_gp_cleanup(rsp, rnp);
|
||||||
raw_spin_unlock_irq(&rnp->lock);
|
raw_spin_unlock_irq(&rnp->lock);
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
}
|
}
|
||||||
rnp = rcu_get_root(rsp);
|
rnp = rcu_get_root(rsp);
|
||||||
raw_spin_lock_irq(&rnp->lock);
|
raw_spin_lock_irq(&rnp->lock);
|
||||||
|
@ -1785,7 +1785,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
|
||||||
/* Locking provides needed memory barrier. */
|
/* Locking provides needed memory barrier. */
|
||||||
if (rcu_gp_init(rsp))
|
if (rcu_gp_init(rsp))
|
||||||
break;
|
break;
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
flush_signals(current);
|
flush_signals(current);
|
||||||
trace_rcu_grace_period(rsp->name,
|
trace_rcu_grace_period(rsp->name,
|
||||||
ACCESS_ONCE(rsp->gpnum),
|
ACCESS_ONCE(rsp->gpnum),
|
||||||
|
@ -1828,10 +1828,10 @@ static int __noreturn rcu_gp_kthread(void *arg)
|
||||||
trace_rcu_grace_period(rsp->name,
|
trace_rcu_grace_period(rsp->name,
|
||||||
ACCESS_ONCE(rsp->gpnum),
|
ACCESS_ONCE(rsp->gpnum),
|
||||||
TPS("fqsend"));
|
TPS("fqsend"));
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
} else {
|
} else {
|
||||||
/* Deal with stray signal. */
|
/* Deal with stray signal. */
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
flush_signals(current);
|
flush_signals(current);
|
||||||
trace_rcu_grace_period(rsp->name,
|
trace_rcu_grace_period(rsp->name,
|
||||||
ACCESS_ONCE(rsp->gpnum),
|
ACCESS_ONCE(rsp->gpnum),
|
||||||
|
@ -2434,7 +2434,7 @@ static void force_qs_rnp(struct rcu_state *rsp,
|
||||||
struct rcu_node *rnp;
|
struct rcu_node *rnp;
|
||||||
|
|
||||||
rcu_for_each_leaf_node(rsp, rnp) {
|
rcu_for_each_leaf_node(rsp, rnp) {
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
mask = 0;
|
mask = 0;
|
||||||
raw_spin_lock_irqsave(&rnp->lock, flags);
|
raw_spin_lock_irqsave(&rnp->lock, flags);
|
||||||
smp_mb__after_unlock_lock();
|
smp_mb__after_unlock_lock();
|
||||||
|
|
|
@ -1848,7 +1848,7 @@ static int rcu_oom_notify(struct notifier_block *self,
|
||||||
get_online_cpus();
|
get_online_cpus();
|
||||||
for_each_online_cpu(cpu) {
|
for_each_online_cpu(cpu) {
|
||||||
smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
|
smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
}
|
}
|
||||||
put_online_cpus();
|
put_online_cpus();
|
||||||
|
|
||||||
|
|
|
@ -789,7 +789,7 @@ static int do_mlockall(int flags)
|
||||||
|
|
||||||
/* Ignore errors */
|
/* Ignore errors */
|
||||||
mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);
|
mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);
|
||||||
cond_resched();
|
cond_resched_rcu_qs();
|
||||||
}
|
}
|
||||||
out:
|
out:
|
||||||
return 0;
|
return 0;
|
||||||
|
|
Loading…
Reference in a new issue