9dd8fb16c3
When running preemptible RCU, if a task exits in an RCU read-side critical section having blocked within that same RCU read-side critical section, the task must be removed from the list of tasks blocking a grace period (perhaps the current grace period, perhaps the next grace period, depending on timing). The exit() path invokes exit_rcu() to do this cleanup. However, the current implementation of exit_rcu() needlessly does the cleanup even if the task did not block within the current RCU read-side critical section, which wastes time and needlessly increases the size of the state space. Fix this by only doing the cleanup if the current task is actually on the list of tasks blocking some grace period. While we are at it, consolidate the two identical exit_rcu() functions into a single function. Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Tested-by: Linus Torvalds <torvalds@linux-foundation.org> Conflicts: kernel/rcupdate.c
100 lines
3.5 KiB
C
100 lines
3.5 KiB
C
/*
|
|
* Read-Copy Update mechanism for mutual exclusion (tree-based version)
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, write to the Free Software
|
|
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
*
|
|
* Copyright IBM Corporation, 2008
|
|
*
|
|
* Author: Dipankar Sarma <dipankar@in.ibm.com>
|
|
* Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical algorithm
|
|
*
|
|
* Based on the original work by Paul McKenney <paulmck@us.ibm.com>
|
|
* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
|
|
*
|
|
* For detailed explanation of Read-Copy Update mechanism see -
|
|
* Documentation/RCU
|
|
*/
|
|
|
|
#ifndef __LINUX_RCUTREE_H
|
|
#define __LINUX_RCUTREE_H
|
|
|
|
extern void rcu_init(void);
|
|
extern void rcu_note_context_switch(int cpu);
|
|
extern int rcu_needs_cpu(int cpu);
|
|
extern void rcu_cpu_stall_reset(void);
|
|
|
|
/*
|
|
* Note a virtualization-based context switch. This is simply a
|
|
* wrapper around rcu_note_context_switch(), which allows TINY_RCU
|
|
* to save a few bytes.
|
|
*/
|
|
static inline void rcu_virt_note_context_switch(int cpu)
|
|
{
|
|
rcu_note_context_switch(cpu);
|
|
}
|
|
|
|
extern void synchronize_rcu_bh(void);
|
|
extern void synchronize_sched_expedited(void);
|
|
extern void synchronize_rcu_expedited(void);
|
|
|
|
void kfree_call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
|
|
|
|
/**
|
|
* synchronize_rcu_bh_expedited - Brute-force RCU-bh grace period
|
|
*
|
|
* Wait for an RCU-bh grace period to elapse, but use a "big hammer"
|
|
* approach to force the grace period to end quickly. This consumes
|
|
* significant time on all CPUs and is unfriendly to real-time workloads,
|
|
* so is thus not recommended for any sort of common-case code. In fact,
|
|
* if you are using synchronize_rcu_bh_expedited() in a loop, please
|
|
* restructure your code to batch your updates, and then use a single
|
|
* synchronize_rcu_bh() instead.
|
|
*
|
|
* Note that it is illegal to call this function while holding any lock
|
|
* that is acquired by a CPU-hotplug notifier. And yes, it is also illegal
|
|
* to call this function from a CPU-hotplug notifier. Failing to observe
|
|
* these restriction will result in deadlock.
|
|
*/
|
|
static inline void synchronize_rcu_bh_expedited(void)
|
|
{
|
|
synchronize_sched_expedited();
|
|
}
|
|
|
|
extern void rcu_barrier(void);
|
|
extern void rcu_barrier_bh(void);
|
|
extern void rcu_barrier_sched(void);
|
|
|
|
extern unsigned long rcutorture_testseq;
|
|
extern unsigned long rcutorture_vernum;
|
|
extern long rcu_batches_completed(void);
|
|
extern long rcu_batches_completed_bh(void);
|
|
extern long rcu_batches_completed_sched(void);
|
|
|
|
extern void rcu_force_quiescent_state(void);
|
|
extern void rcu_bh_force_quiescent_state(void);
|
|
extern void rcu_sched_force_quiescent_state(void);
|
|
|
|
/* A context switch is a grace period for RCU-sched and RCU-bh. */
|
|
static inline int rcu_blocking_is_gp(void)
|
|
{
|
|
might_sleep(); /* Check for RCU read-side critical section. */
|
|
return num_online_cpus() == 1;
|
|
}
|
|
|
|
extern void rcu_scheduler_starting(void);
|
|
extern int rcu_scheduler_active __read_mostly;
|
|
|
|
#endif /* __LINUX_RCUTREE_H */
|