rcu: Streamline code produced by __rcu_read_unlock()
Given some common flag combinations, particularly -Os, gcc will inline rcu_read_unlock_special() despite its being in an unlikely() clause. Use noinline to prohibit this misoptimization. In addition, move the second barrier() in __rcu_read_unlock() so that it is not on the common-case code path. This will allow the compiler to generate better code for the common-case path through __rcu_read_unlock(). Suggested-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Acked-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
This commit is contained in:
parent
7765be2fec
commit
be0e1e21ef
|
@ -284,7 +284,7 @@ static struct list_head *rcu_next_node_entry(struct task_struct *t,
|
||||||
* notify RCU core processing or task having blocked during the RCU
|
* notify RCU core processing or task having blocked during the RCU
|
||||||
* read-side critical section.
|
* read-side critical section.
|
||||||
*/
|
*/
|
||||||
static void rcu_read_unlock_special(struct task_struct *t)
|
static noinline void rcu_read_unlock_special(struct task_struct *t)
|
||||||
{
|
{
|
||||||
int empty;
|
int empty;
|
||||||
int empty_exp;
|
int empty_exp;
|
||||||
|
@ -391,11 +391,11 @@ void __rcu_read_unlock(void)
|
||||||
struct task_struct *t = current;
|
struct task_struct *t = current;
|
||||||
|
|
||||||
barrier(); /* needed if we ever invoke rcu_read_unlock in rcutree.c */
|
barrier(); /* needed if we ever invoke rcu_read_unlock in rcutree.c */
|
||||||
--t->rcu_read_lock_nesting;
|
if (--t->rcu_read_lock_nesting == 0) {
|
||||||
barrier(); /* decrement before load of ->rcu_read_unlock_special */
|
barrier(); /* decr before ->rcu_read_unlock_special load */
|
||||||
if (t->rcu_read_lock_nesting == 0 &&
|
if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
|
||||||
unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
|
rcu_read_unlock_special(t);
|
||||||
rcu_read_unlock_special(t);
|
}
|
||||||
#ifdef CONFIG_PROVE_LOCKING
|
#ifdef CONFIG_PROVE_LOCKING
|
||||||
WARN_ON_ONCE(ACCESS_ONCE(t->rcu_read_lock_nesting) < 0);
|
WARN_ON_ONCE(ACCESS_ONCE(t->rcu_read_lock_nesting) < 0);
|
||||||
#endif /* #ifdef CONFIG_PROVE_LOCKING */
|
#endif /* #ifdef CONFIG_PROVE_LOCKING */
|
||||||
|
|
Loading…
Reference in a new issue