list: Add hlist_unhashed_lockless()
We would like to use hlist_unhashed() from timer_pending(), which runs without protection of a lock. Note that other callers might also want to use this variant. Instead of forcing a READ_ONCE() for all hlist_unhashed() callers, add a new helper with an explicit _lockless suffix in the name to better document what is going on. Also add various WRITE_ONCE() in __hlist_del(), hlist_add_head() and hlist_add_before()/hlist_add_behind() to pair with the READ_ONCE(). Signed-off-by: Eric Dumazet <edumazet@google.com> Cc: Thomas Gleixner <tglx@linutronix.de> [ paulmck: Also add WRITE_ONCE() to rculist.h. ] Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
parent
f452ee096d
commit
c54a274449
|
@ -749,6 +749,16 @@ static inline int hlist_unhashed(const struct hlist_node *h)
|
|||
return !h->pprev;
|
||||
}
|
||||
|
||||
/* This variant of hlist_unhashed() must be used in lockless contexts
|
||||
* to avoid potential load-tearing.
|
||||
* The READ_ONCE() is paired with the various WRITE_ONCE() in hlist
|
||||
* helpers that are defined below.
|
||||
*/
|
||||
static inline int hlist_unhashed_lockless(const struct hlist_node *h)
|
||||
{
|
||||
return !READ_ONCE(h->pprev);
|
||||
}
|
||||
|
||||
static inline int hlist_empty(const struct hlist_head *h)
|
||||
{
|
||||
return !READ_ONCE(h->first);
|
||||
|
@ -761,7 +771,7 @@ static inline void __hlist_del(struct hlist_node *n)
|
|||
|
||||
WRITE_ONCE(*pprev, next);
|
||||
if (next)
|
||||
next->pprev = pprev;
|
||||
WRITE_ONCE(next->pprev, pprev);
|
||||
}
|
||||
|
||||
static inline void hlist_del(struct hlist_node *n)
|
||||
|
@ -782,32 +792,32 @@ static inline void hlist_del_init(struct hlist_node *n)
|
|||
static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
|
||||
{
|
||||
struct hlist_node *first = h->first;
|
||||
n->next = first;
|
||||
WRITE_ONCE(n->next, first);
|
||||
if (first)
|
||||
first->pprev = &n->next;
|
||||
WRITE_ONCE(first->pprev, &n->next);
|
||||
WRITE_ONCE(h->first, n);
|
||||
n->pprev = &h->first;
|
||||
WRITE_ONCE(n->pprev, &h->first);
|
||||
}
|
||||
|
||||
/* next must be != NULL */
|
||||
static inline void hlist_add_before(struct hlist_node *n,
|
||||
struct hlist_node *next)
|
||||
{
|
||||
n->pprev = next->pprev;
|
||||
n->next = next;
|
||||
next->pprev = &n->next;
|
||||
WRITE_ONCE(n->pprev, next->pprev);
|
||||
WRITE_ONCE(n->next, next);
|
||||
WRITE_ONCE(next->pprev, &n->next);
|
||||
WRITE_ONCE(*(n->pprev), n);
|
||||
}
|
||||
|
||||
static inline void hlist_add_behind(struct hlist_node *n,
|
||||
struct hlist_node *prev)
|
||||
{
|
||||
n->next = prev->next;
|
||||
prev->next = n;
|
||||
n->pprev = &prev->next;
|
||||
WRITE_ONCE(n->next, prev->next);
|
||||
WRITE_ONCE(prev->next, n);
|
||||
WRITE_ONCE(n->pprev, &prev->next);
|
||||
|
||||
if (n->next)
|
||||
n->next->pprev = &n->next;
|
||||
WRITE_ONCE(n->next->pprev, &n->next);
|
||||
}
|
||||
|
||||
/* after that we'll appear to be on some hlist and hlist_del will work */
|
||||
|
|
|
@ -173,7 +173,7 @@ static inline void hlist_del_init_rcu(struct hlist_node *n)
|
|||
{
|
||||
if (!hlist_unhashed(n)) {
|
||||
__hlist_del(n);
|
||||
n->pprev = NULL;
|
||||
WRITE_ONCE(n->pprev, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -473,7 +473,7 @@ static inline void list_splice_tail_init_rcu(struct list_head *list,
|
|||
static inline void hlist_del_rcu(struct hlist_node *n)
|
||||
{
|
||||
__hlist_del(n);
|
||||
n->pprev = LIST_POISON2;
|
||||
WRITE_ONCE(n->pprev, LIST_POISON2);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -489,11 +489,11 @@ static inline void hlist_replace_rcu(struct hlist_node *old,
|
|||
struct hlist_node *next = old->next;
|
||||
|
||||
new->next = next;
|
||||
new->pprev = old->pprev;
|
||||
WRITE_ONCE(new->pprev, old->pprev);
|
||||
rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);
|
||||
if (next)
|
||||
new->next->pprev = &new->next;
|
||||
old->pprev = LIST_POISON2;
|
||||
WRITE_ONCE(new->next->pprev, &new->next);
|
||||
WRITE_ONCE(old->pprev, LIST_POISON2);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -528,10 +528,10 @@ static inline void hlist_add_head_rcu(struct hlist_node *n,
|
|||
struct hlist_node *first = h->first;
|
||||
|
||||
n->next = first;
|
||||
n->pprev = &h->first;
|
||||
WRITE_ONCE(n->pprev, &h->first);
|
||||
rcu_assign_pointer(hlist_first_rcu(h), n);
|
||||
if (first)
|
||||
first->pprev = &n->next;
|
||||
WRITE_ONCE(first->pprev, &n->next);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -564,7 +564,7 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n,
|
|||
|
||||
if (last) {
|
||||
n->next = last->next;
|
||||
n->pprev = &last->next;
|
||||
WRITE_ONCE(n->pprev, &last->next);
|
||||
rcu_assign_pointer(hlist_next_rcu(last), n);
|
||||
} else {
|
||||
hlist_add_head_rcu(n, h);
|
||||
|
@ -592,10 +592,10 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n,
|
|||
static inline void hlist_add_before_rcu(struct hlist_node *n,
|
||||
struct hlist_node *next)
|
||||
{
|
||||
n->pprev = next->pprev;
|
||||
WRITE_ONCE(n->pprev, next->pprev);
|
||||
n->next = next;
|
||||
rcu_assign_pointer(hlist_pprev_rcu(n), n);
|
||||
next->pprev = &n->next;
|
||||
WRITE_ONCE(next->pprev, &n->next);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -620,10 +620,10 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,
|
|||
struct hlist_node *prev)
|
||||
{
|
||||
n->next = prev->next;
|
||||
n->pprev = &prev->next;
|
||||
WRITE_ONCE(n->pprev, &prev->next);
|
||||
rcu_assign_pointer(hlist_next_rcu(prev), n);
|
||||
if (n->next)
|
||||
n->next->pprev = &n->next;
|
||||
WRITE_ONCE(n->next->pprev, &n->next);
|
||||
}
|
||||
|
||||
#define __hlist_for_each_rcu(pos, head) \
|
||||
|
|
Loading…
Reference in a new issue