![Michel Lespinasse](/assets/img/avatar_default.png)
- Use the newly introduced rb_set_parent_color() function to flip the color of nodes whose parent is already known. - Optimize rb_parent() when the node is known to be red - there is no need to mask out the color in that case. - Flipping gparent's color to red requires us to fetch its rb_parent_color field, so we can reuse it as the parent value for the next loop iteration. - Do not use __rb_rotate_left() and __rb_rotate_right() to handle tree rotations: we already have pointers to all relevant nodes, and know their colors (either because we want to adjust it, or because we've tested it, or we can deduce it as black due to the node proximity to a known red node). So we can generate more efficient code by making use of the node pointers we already have, and setting both the parent and color attributes for nodes all at once. Also in Case 2, some node attributes don't have to be set because we know another tree rotation (Case 3) will always follow and override them. Signed-off-by: Michel Lespinasse <walken@google.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Acked-by: David Woodhouse <David.Woodhouse@intel.com> Cc: Rik van Riel <riel@redhat.com> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Daniel Santos <daniel.santos@pobox.com> Cc: Jens Axboe <axboe@kernel.dk> Cc: "Eric W. Biederman" <ebiederm@xmission.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
582 lines
14 KiB
C
582 lines
14 KiB
C
/*
|
|
Red Black Trees
|
|
(C) 1999 Andrea Arcangeli <andrea@suse.de>
|
|
(C) 2002 David Woodhouse <dwmw2@infradead.org>
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 2 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program; if not, write to the Free Software
|
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
linux/lib/rbtree.c
|
|
*/
|
|
|
|
#include <linux/rbtree.h>
|
|
#include <linux/export.h>
|
|
|
|
/*
|
|
* red-black trees properties: http://en.wikipedia.org/wiki/Rbtree
|
|
*
|
|
* 1) A node is either red or black
|
|
* 2) The root is black
|
|
* 3) All leaves (NULL) are black
|
|
* 4) Both children of every red node are black
|
|
* 5) Every simple path from root to leaves contains the same number
|
|
* of black nodes.
|
|
*
|
|
* 4 and 5 give the O(log n) guarantee, since 4 implies you cannot have two
|
|
* consecutive red nodes in a path and every red node is therefore followed by
|
|
* a black. So if B is the number of black nodes on every simple path (as per
|
|
* 5), then the longest possible path due to 4 is 2B.
|
|
*
|
|
* We shall indicate color with case, where black nodes are uppercase and red
|
|
* nodes will be lowercase.
|
|
*/
|
|
|
|
#define RB_RED 0
|
|
#define RB_BLACK 1
|
|
|
|
#define rb_color(r) ((r)->__rb_parent_color & 1)
|
|
#define rb_is_red(r) (!rb_color(r))
|
|
#define rb_is_black(r) rb_color(r)
|
|
#define rb_set_red(r) do { (r)->__rb_parent_color &= ~1; } while (0)
|
|
#define rb_set_black(r) do { (r)->__rb_parent_color |= 1; } while (0)
|
|
|
|
static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p)
|
|
{
|
|
rb->__rb_parent_color = rb_color(rb) | (unsigned long)p;
|
|
}
|
|
static inline void rb_set_color(struct rb_node *rb, int color)
|
|
{
|
|
rb->__rb_parent_color = (rb->__rb_parent_color & ~1) | color;
|
|
}
|
|
|
|
static inline void rb_set_parent_color(struct rb_node *rb,
|
|
struct rb_node *p, int color)
|
|
{
|
|
rb->__rb_parent_color = (unsigned long)p | color;
|
|
}
|
|
|
|
static inline struct rb_node *rb_red_parent(struct rb_node *red)
|
|
{
|
|
return (struct rb_node *)red->__rb_parent_color;
|
|
}
|
|
|
|
static void __rb_rotate_left(struct rb_node *node, struct rb_root *root)
|
|
{
|
|
struct rb_node *right = node->rb_right;
|
|
struct rb_node *parent = rb_parent(node);
|
|
|
|
if ((node->rb_right = right->rb_left))
|
|
rb_set_parent(right->rb_left, node);
|
|
right->rb_left = node;
|
|
|
|
rb_set_parent(right, parent);
|
|
|
|
if (parent)
|
|
{
|
|
if (node == parent->rb_left)
|
|
parent->rb_left = right;
|
|
else
|
|
parent->rb_right = right;
|
|
}
|
|
else
|
|
root->rb_node = right;
|
|
rb_set_parent(node, right);
|
|
}
|
|
|
|
static void __rb_rotate_right(struct rb_node *node, struct rb_root *root)
|
|
{
|
|
struct rb_node *left = node->rb_left;
|
|
struct rb_node *parent = rb_parent(node);
|
|
|
|
if ((node->rb_left = left->rb_right))
|
|
rb_set_parent(left->rb_right, node);
|
|
left->rb_right = node;
|
|
|
|
rb_set_parent(left, parent);
|
|
|
|
if (parent)
|
|
{
|
|
if (node == parent->rb_right)
|
|
parent->rb_right = left;
|
|
else
|
|
parent->rb_left = left;
|
|
}
|
|
else
|
|
root->rb_node = left;
|
|
rb_set_parent(node, left);
|
|
}
|
|
|
|
/*
|
|
* Helper function for rotations:
|
|
* - old's parent and color get assigned to new
|
|
* - old gets assigned new as a parent and 'color' as a color.
|
|
*/
|
|
static inline void
|
|
__rb_rotate_set_parents(struct rb_node *old, struct rb_node *new,
|
|
struct rb_root *root, int color)
|
|
{
|
|
struct rb_node *parent = rb_parent(old);
|
|
new->__rb_parent_color = old->__rb_parent_color;
|
|
rb_set_parent_color(old, new, color);
|
|
if (parent) {
|
|
if (parent->rb_left == old)
|
|
parent->rb_left = new;
|
|
else
|
|
parent->rb_right = new;
|
|
} else
|
|
root->rb_node = new;
|
|
}
|
|
|
|
void rb_insert_color(struct rb_node *node, struct rb_root *root)
|
|
{
|
|
struct rb_node *parent = rb_red_parent(node), *gparent, *tmp;
|
|
|
|
while (true) {
|
|
/*
|
|
* Loop invariant: node is red
|
|
*
|
|
* If there is a black parent, we are done.
|
|
* Otherwise, take some corrective action as we don't
|
|
* want a red root or two consecutive red nodes.
|
|
*/
|
|
if (!parent) {
|
|
rb_set_parent_color(node, NULL, RB_BLACK);
|
|
break;
|
|
} else if (rb_is_black(parent))
|
|
break;
|
|
|
|
gparent = rb_red_parent(parent);
|
|
|
|
if (parent == gparent->rb_left) {
|
|
tmp = gparent->rb_right;
|
|
if (tmp && rb_is_red(tmp)) {
|
|
/*
|
|
* Case 1 - color flips
|
|
*
|
|
* G g
|
|
* / \ / \
|
|
* p u --> P U
|
|
* / /
|
|
* n N
|
|
*
|
|
* However, since g's parent might be red, and
|
|
* 4) does not allow this, we need to recurse
|
|
* at g.
|
|
*/
|
|
rb_set_parent_color(tmp, gparent, RB_BLACK);
|
|
rb_set_parent_color(parent, gparent, RB_BLACK);
|
|
node = gparent;
|
|
parent = rb_parent(node);
|
|
rb_set_parent_color(node, parent, RB_RED);
|
|
continue;
|
|
}
|
|
|
|
if (parent->rb_right == node) {
|
|
/*
|
|
* Case 2 - left rotate at parent
|
|
*
|
|
* G G
|
|
* / \ / \
|
|
* p U --> n U
|
|
* \ /
|
|
* n p
|
|
*
|
|
* This still leaves us in violation of 4), the
|
|
* continuation into Case 3 will fix that.
|
|
*/
|
|
parent->rb_right = tmp = node->rb_left;
|
|
node->rb_left = parent;
|
|
if (tmp)
|
|
rb_set_parent_color(tmp, parent,
|
|
RB_BLACK);
|
|
rb_set_parent_color(parent, node, RB_RED);
|
|
parent = node;
|
|
}
|
|
|
|
/*
|
|
* Case 3 - right rotate at gparent
|
|
*
|
|
* G P
|
|
* / \ / \
|
|
* p U --> n g
|
|
* / \
|
|
* n U
|
|
*/
|
|
gparent->rb_left = tmp = parent->rb_right;
|
|
parent->rb_right = gparent;
|
|
if (tmp)
|
|
rb_set_parent_color(tmp, gparent, RB_BLACK);
|
|
__rb_rotate_set_parents(gparent, parent, root, RB_RED);
|
|
break;
|
|
} else {
|
|
tmp = gparent->rb_left;
|
|
if (tmp && rb_is_red(tmp)) {
|
|
/* Case 1 - color flips */
|
|
rb_set_parent_color(tmp, gparent, RB_BLACK);
|
|
rb_set_parent_color(parent, gparent, RB_BLACK);
|
|
node = gparent;
|
|
parent = rb_parent(node);
|
|
rb_set_parent_color(node, parent, RB_RED);
|
|
continue;
|
|
}
|
|
|
|
if (parent->rb_left == node) {
|
|
/* Case 2 - right rotate at parent */
|
|
parent->rb_left = tmp = node->rb_right;
|
|
node->rb_right = parent;
|
|
if (tmp)
|
|
rb_set_parent_color(tmp, parent,
|
|
RB_BLACK);
|
|
rb_set_parent_color(parent, node, RB_RED);
|
|
parent = node;
|
|
}
|
|
|
|
/* Case 3 - left rotate at gparent */
|
|
gparent->rb_right = tmp = parent->rb_left;
|
|
parent->rb_left = gparent;
|
|
if (tmp)
|
|
rb_set_parent_color(tmp, gparent, RB_BLACK);
|
|
__rb_rotate_set_parents(gparent, parent, root, RB_RED);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(rb_insert_color);
|
|
|
|
static void __rb_erase_color(struct rb_node *node, struct rb_node *parent,
|
|
struct rb_root *root)
|
|
{
|
|
struct rb_node *other;
|
|
|
|
while ((!node || rb_is_black(node)) && node != root->rb_node)
|
|
{
|
|
if (parent->rb_left == node)
|
|
{
|
|
other = parent->rb_right;
|
|
if (rb_is_red(other))
|
|
{
|
|
rb_set_black(other);
|
|
rb_set_red(parent);
|
|
__rb_rotate_left(parent, root);
|
|
other = parent->rb_right;
|
|
}
|
|
if ((!other->rb_left || rb_is_black(other->rb_left)) &&
|
|
(!other->rb_right || rb_is_black(other->rb_right)))
|
|
{
|
|
rb_set_red(other);
|
|
node = parent;
|
|
parent = rb_parent(node);
|
|
}
|
|
else
|
|
{
|
|
if (!other->rb_right || rb_is_black(other->rb_right))
|
|
{
|
|
rb_set_black(other->rb_left);
|
|
rb_set_red(other);
|
|
__rb_rotate_right(other, root);
|
|
other = parent->rb_right;
|
|
}
|
|
rb_set_color(other, rb_color(parent));
|
|
rb_set_black(parent);
|
|
rb_set_black(other->rb_right);
|
|
__rb_rotate_left(parent, root);
|
|
node = root->rb_node;
|
|
break;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
other = parent->rb_left;
|
|
if (rb_is_red(other))
|
|
{
|
|
rb_set_black(other);
|
|
rb_set_red(parent);
|
|
__rb_rotate_right(parent, root);
|
|
other = parent->rb_left;
|
|
}
|
|
if ((!other->rb_left || rb_is_black(other->rb_left)) &&
|
|
(!other->rb_right || rb_is_black(other->rb_right)))
|
|
{
|
|
rb_set_red(other);
|
|
node = parent;
|
|
parent = rb_parent(node);
|
|
}
|
|
else
|
|
{
|
|
if (!other->rb_left || rb_is_black(other->rb_left))
|
|
{
|
|
rb_set_black(other->rb_right);
|
|
rb_set_red(other);
|
|
__rb_rotate_left(other, root);
|
|
other = parent->rb_left;
|
|
}
|
|
rb_set_color(other, rb_color(parent));
|
|
rb_set_black(parent);
|
|
rb_set_black(other->rb_left);
|
|
__rb_rotate_right(parent, root);
|
|
node = root->rb_node;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if (node)
|
|
rb_set_black(node);
|
|
}
|
|
|
|
void rb_erase(struct rb_node *node, struct rb_root *root)
|
|
{
|
|
struct rb_node *child, *parent;
|
|
int color;
|
|
|
|
if (!node->rb_left)
|
|
child = node->rb_right;
|
|
else if (!node->rb_right)
|
|
child = node->rb_left;
|
|
else
|
|
{
|
|
struct rb_node *old = node, *left;
|
|
|
|
node = node->rb_right;
|
|
while ((left = node->rb_left) != NULL)
|
|
node = left;
|
|
|
|
if (rb_parent(old)) {
|
|
if (rb_parent(old)->rb_left == old)
|
|
rb_parent(old)->rb_left = node;
|
|
else
|
|
rb_parent(old)->rb_right = node;
|
|
} else
|
|
root->rb_node = node;
|
|
|
|
child = node->rb_right;
|
|
parent = rb_parent(node);
|
|
color = rb_color(node);
|
|
|
|
if (parent == old) {
|
|
parent = node;
|
|
} else {
|
|
if (child)
|
|
rb_set_parent(child, parent);
|
|
parent->rb_left = child;
|
|
|
|
node->rb_right = old->rb_right;
|
|
rb_set_parent(old->rb_right, node);
|
|
}
|
|
|
|
node->__rb_parent_color = old->__rb_parent_color;
|
|
node->rb_left = old->rb_left;
|
|
rb_set_parent(old->rb_left, node);
|
|
|
|
goto color;
|
|
}
|
|
|
|
parent = rb_parent(node);
|
|
color = rb_color(node);
|
|
|
|
if (child)
|
|
rb_set_parent(child, parent);
|
|
if (parent)
|
|
{
|
|
if (parent->rb_left == node)
|
|
parent->rb_left = child;
|
|
else
|
|
parent->rb_right = child;
|
|
}
|
|
else
|
|
root->rb_node = child;
|
|
|
|
color:
|
|
if (color == RB_BLACK)
|
|
__rb_erase_color(child, parent, root);
|
|
}
|
|
EXPORT_SYMBOL(rb_erase);
|
|
|
|
static void rb_augment_path(struct rb_node *node, rb_augment_f func, void *data)
|
|
{
|
|
struct rb_node *parent;
|
|
|
|
up:
|
|
func(node, data);
|
|
parent = rb_parent(node);
|
|
if (!parent)
|
|
return;
|
|
|
|
if (node == parent->rb_left && parent->rb_right)
|
|
func(parent->rb_right, data);
|
|
else if (parent->rb_left)
|
|
func(parent->rb_left, data);
|
|
|
|
node = parent;
|
|
goto up;
|
|
}
|
|
|
|
/*
|
|
* after inserting @node into the tree, update the tree to account for
|
|
* both the new entry and any damage done by rebalance
|
|
*/
|
|
void rb_augment_insert(struct rb_node *node, rb_augment_f func, void *data)
|
|
{
|
|
if (node->rb_left)
|
|
node = node->rb_left;
|
|
else if (node->rb_right)
|
|
node = node->rb_right;
|
|
|
|
rb_augment_path(node, func, data);
|
|
}
|
|
EXPORT_SYMBOL(rb_augment_insert);
|
|
|
|
/*
|
|
* before removing the node, find the deepest node on the rebalance path
|
|
* that will still be there after @node gets removed
|
|
*/
|
|
struct rb_node *rb_augment_erase_begin(struct rb_node *node)
|
|
{
|
|
struct rb_node *deepest;
|
|
|
|
if (!node->rb_right && !node->rb_left)
|
|
deepest = rb_parent(node);
|
|
else if (!node->rb_right)
|
|
deepest = node->rb_left;
|
|
else if (!node->rb_left)
|
|
deepest = node->rb_right;
|
|
else {
|
|
deepest = rb_next(node);
|
|
if (deepest->rb_right)
|
|
deepest = deepest->rb_right;
|
|
else if (rb_parent(deepest) != node)
|
|
deepest = rb_parent(deepest);
|
|
}
|
|
|
|
return deepest;
|
|
}
|
|
EXPORT_SYMBOL(rb_augment_erase_begin);
|
|
|
|
/*
|
|
* after removal, update the tree to account for the removed entry
|
|
* and any rebalance damage.
|
|
*/
|
|
void rb_augment_erase_end(struct rb_node *node, rb_augment_f func, void *data)
|
|
{
|
|
if (node)
|
|
rb_augment_path(node, func, data);
|
|
}
|
|
EXPORT_SYMBOL(rb_augment_erase_end);
|
|
|
|
/*
|
|
* This function returns the first node (in sort order) of the tree.
|
|
*/
|
|
struct rb_node *rb_first(const struct rb_root *root)
|
|
{
|
|
struct rb_node *n;
|
|
|
|
n = root->rb_node;
|
|
if (!n)
|
|
return NULL;
|
|
while (n->rb_left)
|
|
n = n->rb_left;
|
|
return n;
|
|
}
|
|
EXPORT_SYMBOL(rb_first);
|
|
|
|
struct rb_node *rb_last(const struct rb_root *root)
|
|
{
|
|
struct rb_node *n;
|
|
|
|
n = root->rb_node;
|
|
if (!n)
|
|
return NULL;
|
|
while (n->rb_right)
|
|
n = n->rb_right;
|
|
return n;
|
|
}
|
|
EXPORT_SYMBOL(rb_last);
|
|
|
|
struct rb_node *rb_next(const struct rb_node *node)
|
|
{
|
|
struct rb_node *parent;
|
|
|
|
if (RB_EMPTY_NODE(node))
|
|
return NULL;
|
|
|
|
/* If we have a right-hand child, go down and then left as far
|
|
as we can. */
|
|
if (node->rb_right) {
|
|
node = node->rb_right;
|
|
while (node->rb_left)
|
|
node=node->rb_left;
|
|
return (struct rb_node *)node;
|
|
}
|
|
|
|
/* No right-hand children. Everything down and left is
|
|
smaller than us, so any 'next' node must be in the general
|
|
direction of our parent. Go up the tree; any time the
|
|
ancestor is a right-hand child of its parent, keep going
|
|
up. First time it's a left-hand child of its parent, said
|
|
parent is our 'next' node. */
|
|
while ((parent = rb_parent(node)) && node == parent->rb_right)
|
|
node = parent;
|
|
|
|
return parent;
|
|
}
|
|
EXPORT_SYMBOL(rb_next);
|
|
|
|
struct rb_node *rb_prev(const struct rb_node *node)
|
|
{
|
|
struct rb_node *parent;
|
|
|
|
if (RB_EMPTY_NODE(node))
|
|
return NULL;
|
|
|
|
/* If we have a left-hand child, go down and then right as far
|
|
as we can. */
|
|
if (node->rb_left) {
|
|
node = node->rb_left;
|
|
while (node->rb_right)
|
|
node=node->rb_right;
|
|
return (struct rb_node *)node;
|
|
}
|
|
|
|
/* No left-hand children. Go up till we find an ancestor which
|
|
is a right-hand child of its parent */
|
|
while ((parent = rb_parent(node)) && node == parent->rb_left)
|
|
node = parent;
|
|
|
|
return parent;
|
|
}
|
|
EXPORT_SYMBOL(rb_prev);
|
|
|
|
void rb_replace_node(struct rb_node *victim, struct rb_node *new,
|
|
struct rb_root *root)
|
|
{
|
|
struct rb_node *parent = rb_parent(victim);
|
|
|
|
/* Set the surrounding nodes to point to the replacement */
|
|
if (parent) {
|
|
if (victim == parent->rb_left)
|
|
parent->rb_left = new;
|
|
else
|
|
parent->rb_right = new;
|
|
} else {
|
|
root->rb_node = new;
|
|
}
|
|
if (victim->rb_left)
|
|
rb_set_parent(victim->rb_left, new);
|
|
if (victim->rb_right)
|
|
rb_set_parent(victim->rb_right, new);
|
|
|
|
/* Copy the pointers/colour from the victim to the replacement */
|
|
*new = *victim;
|
|
}
|
|
EXPORT_SYMBOL(rb_replace_node);
|