radix-tree: split node->path into offset and height
Neither piece of information we're storing in node->path can be larger than 64, so store each in its own unsigned char instead of shifting and masking to store them both in an unsigned int. Signed-off-by: Matthew Wilcox <willy@linux.intel.com> Reviewed-by: Ross Zwisler <ross.zwisler@linux.intel.com> Cc: Konstantin Khlebnikov <koct9i@gmail.com> Cc: Kirill Shutemov <kirill.shutemov@linux.intel.com> Cc: Jan Kara <jack@suse.com> Cc: Neil Brown <neilb@suse.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
2fcd9005cc
commit
0c7fa0a841
|
@ -84,16 +84,13 @@ static inline int radix_tree_is_indirect_ptr(void *ptr)
|
||||||
#define RADIX_TREE_MAX_PATH (DIV_ROUND_UP(RADIX_TREE_INDEX_BITS, \
|
#define RADIX_TREE_MAX_PATH (DIV_ROUND_UP(RADIX_TREE_INDEX_BITS, \
|
||||||
RADIX_TREE_MAP_SHIFT))
|
RADIX_TREE_MAP_SHIFT))
|
||||||
|
|
||||||
/* Height component in node->path */
|
|
||||||
#define RADIX_TREE_HEIGHT_SHIFT (RADIX_TREE_MAX_PATH + 1)
|
|
||||||
#define RADIX_TREE_HEIGHT_MASK ((1UL << RADIX_TREE_HEIGHT_SHIFT) - 1)
|
|
||||||
|
|
||||||
/* Internally used bits of node->count */
|
/* Internally used bits of node->count */
|
||||||
#define RADIX_TREE_COUNT_SHIFT (RADIX_TREE_MAP_SHIFT + 1)
|
#define RADIX_TREE_COUNT_SHIFT (RADIX_TREE_MAP_SHIFT + 1)
|
||||||
#define RADIX_TREE_COUNT_MASK ((1UL << RADIX_TREE_COUNT_SHIFT) - 1)
|
#define RADIX_TREE_COUNT_MASK ((1UL << RADIX_TREE_COUNT_SHIFT) - 1)
|
||||||
|
|
||||||
struct radix_tree_node {
|
struct radix_tree_node {
|
||||||
unsigned int path; /* Offset in parent & height from the bottom */
|
unsigned char height; /* From the bottom */
|
||||||
|
unsigned char offset; /* Slot offset in parent */
|
||||||
unsigned int count;
|
unsigned int count;
|
||||||
union {
|
union {
|
||||||
struct {
|
struct {
|
||||||
|
|
|
@ -218,15 +218,15 @@ radix_tree_find_next_bit(const unsigned long *addr,
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifndef __KERNEL__
|
#ifndef __KERNEL__
|
||||||
static void dump_node(struct radix_tree_node *node, unsigned offset,
|
static void dump_node(struct radix_tree_node *node,
|
||||||
unsigned shift, unsigned long index)
|
unsigned shift, unsigned long index)
|
||||||
{
|
{
|
||||||
unsigned long i;
|
unsigned long i;
|
||||||
|
|
||||||
pr_debug("radix node: %p offset %d tags %lx %lx %lx path %x count %d parent %p\n",
|
pr_debug("radix node: %p offset %d tags %lx %lx %lx height %d count %d parent %p\n",
|
||||||
node, offset,
|
node, node->offset,
|
||||||
node->tags[0][0], node->tags[1][0], node->tags[2][0],
|
node->tags[0][0], node->tags[1][0], node->tags[2][0],
|
||||||
node->path, node->count, node->parent);
|
node->height, node->count, node->parent);
|
||||||
|
|
||||||
for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) {
|
for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) {
|
||||||
unsigned long first = index | (i << shift);
|
unsigned long first = index | (i << shift);
|
||||||
|
@ -243,7 +243,7 @@ static void dump_node(struct radix_tree_node *node, unsigned offset,
|
||||||
pr_debug("radix entry %p offset %ld indices %ld-%ld\n",
|
pr_debug("radix entry %p offset %ld indices %ld-%ld\n",
|
||||||
entry, i, first, last);
|
entry, i, first, last);
|
||||||
} else {
|
} else {
|
||||||
dump_node(indirect_to_ptr(entry), i,
|
dump_node(indirect_to_ptr(entry),
|
||||||
shift - RADIX_TREE_MAP_SHIFT, first);
|
shift - RADIX_TREE_MAP_SHIFT, first);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -257,7 +257,7 @@ static void radix_tree_dump(struct radix_tree_root *root)
|
||||||
root->gfp_mask >> __GFP_BITS_SHIFT);
|
root->gfp_mask >> __GFP_BITS_SHIFT);
|
||||||
if (!radix_tree_is_indirect_ptr(root->rnode))
|
if (!radix_tree_is_indirect_ptr(root->rnode))
|
||||||
return;
|
return;
|
||||||
dump_node(indirect_to_ptr(root->rnode), 0,
|
dump_node(indirect_to_ptr(root->rnode),
|
||||||
(root->height - 1) * RADIX_TREE_MAP_SHIFT, 0);
|
(root->height - 1) * RADIX_TREE_MAP_SHIFT, 0);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
@ -421,7 +421,7 @@ static inline unsigned long radix_tree_maxindex(unsigned int height)
|
||||||
|
|
||||||
static inline unsigned long node_maxindex(struct radix_tree_node *node)
|
static inline unsigned long node_maxindex(struct radix_tree_node *node)
|
||||||
{
|
{
|
||||||
return radix_tree_maxindex(node->path & RADIX_TREE_HEIGHT_MASK);
|
return radix_tree_maxindex(node->height);
|
||||||
}
|
}
|
||||||
|
|
||||||
static unsigned radix_tree_load_root(struct radix_tree_root *root,
|
static unsigned radix_tree_load_root(struct radix_tree_root *root,
|
||||||
|
@ -434,8 +434,7 @@ static unsigned radix_tree_load_root(struct radix_tree_root *root,
|
||||||
if (likely(radix_tree_is_indirect_ptr(node))) {
|
if (likely(radix_tree_is_indirect_ptr(node))) {
|
||||||
node = indirect_to_ptr(node);
|
node = indirect_to_ptr(node);
|
||||||
*maxindex = node_maxindex(node);
|
*maxindex = node_maxindex(node);
|
||||||
return (node->path & RADIX_TREE_HEIGHT_MASK) *
|
return node->height * RADIX_TREE_MAP_SHIFT;
|
||||||
RADIX_TREE_MAP_SHIFT;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
*maxindex = 0;
|
*maxindex = 0;
|
||||||
|
@ -476,9 +475,10 @@ static int radix_tree_extend(struct radix_tree_root *root,
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Increase the height. */
|
/* Increase the height. */
|
||||||
newheight = root->height+1;
|
newheight = root->height + 1;
|
||||||
BUG_ON(newheight & ~RADIX_TREE_HEIGHT_MASK);
|
BUG_ON(newheight > BITS_PER_LONG);
|
||||||
node->path = newheight;
|
node->height = newheight;
|
||||||
|
node->offset = 0;
|
||||||
node->count = 1;
|
node->count = 1;
|
||||||
node->parent = NULL;
|
node->parent = NULL;
|
||||||
slot = root->rnode;
|
slot = root->rnode;
|
||||||
|
@ -546,13 +546,13 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index,
|
||||||
slot = radix_tree_node_alloc(root);
|
slot = radix_tree_node_alloc(root);
|
||||||
if (!slot)
|
if (!slot)
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
slot->path = height;
|
slot->height = height;
|
||||||
|
slot->offset = offset;
|
||||||
slot->parent = node;
|
slot->parent = node;
|
||||||
if (node) {
|
if (node) {
|
||||||
rcu_assign_pointer(node->slots[offset],
|
rcu_assign_pointer(node->slots[offset],
|
||||||
ptr_to_indirect(slot));
|
ptr_to_indirect(slot));
|
||||||
node->count++;
|
node->count++;
|
||||||
slot->path |= offset << RADIX_TREE_HEIGHT_SHIFT;
|
|
||||||
} else
|
} else
|
||||||
rcu_assign_pointer(root->rnode,
|
rcu_assign_pointer(root->rnode,
|
||||||
ptr_to_indirect(slot));
|
ptr_to_indirect(slot));
|
||||||
|
@ -1319,11 +1319,10 @@ struct locate_info {
|
||||||
static unsigned long __locate(struct radix_tree_node *slot, void *item,
|
static unsigned long __locate(struct radix_tree_node *slot, void *item,
|
||||||
unsigned long index, struct locate_info *info)
|
unsigned long index, struct locate_info *info)
|
||||||
{
|
{
|
||||||
unsigned int shift, height;
|
unsigned int shift;
|
||||||
unsigned long i;
|
unsigned long i;
|
||||||
|
|
||||||
height = slot->path & RADIX_TREE_HEIGHT_MASK;
|
shift = slot->height * RADIX_TREE_MAP_SHIFT;
|
||||||
shift = height * RADIX_TREE_MAP_SHIFT;
|
|
||||||
|
|
||||||
do {
|
do {
|
||||||
shift -= RADIX_TREE_MAP_SHIFT;
|
shift -= RADIX_TREE_MAP_SHIFT;
|
||||||
|
@ -1508,10 +1507,7 @@ bool __radix_tree_delete_node(struct radix_tree_root *root,
|
||||||
|
|
||||||
parent = node->parent;
|
parent = node->parent;
|
||||||
if (parent) {
|
if (parent) {
|
||||||
unsigned int offset;
|
parent->slots[node->offset] = NULL;
|
||||||
|
|
||||||
offset = node->path >> RADIX_TREE_HEIGHT_SHIFT;
|
|
||||||
parent->slots[offset] = NULL;
|
|
||||||
parent->count--;
|
parent->count--;
|
||||||
} else {
|
} else {
|
||||||
root_tag_clear_all(root);
|
root_tag_clear_all(root);
|
||||||
|
|
Loading…
Reference in a new issue