XArray updates for 5.5
Primarily bugfixes, mostly around handling index wrap-around correctly. A couple of doc fixes and adding missing APIs. -----BEGIN PGP SIGNATURE----- iQEzBAABCgAdFiEEejHryeLBw/spnjHrDpNsjXcpgj4FAl4pJv4ACgkQDpNsjXcp gj5OXgf/Q52Am0tLS6iC7XtcojjidnSFzpAqn9OGo7zuTsP+QURsiZAXJpjvUF8y kGIcD9R/tobEADkyDnj8D64yO+CqT1PlkaYvO5R+nYx/WTtmvr3/O/7Y4/ZTAwcf tTvj3yXlKbsA+9YJa34t43Ul4EDCOrQTz+oIZUQsiaRNzIdmCCguLg5gOkIhoXt6 wJwp2Fc3YN6McBBpZH2z0GRwdJWPcixtKH0RNhOyGUiWBhkWYFXNvNEdxKJmTXdK qqkoW9So8IkyyoXdW8ZpQqCCk9tPRv9+7adDFVwl/T8q/RBrHNtcyQqJL3EsScsi 9IqCrKldzRenk0v6lG+mgJEuTvYs8A== =7cpi -----END PGP SIGNATURE----- Merge tag 'xarray-5.5' of git://git.infradead.org/users/willy/linux-dax Pull XArray fixes from Matthew Wilcox: "Primarily bugfixes, mostly around handling index wrap-around correctly. A couple of doc fixes and adding missing APIs. I had an oops live on stage at linux.conf.au this year, and it turned out to be a bug in xas_find() which I can't prove isn't triggerable in the current codebase. Then in looking for the bug, I spotted two more bugs. The bots have had a few days to chew on this with no problems reported, and it passes the test-suite (which now has more tests to make sure these problems don't come back)" * tag 'xarray-5.5' of git://git.infradead.org/users/willy/linux-dax: XArray: Add xa_for_each_range XArray: Fix xas_find returning too many entries XArray: Fix xa_find_after with multi-index entries XArray: Fix infinite loop with entry at ULONG_MAX XArray: Add wrappers for nested spinlocks XArray: Improve documentation of search marks XArray: Fix xas_pause at ULONG_MAXalistair/sunxi64-5.5-dsi
Linus Torvalds
commit
4703d91199
|
|
@ -25,10 +25,6 @@ good performance with large indices. If your index can be larger than
|
|||
``ULONG_MAX`` then the XArray is not the data type for you. The most
|
||||
important user of the XArray is the page cache.
|
||||
|
||||
Each non-``NULL`` entry in the array has three bits associated with
|
||||
it called marks. Each mark may be set or cleared independently of
|
||||
the others. You can iterate over entries which are marked.
|
||||
|
||||
Normal pointers may be stored in the XArray directly. They must be 4-byte
|
||||
aligned, which is true for any pointer returned from kmalloc() and
|
||||
alloc_page(). It isn't true for arbitrary user-space pointers,
|
||||
|
|
@ -41,12 +37,11 @@ When you retrieve an entry from the XArray, you can check whether it is
|
|||
a value entry by calling xa_is_value(), and convert it back to
|
||||
an integer by calling xa_to_value().
|
||||
|
||||
Some users want to store tagged pointers instead of using the marks
|
||||
described above. They can call xa_tag_pointer() to create an
|
||||
entry with a tag, xa_untag_pointer() to turn a tagged entry
|
||||
back into an untagged pointer and xa_pointer_tag() to retrieve
|
||||
the tag of an entry. Tagged pointers use the same bits that are used
|
||||
to distinguish value entries from normal pointers, so each user must
|
||||
Some users want to tag the pointers they store in the XArray. You can
|
||||
call xa_tag_pointer() to create an entry with a tag, xa_untag_pointer()
|
||||
to turn a tagged entry back into an untagged pointer and xa_pointer_tag()
|
||||
to retrieve the tag of an entry. Tagged pointers use the same bits that
|
||||
are used to distinguish value entries from normal pointers, so you must
|
||||
decide whether they want to store value entries or tagged pointers in
|
||||
any particular XArray.
|
||||
|
||||
|
|
@ -56,10 +51,9 @@ conflict with value entries or internal entries.
|
|||
An unusual feature of the XArray is the ability to create entries which
|
||||
occupy a range of indices. Once stored to, looking up any index in
|
||||
the range will return the same entry as looking up any other index in
|
||||
the range. Setting a mark on one index will set it on all of them.
|
||||
Storing to any index will store to all of them. Multi-index entries can
|
||||
be explicitly split into smaller entries, or storing ``NULL`` into any
|
||||
entry will cause the XArray to forget about the range.
|
||||
the range. Storing to any index will store to all of them. Multi-index
|
||||
entries can be explicitly split into smaller entries, or storing ``NULL``
|
||||
into any entry will cause the XArray to forget about the range.
|
||||
|
||||
Normal API
|
||||
==========
|
||||
|
|
@ -87,17 +81,11 @@ If you want to only store a new entry to an index if the current entry
|
|||
at that index is ``NULL``, you can use xa_insert() which
|
||||
returns ``-EBUSY`` if the entry is not empty.
|
||||
|
||||
You can enquire whether a mark is set on an entry by using
|
||||
xa_get_mark(). If the entry is not ``NULL``, you can set a mark
|
||||
on it by using xa_set_mark() and remove the mark from an entry by
|
||||
calling xa_clear_mark(). You can ask whether any entry in the
|
||||
XArray has a particular mark set by calling xa_marked().
|
||||
|
||||
You can copy entries out of the XArray into a plain array by calling
|
||||
xa_extract(). Or you can iterate over the present entries in
|
||||
the XArray by calling xa_for_each(). You may prefer to use
|
||||
xa_find() or xa_find_after() to move to the next present
|
||||
entry in the XArray.
|
||||
xa_extract(). Or you can iterate over the present entries in the XArray
|
||||
by calling xa_for_each(), xa_for_each_start() or xa_for_each_range().
|
||||
You may prefer to use xa_find() or xa_find_after() to move to the next
|
||||
present entry in the XArray.
|
||||
|
||||
Calling xa_store_range() stores the same entry in a range
|
||||
of indices. If you do this, some of the other operations will behave
|
||||
|
|
@ -124,6 +112,31 @@ xa_destroy(). If the XArray entries are pointers, you may wish
|
|||
to free the entries first. You can do this by iterating over all present
|
||||
entries in the XArray using the xa_for_each() iterator.
|
||||
|
||||
Search Marks
|
||||
------------
|
||||
|
||||
Each entry in the array has three bits associated with it called marks.
|
||||
Each mark may be set or cleared independently of the others. You can
|
||||
iterate over marked entries by using the xa_for_each_marked() iterator.
|
||||
|
||||
You can enquire whether a mark is set on an entry by using
|
||||
xa_get_mark(). If the entry is not ``NULL``, you can set a mark on it
|
||||
by using xa_set_mark() and remove the mark from an entry by calling
|
||||
xa_clear_mark(). You can ask whether any entry in the XArray has a
|
||||
particular mark set by calling xa_marked(). Erasing an entry from the
|
||||
XArray causes all marks associated with that entry to be cleared.
|
||||
|
||||
Setting or clearing a mark on any index of a multi-index entry will
|
||||
affect all indices covered by that entry. Querying the mark on any
|
||||
index will return the same result.
|
||||
|
||||
There is no way to iterate over entries which are not marked; the data
|
||||
structure does not allow this to be implemented efficiently. There are
|
||||
not currently iterators to search for logical combinations of bits (eg
|
||||
iterate over all entries which have both ``XA_MARK_1`` and ``XA_MARK_2``
|
||||
set, or iterate over all entries which have ``XA_MARK_0`` or ``XA_MARK_2``
|
||||
set). It would be possible to add these if a user arises.
|
||||
|
||||
Allocating XArrays
|
||||
------------------
|
||||
|
||||
|
|
@ -180,6 +193,8 @@ No lock needed:
|
|||
Takes RCU read lock:
|
||||
* xa_load()
|
||||
* xa_for_each()
|
||||
* xa_for_each_start()
|
||||
* xa_for_each_range()
|
||||
* xa_find()
|
||||
* xa_find_after()
|
||||
* xa_extract()
|
||||
|
|
@ -419,10 +434,9 @@ you last processed. If you have interrupts disabled while iterating,
|
|||
then it is good manners to pause the iteration and reenable interrupts
|
||||
every ``XA_CHECK_SCHED`` entries.
|
||||
|
||||
The xas_get_mark(), xas_set_mark() and
|
||||
xas_clear_mark() functions require the xa_state cursor to have
|
||||
been moved to the appropriate location in the xarray; they will do
|
||||
nothing if you have called xas_pause() or xas_set()
|
||||
The xas_get_mark(), xas_set_mark() and xas_clear_mark() functions require
|
||||
the xa_state cursor to have been moved to the appropriate location in the
|
||||
XArray; they will do nothing if you have called xas_pause() or xas_set()
|
||||
immediately before.
|
||||
|
||||
You can call xas_set_update() to have a callback function
|
||||
|
|
|
|||
|
|
@ -416,6 +416,36 @@ static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark)
|
|||
return xa->xa_flags & XA_FLAGS_MARK(mark);
|
||||
}
|
||||
|
||||
/**
|
||||
* xa_for_each_range() - Iterate over a portion of an XArray.
|
||||
* @xa: XArray.
|
||||
* @index: Index of @entry.
|
||||
* @entry: Entry retrieved from array.
|
||||
* @start: First index to retrieve from array.
|
||||
* @last: Last index to retrieve from array.
|
||||
*
|
||||
* During the iteration, @entry will have the value of the entry stored
|
||||
* in @xa at @index. You may modify @index during the iteration if you
|
||||
* want to skip or reprocess indices. It is safe to modify the array
|
||||
* during the iteration. At the end of the iteration, @entry will be set
|
||||
* to NULL and @index will have a value less than or equal to max.
|
||||
*
|
||||
* xa_for_each_range() is O(n.log(n)) while xas_for_each() is O(n). You have
|
||||
* to handle your own locking with xas_for_each(), and if you have to unlock
|
||||
* after each iteration, it will also end up being O(n.log(n)).
|
||||
* xa_for_each_range() will spin if it hits a retry entry; if you intend to
|
||||
* see retry entries, you should use the xas_for_each() iterator instead.
|
||||
* The xas_for_each() iterator will expand into more inline code than
|
||||
* xa_for_each_range().
|
||||
*
|
||||
* Context: Any context. Takes and releases the RCU lock.
|
||||
*/
|
||||
#define xa_for_each_range(xa, index, entry, start, last) \
|
||||
for (index = start, \
|
||||
entry = xa_find(xa, &index, last, XA_PRESENT); \
|
||||
entry; \
|
||||
entry = xa_find_after(xa, &index, last, XA_PRESENT))
|
||||
|
||||
/**
|
||||
* xa_for_each_start() - Iterate over a portion of an XArray.
|
||||
* @xa: XArray.
|
||||
|
|
@ -439,11 +469,8 @@ static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark)
|
|||
*
|
||||
* Context: Any context. Takes and releases the RCU lock.
|
||||
*/
|
||||
#define xa_for_each_start(xa, index, entry, start) \
|
||||
for (index = start, \
|
||||
entry = xa_find(xa, &index, ULONG_MAX, XA_PRESENT); \
|
||||
entry; \
|
||||
entry = xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT))
|
||||
#define xa_for_each_start(xa, index, entry, start) \
|
||||
xa_for_each_range(xa, index, entry, start, ULONG_MAX)
|
||||
|
||||
/**
|
||||
* xa_for_each() - Iterate over present entries in an XArray.
|
||||
|
|
@ -508,6 +535,14 @@ static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark)
|
|||
spin_lock_irqsave(&(xa)->xa_lock, flags)
|
||||
#define xa_unlock_irqrestore(xa, flags) \
|
||||
spin_unlock_irqrestore(&(xa)->xa_lock, flags)
|
||||
#define xa_lock_nested(xa, subclass) \
|
||||
spin_lock_nested(&(xa)->xa_lock, subclass)
|
||||
#define xa_lock_bh_nested(xa, subclass) \
|
||||
spin_lock_bh_nested(&(xa)->xa_lock, subclass)
|
||||
#define xa_lock_irq_nested(xa, subclass) \
|
||||
spin_lock_irq_nested(&(xa)->xa_lock, subclass)
|
||||
#define xa_lock_irqsave_nested(xa, flags, subclass) \
|
||||
spin_lock_irqsave_nested(&(xa)->xa_lock, flags, subclass)
|
||||
|
||||
/*
|
||||
* Versions of the normal API which require the caller to hold the
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
/*
|
||||
* test_xarray.c: Test the XArray API
|
||||
* Copyright (c) 2017-2018 Microsoft Corporation
|
||||
* Copyright (c) 2019-2020 Oracle
|
||||
* Author: Matthew Wilcox <willy@infradead.org>
|
||||
*/
|
||||
|
||||
|
|
@ -902,28 +903,34 @@ static noinline void check_store_iter(struct xarray *xa)
|
|||
XA_BUG_ON(xa, !xa_empty(xa));
|
||||
}
|
||||
|
||||
static noinline void check_multi_find(struct xarray *xa)
|
||||
static noinline void check_multi_find_1(struct xarray *xa, unsigned order)
|
||||
{
|
||||
#ifdef CONFIG_XARRAY_MULTI
|
||||
unsigned long multi = 3 << order;
|
||||
unsigned long next = 4 << order;
|
||||
unsigned long index;
|
||||
|
||||
xa_store_order(xa, 12, 2, xa_mk_value(12), GFP_KERNEL);
|
||||
XA_BUG_ON(xa, xa_store_index(xa, 16, GFP_KERNEL) != NULL);
|
||||
xa_store_order(xa, multi, order, xa_mk_value(multi), GFP_KERNEL);
|
||||
XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL);
|
||||
XA_BUG_ON(xa, xa_store_index(xa, next + 1, GFP_KERNEL) != NULL);
|
||||
|
||||
index = 0;
|
||||
XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) !=
|
||||
xa_mk_value(12));
|
||||
XA_BUG_ON(xa, index != 12);
|
||||
index = 13;
|
||||
xa_mk_value(multi));
|
||||
XA_BUG_ON(xa, index != multi);
|
||||
index = multi + 1;
|
||||
XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) !=
|
||||
xa_mk_value(12));
|
||||
XA_BUG_ON(xa, (index < 12) || (index >= 16));
|
||||
xa_mk_value(multi));
|
||||
XA_BUG_ON(xa, (index < multi) || (index >= next));
|
||||
XA_BUG_ON(xa, xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT) !=
|
||||
xa_mk_value(16));
|
||||
XA_BUG_ON(xa, index != 16);
|
||||
xa_mk_value(next));
|
||||
XA_BUG_ON(xa, index != next);
|
||||
XA_BUG_ON(xa, xa_find_after(xa, &index, next, XA_PRESENT) != NULL);
|
||||
XA_BUG_ON(xa, index != next);
|
||||
|
||||
xa_erase_index(xa, 12);
|
||||
xa_erase_index(xa, 16);
|
||||
xa_erase_index(xa, multi);
|
||||
xa_erase_index(xa, next);
|
||||
xa_erase_index(xa, next + 1);
|
||||
XA_BUG_ON(xa, !xa_empty(xa));
|
||||
#endif
|
||||
}
|
||||
|
|
@ -1046,12 +1053,33 @@ static noinline void check_find_3(struct xarray *xa)
|
|||
xa_destroy(xa);
|
||||
}
|
||||
|
||||
static noinline void check_find_4(struct xarray *xa)
|
||||
{
|
||||
unsigned long index = 0;
|
||||
void *entry;
|
||||
|
||||
xa_store_index(xa, ULONG_MAX, GFP_KERNEL);
|
||||
|
||||
entry = xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT);
|
||||
XA_BUG_ON(xa, entry != xa_mk_index(ULONG_MAX));
|
||||
|
||||
entry = xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT);
|
||||
XA_BUG_ON(xa, entry);
|
||||
|
||||
xa_erase_index(xa, ULONG_MAX);
|
||||
}
|
||||
|
||||
static noinline void check_find(struct xarray *xa)
|
||||
{
|
||||
unsigned i;
|
||||
|
||||
check_find_1(xa);
|
||||
check_find_2(xa);
|
||||
check_find_3(xa);
|
||||
check_multi_find(xa);
|
||||
check_find_4(xa);
|
||||
|
||||
for (i = 2; i < 10; i++)
|
||||
check_multi_find_1(xa, i);
|
||||
check_multi_find_2(xa);
|
||||
}
|
||||
|
||||
|
|
@ -1132,6 +1160,27 @@ static noinline void check_move_tiny(struct xarray *xa)
|
|||
XA_BUG_ON(xa, !xa_empty(xa));
|
||||
}
|
||||
|
||||
static noinline void check_move_max(struct xarray *xa)
|
||||
{
|
||||
XA_STATE(xas, xa, 0);
|
||||
|
||||
xa_store_index(xa, ULONG_MAX, GFP_KERNEL);
|
||||
rcu_read_lock();
|
||||
XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_index(ULONG_MAX));
|
||||
XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != NULL);
|
||||
rcu_read_unlock();
|
||||
|
||||
xas_set(&xas, 0);
|
||||
rcu_read_lock();
|
||||
XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_index(ULONG_MAX));
|
||||
xas_pause(&xas);
|
||||
XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != NULL);
|
||||
rcu_read_unlock();
|
||||
|
||||
xa_erase_index(xa, ULONG_MAX);
|
||||
XA_BUG_ON(xa, !xa_empty(xa));
|
||||
}
|
||||
|
||||
static noinline void check_move_small(struct xarray *xa, unsigned long idx)
|
||||
{
|
||||
XA_STATE(xas, xa, 0);
|
||||
|
|
@ -1240,6 +1289,7 @@ static noinline void check_move(struct xarray *xa)
|
|||
xa_destroy(xa);
|
||||
|
||||
check_move_tiny(xa);
|
||||
check_move_max(xa);
|
||||
|
||||
for (i = 0; i < 16; i++)
|
||||
check_move_small(xa, 1UL << i);
|
||||
|
|
|
|||
41
lib/xarray.c
41
lib/xarray.c
|
|
@ -1,7 +1,8 @@
|
|||
// SPDX-License-Identifier: GPL-2.0+
|
||||
/*
|
||||
* XArray implementation
|
||||
* Copyright (c) 2017 Microsoft Corporation
|
||||
* Copyright (c) 2017-2018 Microsoft Corporation
|
||||
* Copyright (c) 2018-2020 Oracle
|
||||
* Author: Matthew Wilcox <willy@infradead.org>
|
||||
*/
|
||||
|
||||
|
|
@ -967,6 +968,7 @@ void xas_pause(struct xa_state *xas)
|
|||
if (xas_invalid(xas))
|
||||
return;
|
||||
|
||||
xas->xa_node = XAS_RESTART;
|
||||
if (node) {
|
||||
unsigned int offset = xas->xa_offset;
|
||||
while (++offset < XA_CHUNK_SIZE) {
|
||||
|
|
@ -974,10 +976,11 @@ void xas_pause(struct xa_state *xas)
|
|||
break;
|
||||
}
|
||||
xas->xa_index += (offset - xas->xa_offset) << node->shift;
|
||||
if (xas->xa_index == 0)
|
||||
xas->xa_node = XAS_BOUNDS;
|
||||
} else {
|
||||
xas->xa_index++;
|
||||
}
|
||||
xas->xa_node = XAS_RESTART;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(xas_pause);
|
||||
|
||||
|
|
@ -1079,13 +1082,15 @@ void *xas_find(struct xa_state *xas, unsigned long max)
|
|||
{
|
||||
void *entry;
|
||||
|
||||
if (xas_error(xas))
|
||||
if (xas_error(xas) || xas->xa_node == XAS_BOUNDS)
|
||||
return NULL;
|
||||
if (xas->xa_index > max)
|
||||
return set_bounds(xas);
|
||||
|
||||
if (!xas->xa_node) {
|
||||
xas->xa_index = 1;
|
||||
return set_bounds(xas);
|
||||
} else if (xas_top(xas->xa_node)) {
|
||||
} else if (xas->xa_node == XAS_RESTART) {
|
||||
entry = xas_load(xas);
|
||||
if (entry || xas_not_node(xas->xa_node))
|
||||
return entry;
|
||||
|
|
@ -1150,6 +1155,8 @@ void *xas_find_marked(struct xa_state *xas, unsigned long max, xa_mark_t mark)
|
|||
|
||||
if (xas_error(xas))
|
||||
return NULL;
|
||||
if (xas->xa_index > max)
|
||||
goto max;
|
||||
|
||||
if (!xas->xa_node) {
|
||||
xas->xa_index = 1;
|
||||
|
|
@ -1824,6 +1831,17 @@ void *xa_find(struct xarray *xa, unsigned long *indexp,
|
|||
}
|
||||
EXPORT_SYMBOL(xa_find);
|
||||
|
||||
static bool xas_sibling(struct xa_state *xas)
|
||||
{
|
||||
struct xa_node *node = xas->xa_node;
|
||||
unsigned long mask;
|
||||
|
||||
if (!node)
|
||||
return false;
|
||||
mask = (XA_CHUNK_SIZE << node->shift) - 1;
|
||||
return (xas->xa_index & mask) > (xas->xa_offset << node->shift);
|
||||
}
|
||||
|
||||
/**
|
||||
* xa_find_after() - Search the XArray for a present entry.
|
||||
* @xa: XArray.
|
||||
|
|
@ -1847,21 +1865,20 @@ void *xa_find_after(struct xarray *xa, unsigned long *indexp,
|
|||
XA_STATE(xas, xa, *indexp + 1);
|
||||
void *entry;
|
||||
|
||||
if (xas.xa_index == 0)
|
||||
return NULL;
|
||||
|
||||
rcu_read_lock();
|
||||
for (;;) {
|
||||
if ((__force unsigned int)filter < XA_MAX_MARKS)
|
||||
entry = xas_find_marked(&xas, max, filter);
|
||||
else
|
||||
entry = xas_find(&xas, max);
|
||||
if (xas.xa_node == XAS_BOUNDS)
|
||||
|
||||
if (xas_invalid(&xas))
|
||||
break;
|
||||
if (xas.xa_shift) {
|
||||
if (xas.xa_index & ((1UL << xas.xa_shift) - 1))
|
||||
continue;
|
||||
} else {
|
||||
if (xas.xa_offset < (xas.xa_index & XA_CHUNK_MASK))
|
||||
continue;
|
||||
}
|
||||
if (xas_sibling(&xas))
|
||||
continue;
|
||||
if (!xas_retry(&xas, entry))
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue