sbitmap: randomize initial alloc_hint values
In order to get good cache behavior from a sbitmap, we want each CPU to stick to its own cacheline(s) as much as possible. This might happen naturally as the bitmap gets filled up and the alloc_hint values spread out, but we really want this behavior from the start. blk-mq apparently intended to do this, but the code to do this was never wired up. Get rid of the dead code and make it part of the sbitmap library. Signed-off-by: Omar Sandoval <osandov@fb.com> Signed-off-by: Jens Axboe <axboe@fb.com>hifive-unleashed-5.1
parent
f4a644db86
commit
98d95416db
|
@ -7,7 +7,6 @@
|
||||||
*/
|
*/
|
||||||
#include <linux/kernel.h>
|
#include <linux/kernel.h>
|
||||||
#include <linux/module.h>
|
#include <linux/module.h>
|
||||||
#include <linux/random.h>
|
|
||||||
|
|
||||||
#include <linux/blk-mq.h>
|
#include <linux/blk-mq.h>
|
||||||
#include "blk.h"
|
#include "blk.h"
|
||||||
|
@ -419,13 +418,6 @@ void blk_mq_free_tags(struct blk_mq_tags *tags)
|
||||||
kfree(tags);
|
kfree(tags);
|
||||||
}
|
}
|
||||||
|
|
||||||
void blk_mq_tag_init_last_tag(struct blk_mq_tags *tags, unsigned int *tag)
|
|
||||||
{
|
|
||||||
unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
|
|
||||||
|
|
||||||
*tag = prandom_u32() % depth;
|
|
||||||
}
|
|
||||||
|
|
||||||
int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int tdepth)
|
int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int tdepth)
|
||||||
{
|
{
|
||||||
tdepth -= tags->nr_reserved_tags;
|
tdepth -= tags->nr_reserved_tags;
|
||||||
|
|
|
@ -30,7 +30,6 @@ extern void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
|
||||||
unsigned int tag);
|
unsigned int tag);
|
||||||
extern bool blk_mq_has_free_tags(struct blk_mq_tags *tags);
|
extern bool blk_mq_has_free_tags(struct blk_mq_tags *tags);
|
||||||
extern ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page);
|
extern ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page);
|
||||||
extern void blk_mq_tag_init_last_tag(struct blk_mq_tags *tags, unsigned int *last_tag);
|
|
||||||
extern int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int depth);
|
extern int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int depth);
|
||||||
extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);
|
extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);
|
||||||
void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
|
void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
|
||||||
|
|
|
@ -15,6 +15,7 @@
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
#include <linux/random.h>
|
||||||
#include <linux/sbitmap.h>
|
#include <linux/sbitmap.h>
|
||||||
|
|
||||||
int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
|
int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
|
||||||
|
@ -211,6 +212,11 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (depth && !round_robin) {
|
||||||
|
for_each_possible_cpu(i)
|
||||||
|
*per_cpu_ptr(sbq->alloc_hint, i) = prandom_u32() % depth;
|
||||||
|
}
|
||||||
|
|
||||||
sbq->wake_batch = sbq_calc_wake_batch(depth);
|
sbq->wake_batch = sbq_calc_wake_batch(depth);
|
||||||
atomic_set(&sbq->wake_index, 0);
|
atomic_set(&sbq->wake_index, 0);
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue