2013-10-25 04:52:25 -06:00
|
|
|
#include <linux/module.h>
|
2013-12-20 16:11:01 -07:00
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
#include <linux/moduleparam.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/blkdev.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/blk-mq.h>
|
|
|
|
#include <linux/hrtimer.h>
|
2015-11-12 12:25:10 -07:00
|
|
|
#include <linux/lightnvm.h>
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
struct nullb_cmd {
|
|
|
|
struct list_head list;
|
|
|
|
struct llist_node ll_list;
|
|
|
|
struct call_single_data csd;
|
|
|
|
struct request *rq;
|
|
|
|
struct bio *bio;
|
|
|
|
unsigned int tag;
|
|
|
|
struct nullb_queue *nq;
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
struct hrtimer timer;
|
2013-10-25 04:52:25 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
struct nullb_queue {
|
|
|
|
unsigned long *tag_map;
|
|
|
|
wait_queue_head_t wait;
|
|
|
|
unsigned int queue_depth;
|
|
|
|
|
|
|
|
struct nullb_cmd *cmds;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct nullb {
|
|
|
|
struct list_head list;
|
|
|
|
unsigned int index;
|
|
|
|
struct request_queue *q;
|
|
|
|
struct gendisk *disk;
|
2014-04-15 14:14:00 -06:00
|
|
|
struct blk_mq_tag_set tag_set;
|
2013-10-25 04:52:25 -06:00
|
|
|
struct hrtimer timer;
|
|
|
|
unsigned int queue_depth;
|
|
|
|
spinlock_t lock;
|
|
|
|
|
|
|
|
struct nullb_queue *queues;
|
|
|
|
unsigned int nr_queues;
|
2015-11-12 12:25:10 -07:00
|
|
|
char disk_name[DISK_NAME_LEN];
|
2013-10-25 04:52:25 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
static LIST_HEAD(nullb_list);
|
|
|
|
static struct mutex lock;
|
|
|
|
static int null_major;
|
|
|
|
static int nullb_indexes;
|
2015-11-19 04:50:08 -07:00
|
|
|
static struct kmem_cache *ppa_cache;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
enum {
|
|
|
|
NULL_IRQ_NONE = 0,
|
|
|
|
NULL_IRQ_SOFTIRQ = 1,
|
|
|
|
NULL_IRQ_TIMER = 2,
|
2014-02-10 04:24:40 -07:00
|
|
|
};
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2014-02-10 04:24:40 -07:00
|
|
|
enum {
|
2013-10-25 04:52:25 -06:00
|
|
|
NULL_Q_BIO = 0,
|
|
|
|
NULL_Q_RQ = 1,
|
|
|
|
NULL_Q_MQ = 2,
|
|
|
|
};
|
|
|
|
|
2013-12-18 05:41:43 -07:00
|
|
|
static int submit_queues;
|
2013-10-25 04:52:25 -06:00
|
|
|
module_param(submit_queues, int, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(submit_queues, "Number of submission queues");
|
|
|
|
|
|
|
|
static int home_node = NUMA_NO_NODE;
|
|
|
|
module_param(home_node, int, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(home_node, "Home node for the device");
|
|
|
|
|
|
|
|
static int queue_mode = NULL_Q_MQ;
|
2014-11-26 14:45:48 -07:00
|
|
|
|
|
|
|
static int null_param_store_val(const char *str, int *val, int min, int max)
|
|
|
|
{
|
|
|
|
int ret, new_val;
|
|
|
|
|
|
|
|
ret = kstrtoint(str, 10, &new_val);
|
|
|
|
if (ret)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (new_val < min || new_val > max)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
*val = new_val;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
|
|
|
|
{
|
|
|
|
return null_param_store_val(str, &queue_mode, NULL_Q_BIO, NULL_Q_MQ);
|
|
|
|
}
|
|
|
|
|
2015-05-26 19:39:38 -06:00
|
|
|
static const struct kernel_param_ops null_queue_mode_param_ops = {
|
2014-11-26 14:45:48 -07:00
|
|
|
.set = null_set_queue_mode,
|
|
|
|
.get = param_get_int,
|
|
|
|
};
|
|
|
|
|
|
|
|
device_param_cb(queue_mode, &null_queue_mode_param_ops, &queue_mode, S_IRUGO);
|
2014-06-11 15:13:50 -06:00
|
|
|
MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
static int gb = 250;
|
|
|
|
module_param(gb, int, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(gb, "Size in GB");
|
|
|
|
|
|
|
|
static int bs = 512;
|
|
|
|
module_param(bs, int, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(bs, "Block size (in bytes)");
|
|
|
|
|
|
|
|
static int nr_devices = 2;
|
|
|
|
module_param(nr_devices, int, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(nr_devices, "Number of devices to register");
|
|
|
|
|
2015-11-12 12:25:10 -07:00
|
|
|
static bool use_lightnvm;
|
|
|
|
module_param(use_lightnvm, bool, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(use_lightnvm, "Register as a LightNVM device");
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
static int irqmode = NULL_IRQ_SOFTIRQ;
|
2014-11-26 14:45:48 -07:00
|
|
|
|
|
|
|
static int null_set_irqmode(const char *str, const struct kernel_param *kp)
|
|
|
|
{
|
|
|
|
return null_param_store_val(str, &irqmode, NULL_IRQ_NONE,
|
|
|
|
NULL_IRQ_TIMER);
|
|
|
|
}
|
|
|
|
|
2015-05-26 19:39:38 -06:00
|
|
|
static const struct kernel_param_ops null_irqmode_param_ops = {
|
2014-11-26 14:45:48 -07:00
|
|
|
.set = null_set_irqmode,
|
|
|
|
.get = param_get_int,
|
|
|
|
};
|
|
|
|
|
|
|
|
device_param_cb(irqmode, &null_irqmode_param_ops, &irqmode, S_IRUGO);
|
2013-10-25 04:52:25 -06:00
|
|
|
MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
|
|
|
|
|
2015-12-01 03:48:19 -07:00
|
|
|
static unsigned long completion_nsec = 10000;
|
|
|
|
module_param(completion_nsec, ulong, S_IRUGO);
|
2013-10-25 04:52:25 -06:00
|
|
|
MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
|
|
|
|
|
|
|
|
static int hw_queue_depth = 64;
|
|
|
|
module_param(hw_queue_depth, int, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
|
|
|
|
|
2013-12-20 16:11:00 -07:00
|
|
|
static bool use_per_node_hctx = false;
|
2013-10-25 04:52:25 -06:00
|
|
|
module_param(use_per_node_hctx, bool, S_IRUGO);
|
2013-12-20 16:11:00 -07:00
|
|
|
MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
static void put_tag(struct nullb_queue *nq, unsigned int tag)
|
|
|
|
{
|
|
|
|
clear_bit_unlock(tag, nq->tag_map);
|
|
|
|
|
|
|
|
if (waitqueue_active(&nq->wait))
|
|
|
|
wake_up(&nq->wait);
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned int get_tag(struct nullb_queue *nq)
|
|
|
|
{
|
|
|
|
unsigned int tag;
|
|
|
|
|
|
|
|
do {
|
|
|
|
tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
|
|
|
|
if (tag >= nq->queue_depth)
|
|
|
|
return -1U;
|
|
|
|
} while (test_and_set_bit_lock(tag, nq->tag_map));
|
|
|
|
|
|
|
|
return tag;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void free_cmd(struct nullb_cmd *cmd)
|
|
|
|
{
|
|
|
|
put_tag(cmd->nq, cmd->tag);
|
|
|
|
}
|
|
|
|
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer);
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
|
|
|
|
{
|
|
|
|
struct nullb_cmd *cmd;
|
|
|
|
unsigned int tag;
|
|
|
|
|
|
|
|
tag = get_tag(nq);
|
|
|
|
if (tag != -1U) {
|
|
|
|
cmd = &nq->cmds[tag];
|
|
|
|
cmd->tag = tag;
|
|
|
|
cmd->nq = nq;
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
if (irqmode == NULL_IRQ_TIMER) {
|
|
|
|
hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
|
|
|
|
HRTIMER_MODE_REL);
|
|
|
|
cmd->timer.function = null_cmd_timer_expired;
|
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
return cmd;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
|
|
|
|
{
|
|
|
|
struct nullb_cmd *cmd;
|
|
|
|
DEFINE_WAIT(wait);
|
|
|
|
|
|
|
|
cmd = __alloc_cmd(nq);
|
|
|
|
if (cmd || !can_wait)
|
|
|
|
return cmd;
|
|
|
|
|
|
|
|
do {
|
|
|
|
prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
|
|
|
|
cmd = __alloc_cmd(nq);
|
|
|
|
if (cmd)
|
|
|
|
break;
|
|
|
|
|
|
|
|
io_schedule();
|
|
|
|
} while (1);
|
|
|
|
|
|
|
|
finish_wait(&nq->wait, &wait);
|
|
|
|
return cmd;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void end_cmd(struct nullb_cmd *cmd)
|
|
|
|
{
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
struct request_queue *q = NULL;
|
|
|
|
|
2015-12-15 02:56:40 -07:00
|
|
|
if (cmd->rq)
|
|
|
|
q = cmd->rq->q;
|
|
|
|
|
2014-02-10 04:24:40 -07:00
|
|
|
switch (queue_mode) {
|
|
|
|
case NULL_Q_MQ:
|
2014-09-13 17:40:10 -06:00
|
|
|
blk_mq_end_request(cmd->rq, 0);
|
2014-02-10 04:24:40 -07:00
|
|
|
return;
|
|
|
|
case NULL_Q_RQ:
|
|
|
|
INIT_LIST_HEAD(&cmd->rq->queuelist);
|
|
|
|
blk_end_request_all(cmd->rq, 0);
|
|
|
|
break;
|
|
|
|
case NULL_Q_BIO:
|
2015-07-20 07:29:37 -06:00
|
|
|
bio_endio(cmd->bio);
|
2015-12-28 13:02:47 -07:00
|
|
|
break;
|
2014-02-10 04:24:40 -07:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2015-12-28 13:02:47 -07:00
|
|
|
free_cmd(cmd);
|
|
|
|
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
/* Restart queue if needed, as we are freeing a tag */
|
2015-12-28 13:02:47 -07:00
|
|
|
if (queue_mode == NULL_Q_RQ && blk_queue_stopped(q)) {
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(q->queue_lock, flags);
|
2015-12-28 13:02:47 -07:00
|
|
|
blk_start_queue_async(q);
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
spin_unlock_irqrestore(q->queue_lock, flags);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
end_cmd(container_of(timer, struct nullb_cmd, timer));
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
return HRTIMER_NORESTART;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_cmd_end_timer(struct nullb_cmd *cmd)
|
|
|
|
{
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
ktime_t kt = ktime_set(0, completion_nsec);
|
2013-10-25 04:52:25 -06:00
|
|
|
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void null_softirq_done_fn(struct request *rq)
|
|
|
|
{
|
2014-06-16 11:40:25 -06:00
|
|
|
if (queue_mode == NULL_Q_MQ)
|
|
|
|
end_cmd(blk_mq_rq_to_pdu(rq));
|
|
|
|
else
|
|
|
|
end_cmd(rq->special);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void null_handle_cmd(struct nullb_cmd *cmd)
|
|
|
|
{
|
|
|
|
/* Complete IO by inline, softirq or timer */
|
|
|
|
switch (irqmode) {
|
|
|
|
case NULL_IRQ_SOFTIRQ:
|
2014-02-10 04:24:40 -07:00
|
|
|
switch (queue_mode) {
|
|
|
|
case NULL_Q_MQ:
|
2015-09-27 13:01:50 -06:00
|
|
|
blk_mq_complete_request(cmd->rq, cmd->rq->errors);
|
2014-02-10 04:24:40 -07:00
|
|
|
break;
|
|
|
|
case NULL_Q_RQ:
|
|
|
|
blk_complete_request(cmd->rq);
|
|
|
|
break;
|
|
|
|
case NULL_Q_BIO:
|
|
|
|
/*
|
|
|
|
* XXX: no proper submitting cpu information available.
|
|
|
|
*/
|
|
|
|
end_cmd(cmd);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case NULL_IRQ_NONE:
|
2013-10-25 04:52:25 -06:00
|
|
|
end_cmd(cmd);
|
|
|
|
break;
|
|
|
|
case NULL_IRQ_TIMER:
|
|
|
|
null_cmd_end_timer(cmd);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
int index = 0;
|
|
|
|
|
|
|
|
if (nullb->nr_queues != 1)
|
|
|
|
index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
|
|
|
|
|
|
|
|
return &nullb->queues[index];
|
|
|
|
}
|
|
|
|
|
2015-11-05 10:41:16 -07:00
|
|
|
static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
|
2013-10-25 04:52:25 -06:00
|
|
|
{
|
|
|
|
struct nullb *nullb = q->queuedata;
|
|
|
|
struct nullb_queue *nq = nullb_to_queue(nullb);
|
|
|
|
struct nullb_cmd *cmd;
|
|
|
|
|
|
|
|
cmd = alloc_cmd(nq, 1);
|
|
|
|
cmd->bio = bio;
|
|
|
|
|
|
|
|
null_handle_cmd(cmd);
|
2015-11-05 10:41:16 -07:00
|
|
|
return BLK_QC_T_NONE;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int null_rq_prep_fn(struct request_queue *q, struct request *req)
|
|
|
|
{
|
|
|
|
struct nullb *nullb = q->queuedata;
|
|
|
|
struct nullb_queue *nq = nullb_to_queue(nullb);
|
|
|
|
struct nullb_cmd *cmd;
|
|
|
|
|
|
|
|
cmd = alloc_cmd(nq, 0);
|
|
|
|
if (cmd) {
|
|
|
|
cmd->rq = req;
|
|
|
|
req->special = cmd;
|
|
|
|
return BLKPREP_OK;
|
|
|
|
}
|
2015-06-01 17:35:10 -06:00
|
|
|
blk_stop_queue(q);
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
return BLKPREP_DEFER;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_request_fn(struct request_queue *q)
|
|
|
|
{
|
|
|
|
struct request *rq;
|
|
|
|
|
|
|
|
while ((rq = blk_fetch_request(q)) != NULL) {
|
|
|
|
struct nullb_cmd *cmd = rq->special;
|
|
|
|
|
|
|
|
spin_unlock_irq(q->queue_lock);
|
|
|
|
null_handle_cmd(cmd);
|
|
|
|
spin_lock_irq(q->queue_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-10-29 11:14:52 -06:00
|
|
|
static int null_queue_rq(struct blk_mq_hw_ctx *hctx,
|
|
|
|
const struct blk_mq_queue_data *bd)
|
2013-10-25 04:52:25 -06:00
|
|
|
{
|
2014-10-29 11:14:52 -06:00
|
|
|
struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
|
2013-10-25 04:52:25 -06:00
|
|
|
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
if (irqmode == NULL_IRQ_TIMER) {
|
|
|
|
hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
|
|
|
cmd->timer.function = null_cmd_timer_expired;
|
|
|
|
}
|
2014-10-29 11:14:52 -06:00
|
|
|
cmd->rq = bd->rq;
|
2013-10-25 04:52:25 -06:00
|
|
|
cmd->nq = hctx->driver_data;
|
|
|
|
|
2014-10-29 11:14:52 -06:00
|
|
|
blk_mq_start_request(bd->rq);
|
2014-09-13 17:40:09 -06:00
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
null_handle_cmd(cmd);
|
|
|
|
return BLK_MQ_RQ_QUEUE_OK;
|
|
|
|
}
|
|
|
|
|
2013-12-18 05:41:43 -07:00
|
|
|
static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
|
|
|
|
{
|
|
|
|
BUG_ON(!nullb);
|
|
|
|
BUG_ON(!nq);
|
|
|
|
|
|
|
|
init_waitqueue_head(&nq->wait);
|
|
|
|
nq->queue_depth = nullb->queue_depth;
|
|
|
|
}
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
|
|
|
|
unsigned int index)
|
|
|
|
{
|
|
|
|
struct nullb *nullb = data;
|
|
|
|
struct nullb_queue *nq = &nullb->queues[index];
|
|
|
|
|
|
|
|
hctx->driver_data = nq;
|
2013-12-18 05:41:43 -07:00
|
|
|
null_init_queue(nullb, nq);
|
|
|
|
nullb->nr_queues++;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct blk_mq_ops null_mq_ops = {
|
|
|
|
.queue_rq = null_queue_rq,
|
|
|
|
.map_queue = blk_mq_map_queue,
|
|
|
|
.init_hctx = null_init_hctx,
|
2014-02-10 04:24:40 -07:00
|
|
|
.complete = null_softirq_done_fn,
|
2013-10-25 04:52:25 -06:00
|
|
|
};
|
|
|
|
|
2015-08-31 06:17:18 -06:00
|
|
|
static void cleanup_queue(struct nullb_queue *nq)
|
|
|
|
{
|
|
|
|
kfree(nq->tag_map);
|
|
|
|
kfree(nq->cmds);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void cleanup_queues(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < nullb->nr_queues; i++)
|
|
|
|
cleanup_queue(&nullb->queues[i]);
|
|
|
|
|
|
|
|
kfree(nullb->queues);
|
|
|
|
}
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
static void null_del_dev(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
list_del_init(&nullb->list);
|
|
|
|
|
2015-11-12 12:25:10 -07:00
|
|
|
if (use_lightnvm)
|
2015-11-19 04:50:10 -07:00
|
|
|
nvm_unregister(nullb->disk_name);
|
|
|
|
else
|
|
|
|
del_gendisk(nullb->disk);
|
2013-12-26 06:31:37 -07:00
|
|
|
blk_cleanup_queue(nullb->q);
|
2014-04-15 14:14:00 -06:00
|
|
|
if (queue_mode == NULL_Q_MQ)
|
|
|
|
blk_mq_free_tag_set(&nullb->tag_set);
|
2015-11-19 04:50:10 -07:00
|
|
|
if (!use_lightnvm)
|
|
|
|
put_disk(nullb->disk);
|
2015-08-31 06:17:18 -06:00
|
|
|
cleanup_queues(nullb);
|
2013-10-25 04:52:25 -06:00
|
|
|
kfree(nullb);
|
|
|
|
}
|
|
|
|
|
2015-11-12 12:25:10 -07:00
|
|
|
#ifdef CONFIG_NVM
|
|
|
|
|
|
|
|
static void null_lnvm_end_io(struct request *rq, int error)
|
|
|
|
{
|
|
|
|
struct nvm_rq *rqd = rq->end_io_data;
|
|
|
|
|
2016-01-11 23:49:21 -07:00
|
|
|
nvm_end_io(rqd, error);
|
2015-11-12 12:25:10 -07:00
|
|
|
|
|
|
|
blk_put_request(rq);
|
|
|
|
}
|
|
|
|
|
2015-12-06 03:25:48 -07:00
|
|
|
static int null_lnvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
|
2015-11-12 12:25:10 -07:00
|
|
|
{
|
2015-12-06 03:25:48 -07:00
|
|
|
struct request_queue *q = dev->q;
|
2015-11-12 12:25:10 -07:00
|
|
|
struct request *rq;
|
|
|
|
struct bio *bio = rqd->bio;
|
|
|
|
|
2016-07-19 03:28:41 -06:00
|
|
|
rq = blk_mq_alloc_request(q, bio_data_dir(bio), 0);
|
2015-11-12 12:25:10 -07:00
|
|
|
if (IS_ERR(rq))
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
rq->cmd_type = REQ_TYPE_DRV_PRIV;
|
|
|
|
rq->__sector = bio->bi_iter.bi_sector;
|
|
|
|
rq->ioprio = bio_prio(bio);
|
|
|
|
|
|
|
|
if (bio_has_data(bio))
|
|
|
|
rq->nr_phys_segments = bio_phys_segments(q, bio);
|
|
|
|
|
|
|
|
rq->__data_len = bio->bi_iter.bi_size;
|
|
|
|
rq->bio = rq->biotail = bio;
|
|
|
|
|
|
|
|
rq->end_io_data = rqd;
|
|
|
|
|
|
|
|
blk_execute_rq_nowait(q, NULL, rq, 0, null_lnvm_end_io);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-12-06 03:25:48 -07:00
|
|
|
static int null_lnvm_id(struct nvm_dev *dev, struct nvm_id *id)
|
2015-11-12 12:25:10 -07:00
|
|
|
{
|
|
|
|
sector_t size = gb * 1024 * 1024 * 1024ULL;
|
2015-11-19 04:50:09 -07:00
|
|
|
sector_t blksize;
|
2015-11-12 12:25:10 -07:00
|
|
|
struct nvm_id_group *grp;
|
|
|
|
|
|
|
|
id->ver_id = 0x1;
|
|
|
|
id->vmnt = 0;
|
|
|
|
id->cgrps = 1;
|
2016-02-04 07:13:27 -07:00
|
|
|
id->cap = 0x2;
|
2015-11-12 12:25:10 -07:00
|
|
|
id->dom = 0x1;
|
2015-11-19 04:50:09 -07:00
|
|
|
|
|
|
|
id->ppaf.blk_offset = 0;
|
|
|
|
id->ppaf.blk_len = 16;
|
|
|
|
id->ppaf.pg_offset = 16;
|
|
|
|
id->ppaf.pg_len = 16;
|
|
|
|
id->ppaf.sect_offset = 32;
|
|
|
|
id->ppaf.sect_len = 8;
|
|
|
|
id->ppaf.pln_offset = 40;
|
|
|
|
id->ppaf.pln_len = 8;
|
|
|
|
id->ppaf.lun_offset = 48;
|
|
|
|
id->ppaf.lun_len = 8;
|
|
|
|
id->ppaf.ch_offset = 56;
|
|
|
|
id->ppaf.ch_len = 8;
|
2015-11-12 12:25:10 -07:00
|
|
|
|
2016-01-13 15:04:08 -07:00
|
|
|
sector_div(size, bs); /* convert size to pages */
|
|
|
|
size >>= 8; /* concert size to pgs pr blk */
|
2015-11-12 12:25:10 -07:00
|
|
|
grp = &id->groups[0];
|
|
|
|
grp->mtype = 0;
|
2015-11-19 04:50:09 -07:00
|
|
|
grp->fmtype = 0;
|
2015-11-12 12:25:10 -07:00
|
|
|
grp->num_ch = 1;
|
|
|
|
grp->num_pg = 256;
|
2015-11-19 04:50:09 -07:00
|
|
|
blksize = size;
|
2016-01-13 15:04:08 -07:00
|
|
|
size >>= 16;
|
2015-11-19 04:50:09 -07:00
|
|
|
grp->num_lun = size + 1;
|
2016-01-13 15:04:08 -07:00
|
|
|
sector_div(blksize, grp->num_lun);
|
2015-11-19 04:50:09 -07:00
|
|
|
grp->num_blk = blksize;
|
|
|
|
grp->num_pln = 1;
|
|
|
|
|
2015-11-12 12:25:10 -07:00
|
|
|
grp->fpg_sz = bs;
|
|
|
|
grp->csecs = bs;
|
|
|
|
grp->trdt = 25000;
|
|
|
|
grp->trdm = 25000;
|
|
|
|
grp->tprt = 500000;
|
|
|
|
grp->tprm = 500000;
|
|
|
|
grp->tbet = 1500000;
|
|
|
|
grp->tbem = 1500000;
|
|
|
|
grp->mpos = 0x010101; /* single plane rwe */
|
|
|
|
grp->cpar = hw_queue_depth;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-12-06 03:25:48 -07:00
|
|
|
static void *null_lnvm_create_dma_pool(struct nvm_dev *dev, char *name)
|
2015-11-12 12:25:10 -07:00
|
|
|
{
|
|
|
|
mempool_t *virtmem_pool;
|
|
|
|
|
2015-11-19 04:50:08 -07:00
|
|
|
virtmem_pool = mempool_create_slab_pool(64, ppa_cache);
|
2015-11-12 12:25:10 -07:00
|
|
|
if (!virtmem_pool) {
|
|
|
|
pr_err("null_blk: Unable to create virtual memory pool\n");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return virtmem_pool;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_lnvm_destroy_dma_pool(void *pool)
|
|
|
|
{
|
|
|
|
mempool_destroy(pool);
|
|
|
|
}
|
|
|
|
|
2015-12-06 03:25:48 -07:00
|
|
|
static void *null_lnvm_dev_dma_alloc(struct nvm_dev *dev, void *pool,
|
2015-11-12 12:25:10 -07:00
|
|
|
gfp_t mem_flags, dma_addr_t *dma_handler)
|
|
|
|
{
|
|
|
|
return mempool_alloc(pool, mem_flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_lnvm_dev_dma_free(void *pool, void *entry,
|
|
|
|
dma_addr_t dma_handler)
|
|
|
|
{
|
|
|
|
mempool_free(entry, pool);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct nvm_dev_ops null_lnvm_dev_ops = {
|
|
|
|
.identity = null_lnvm_id,
|
|
|
|
.submit_io = null_lnvm_submit_io,
|
|
|
|
|
|
|
|
.create_dma_pool = null_lnvm_create_dma_pool,
|
|
|
|
.destroy_dma_pool = null_lnvm_destroy_dma_pool,
|
|
|
|
.dev_dma_alloc = null_lnvm_dev_dma_alloc,
|
|
|
|
.dev_dma_free = null_lnvm_dev_dma_free,
|
|
|
|
|
|
|
|
/* Simulate nvme protocol restriction */
|
|
|
|
.max_phys_sect = 64,
|
|
|
|
};
|
|
|
|
#else
|
|
|
|
static struct nvm_dev_ops null_lnvm_dev_ops;
|
|
|
|
#endif /* CONFIG_NVM */
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
static int null_open(struct block_device *bdev, fmode_t mode)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_release(struct gendisk *disk, fmode_t mode)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct block_device_operations null_fops = {
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.open = null_open,
|
|
|
|
.release = null_release,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int setup_commands(struct nullb_queue *nq)
|
|
|
|
{
|
|
|
|
struct nullb_cmd *cmd;
|
|
|
|
int i, tag_size;
|
|
|
|
|
|
|
|
nq->cmds = kzalloc(nq->queue_depth * sizeof(*cmd), GFP_KERNEL);
|
|
|
|
if (!nq->cmds)
|
2013-12-18 05:41:43 -07:00
|
|
|
return -ENOMEM;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
|
|
|
|
nq->tag_map = kzalloc(tag_size * sizeof(unsigned long), GFP_KERNEL);
|
|
|
|
if (!nq->tag_map) {
|
|
|
|
kfree(nq->cmds);
|
2013-12-18 05:41:43 -07:00
|
|
|
return -ENOMEM;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < nq->queue_depth; i++) {
|
|
|
|
cmd = &nq->cmds[i];
|
|
|
|
INIT_LIST_HEAD(&cmd->list);
|
|
|
|
cmd->ll_list.next = NULL;
|
|
|
|
cmd->tag = -1U;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int setup_queues(struct nullb *nullb)
|
|
|
|
{
|
2013-12-18 05:41:43 -07:00
|
|
|
nullb->queues = kzalloc(submit_queues * sizeof(struct nullb_queue),
|
|
|
|
GFP_KERNEL);
|
2013-10-25 04:52:25 -06:00
|
|
|
if (!nullb->queues)
|
2013-12-18 05:41:43 -07:00
|
|
|
return -ENOMEM;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
nullb->nr_queues = 0;
|
|
|
|
nullb->queue_depth = hw_queue_depth;
|
|
|
|
|
2013-12-18 05:41:43 -07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int init_driver_queues(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
struct nullb_queue *nq;
|
|
|
|
int i, ret = 0;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
for (i = 0; i < submit_queues; i++) {
|
|
|
|
nq = &nullb->queues[i];
|
2013-12-18 05:41:43 -07:00
|
|
|
|
|
|
|
null_init_queue(nullb, nq);
|
|
|
|
|
|
|
|
ret = setup_commands(nq);
|
|
|
|
if (ret)
|
2014-10-22 07:34:21 -06:00
|
|
|
return ret;
|
2013-10-25 04:52:25 -06:00
|
|
|
nullb->nr_queues++;
|
|
|
|
}
|
2013-12-18 05:41:43 -07:00
|
|
|
return 0;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int null_add_dev(void)
|
|
|
|
{
|
|
|
|
struct gendisk *disk;
|
|
|
|
struct nullb *nullb;
|
|
|
|
sector_t size;
|
2014-09-02 10:38:49 -06:00
|
|
|
int rv;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, home_node);
|
2014-09-02 10:38:49 -06:00
|
|
|
if (!nullb) {
|
|
|
|
rv = -ENOMEM;
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out;
|
2014-09-02 10:38:49 -06:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
spin_lock_init(&nullb->lock);
|
|
|
|
|
2013-12-10 08:50:38 -07:00
|
|
|
if (queue_mode == NULL_Q_MQ && use_per_node_hctx)
|
|
|
|
submit_queues = nr_online_nodes;
|
|
|
|
|
2014-09-02 10:38:49 -06:00
|
|
|
rv = setup_queues(nullb);
|
|
|
|
if (rv)
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out_free_nullb;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
if (queue_mode == NULL_Q_MQ) {
|
2014-05-28 10:11:06 -06:00
|
|
|
nullb->tag_set.ops = &null_mq_ops;
|
2014-04-15 14:14:00 -06:00
|
|
|
nullb->tag_set.nr_hw_queues = submit_queues;
|
|
|
|
nullb->tag_set.queue_depth = hw_queue_depth;
|
|
|
|
nullb->tag_set.numa_node = home_node;
|
|
|
|
nullb->tag_set.cmd_size = sizeof(struct nullb_cmd);
|
|
|
|
nullb->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
|
|
|
|
nullb->tag_set.driver_data = nullb;
|
|
|
|
|
2014-09-02 10:38:49 -06:00
|
|
|
rv = blk_mq_alloc_tag_set(&nullb->tag_set);
|
|
|
|
if (rv)
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out_cleanup_queues;
|
|
|
|
|
|
|
|
nullb->q = blk_mq_init_queue(&nullb->tag_set);
|
2015-01-02 07:25:27 -07:00
|
|
|
if (IS_ERR(nullb->q)) {
|
2014-09-02 10:38:49 -06:00
|
|
|
rv = -ENOMEM;
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out_cleanup_tags;
|
2014-09-02 10:38:49 -06:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
} else if (queue_mode == NULL_Q_BIO) {
|
|
|
|
nullb->q = blk_alloc_queue_node(GFP_KERNEL, home_node);
|
2014-09-02 10:38:49 -06:00
|
|
|
if (!nullb->q) {
|
|
|
|
rv = -ENOMEM;
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out_cleanup_queues;
|
2014-09-02 10:38:49 -06:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
blk_queue_make_request(nullb->q, null_queue_bio);
|
2014-10-22 07:34:21 -06:00
|
|
|
rv = init_driver_queues(nullb);
|
|
|
|
if (rv)
|
|
|
|
goto out_cleanup_blk_queue;
|
2013-10-25 04:52:25 -06:00
|
|
|
} else {
|
|
|
|
nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock, home_node);
|
2014-09-02 10:38:49 -06:00
|
|
|
if (!nullb->q) {
|
|
|
|
rv = -ENOMEM;
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out_cleanup_queues;
|
2014-09-02 10:38:49 -06:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
blk_queue_prep_rq(nullb->q, null_rq_prep_fn);
|
2014-04-15 14:14:00 -06:00
|
|
|
blk_queue_softirq_done(nullb->q, null_softirq_done_fn);
|
2014-10-22 07:34:21 -06:00
|
|
|
rv = init_driver_queues(nullb);
|
|
|
|
if (rv)
|
|
|
|
goto out_cleanup_blk_queue;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
nullb->q->queuedata = nullb;
|
|
|
|
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, nullb->q);
|
2014-10-04 10:55:32 -06:00
|
|
|
queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, nullb->q);
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
mutex_lock(&lock);
|
|
|
|
nullb->index = nullb_indexes++;
|
|
|
|
mutex_unlock(&lock);
|
|
|
|
|
|
|
|
blk_queue_logical_block_size(nullb->q, bs);
|
|
|
|
blk_queue_physical_block_size(nullb->q, bs);
|
|
|
|
|
2015-11-12 12:25:10 -07:00
|
|
|
sprintf(nullb->disk_name, "nullb%d", nullb->index);
|
|
|
|
|
|
|
|
if (use_lightnvm) {
|
|
|
|
rv = nvm_register(nullb->q, nullb->disk_name,
|
|
|
|
&null_lnvm_dev_ops);
|
|
|
|
if (rv)
|
|
|
|
goto out_cleanup_blk_queue;
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
disk = nullb->disk = alloc_disk_node(1, home_node);
|
|
|
|
if (!disk) {
|
|
|
|
rv = -ENOMEM;
|
|
|
|
goto out_cleanup_lightnvm;
|
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
size = gb * 1024 * 1024 * 1024ULL;
|
2015-08-31 06:17:31 -06:00
|
|
|
set_capacity(disk, size >> 9);
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2015-01-16 16:02:24 -07:00
|
|
|
disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
|
2013-10-25 04:52:25 -06:00
|
|
|
disk->major = null_major;
|
|
|
|
disk->first_minor = nullb->index;
|
|
|
|
disk->fops = &null_fops;
|
|
|
|
disk->private_data = nullb;
|
|
|
|
disk->queue = nullb->q;
|
2015-11-12 12:25:10 -07:00
|
|
|
strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
add_disk(disk);
|
2016-02-11 06:49:13 -07:00
|
|
|
|
2016-03-04 09:27:04 -07:00
|
|
|
done:
|
2016-02-11 06:49:13 -07:00
|
|
|
mutex_lock(&lock);
|
|
|
|
list_add_tail(&nullb->list, &nullb_list);
|
|
|
|
mutex_unlock(&lock);
|
2016-03-04 09:27:04 -07:00
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
return 0;
|
2014-04-15 14:14:00 -06:00
|
|
|
|
2015-11-12 12:25:10 -07:00
|
|
|
out_cleanup_lightnvm:
|
|
|
|
if (use_lightnvm)
|
|
|
|
nvm_unregister(nullb->disk_name);
|
2014-04-15 14:14:00 -06:00
|
|
|
out_cleanup_blk_queue:
|
|
|
|
blk_cleanup_queue(nullb->q);
|
|
|
|
out_cleanup_tags:
|
|
|
|
if (queue_mode == NULL_Q_MQ)
|
|
|
|
blk_mq_free_tag_set(&nullb->tag_set);
|
|
|
|
out_cleanup_queues:
|
|
|
|
cleanup_queues(nullb);
|
|
|
|
out_free_nullb:
|
|
|
|
kfree(nullb);
|
|
|
|
out:
|
2014-09-02 10:38:49 -06:00
|
|
|
return rv;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int __init null_init(void)
|
|
|
|
{
|
2015-12-08 13:47:34 -07:00
|
|
|
int ret = 0;
|
2013-10-25 04:52:25 -06:00
|
|
|
unsigned int i;
|
2015-12-08 13:47:34 -07:00
|
|
|
struct nullb *nullb;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2014-01-21 04:29:59 -07:00
|
|
|
if (bs > PAGE_SIZE) {
|
|
|
|
pr_warn("null_blk: invalid block size\n");
|
|
|
|
pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE);
|
|
|
|
bs = PAGE_SIZE;
|
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2015-11-19 04:50:08 -07:00
|
|
|
if (use_lightnvm && bs != 4096) {
|
|
|
|
pr_warn("null_blk: LightNVM only supports 4k block size\n");
|
|
|
|
pr_warn("null_blk: defaults block size to 4k\n");
|
|
|
|
bs = 4096;
|
|
|
|
}
|
|
|
|
|
2015-11-12 12:25:10 -07:00
|
|
|
if (use_lightnvm && queue_mode != NULL_Q_MQ) {
|
|
|
|
pr_warn("null_blk: LightNVM only supported for blk-mq\n");
|
|
|
|
pr_warn("null_blk: defaults queue mode to blk-mq\n");
|
|
|
|
queue_mode = NULL_Q_MQ;
|
|
|
|
}
|
|
|
|
|
2013-12-18 05:41:44 -07:00
|
|
|
if (queue_mode == NULL_Q_MQ && use_per_node_hctx) {
|
2013-12-20 16:11:01 -07:00
|
|
|
if (submit_queues < nr_online_nodes) {
|
2013-12-18 05:41:44 -07:00
|
|
|
pr_warn("null_blk: submit_queues param is set to %u.",
|
|
|
|
nr_online_nodes);
|
2013-12-20 16:11:01 -07:00
|
|
|
submit_queues = nr_online_nodes;
|
|
|
|
}
|
2013-12-18 05:41:44 -07:00
|
|
|
} else if (submit_queues > nr_cpu_ids)
|
2013-10-25 04:52:25 -06:00
|
|
|
submit_queues = nr_cpu_ids;
|
|
|
|
else if (!submit_queues)
|
|
|
|
submit_queues = 1;
|
|
|
|
|
|
|
|
mutex_init(&lock);
|
|
|
|
|
|
|
|
null_major = register_blkdev(0, "nullb");
|
|
|
|
if (null_major < 0)
|
|
|
|
return null_major;
|
|
|
|
|
2015-11-19 04:50:08 -07:00
|
|
|
if (use_lightnvm) {
|
|
|
|
ppa_cache = kmem_cache_create("ppa_cache", 64 * sizeof(u64),
|
|
|
|
0, 0, NULL);
|
|
|
|
if (!ppa_cache) {
|
|
|
|
pr_err("null_blk: unable to create ppa cache\n");
|
2015-12-08 13:47:34 -07:00
|
|
|
ret = -ENOMEM;
|
|
|
|
goto err_ppa;
|
2015-11-19 04:50:08 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
for (i = 0; i < nr_devices; i++) {
|
2015-12-08 13:47:34 -07:00
|
|
|
ret = null_add_dev();
|
|
|
|
if (ret)
|
|
|
|
goto err_dev;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
pr_info("null: module loaded\n");
|
|
|
|
return 0;
|
2015-12-08 13:47:34 -07:00
|
|
|
|
|
|
|
err_dev:
|
|
|
|
while (!list_empty(&nullb_list)) {
|
|
|
|
nullb = list_entry(nullb_list.next, struct nullb, list);
|
|
|
|
null_del_dev(nullb);
|
|
|
|
}
|
2015-11-19 04:50:08 -07:00
|
|
|
kmem_cache_destroy(ppa_cache);
|
2015-12-08 13:47:34 -07:00
|
|
|
err_ppa:
|
|
|
|
unregister_blkdev(null_major, "nullb");
|
|
|
|
return ret;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void __exit null_exit(void)
|
|
|
|
{
|
|
|
|
struct nullb *nullb;
|
|
|
|
|
|
|
|
unregister_blkdev(null_major, "nullb");
|
|
|
|
|
|
|
|
mutex_lock(&lock);
|
|
|
|
while (!list_empty(&nullb_list)) {
|
|
|
|
nullb = list_entry(nullb_list.next, struct nullb, list);
|
|
|
|
null_del_dev(nullb);
|
|
|
|
}
|
|
|
|
mutex_unlock(&lock);
|
2015-11-19 04:50:08 -07:00
|
|
|
|
|
|
|
kmem_cache_destroy(ppa_cache);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
module_init(null_init);
|
|
|
|
module_exit(null_exit);
|
|
|
|
|
|
|
|
MODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>");
|
|
|
|
MODULE_LICENSE("GPL");
|