2017-08-14 16:04:53 -06:00
|
|
|
/*
|
|
|
|
* Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
|
|
|
|
* Shaohua Li <shli@fb.com>
|
|
|
|
*/
|
2013-10-25 04:52:25 -06:00
|
|
|
#include <linux/module.h>
|
2013-12-20 16:11:01 -07:00
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
#include <linux/moduleparam.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/blkdev.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/blk-mq.h>
|
|
|
|
#include <linux/hrtimer.h>
|
2015-11-12 12:25:10 -07:00
|
|
|
#include <linux/lightnvm.h>
|
2017-08-14 16:04:53 -06:00
|
|
|
#include <linux/configfs.h>
|
2017-08-14 16:05:00 -06:00
|
|
|
#include <linux/badblocks.h>
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
#define SECTOR_SHIFT 9
|
|
|
|
#define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT)
|
|
|
|
#define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT)
|
|
|
|
#define SECTOR_SIZE (1 << SECTOR_SHIFT)
|
|
|
|
#define SECTOR_MASK (PAGE_SECTORS - 1)
|
|
|
|
|
|
|
|
#define FREE_BATCH 16
|
|
|
|
|
2017-08-14 16:04:58 -06:00
|
|
|
#define TICKS_PER_SEC 50ULL
|
|
|
|
#define TIMER_INTERVAL (NSEC_PER_SEC / TICKS_PER_SEC)
|
|
|
|
|
|
|
|
static inline u64 mb_per_tick(int mbps)
|
|
|
|
{
|
|
|
|
return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
|
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
struct nullb_cmd {
|
|
|
|
struct list_head list;
|
|
|
|
struct llist_node ll_list;
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
call_single_data_t csd;
|
2013-10-25 04:52:25 -06:00
|
|
|
struct request *rq;
|
|
|
|
struct bio *bio;
|
|
|
|
unsigned int tag;
|
|
|
|
struct nullb_queue *nq;
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
struct hrtimer timer;
|
2017-08-14 16:04:56 -06:00
|
|
|
blk_status_t error;
|
2013-10-25 04:52:25 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
struct nullb_queue {
|
|
|
|
unsigned long *tag_map;
|
|
|
|
wait_queue_head_t wait;
|
|
|
|
unsigned int queue_depth;
|
2017-08-14 16:04:52 -06:00
|
|
|
struct nullb_device *dev;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
struct nullb_cmd *cmds;
|
|
|
|
};
|
|
|
|
|
2017-08-14 16:04:53 -06:00
|
|
|
/*
|
|
|
|
* Status flags for nullb_device.
|
|
|
|
*
|
|
|
|
* CONFIGURED: Device has been configured and turned on. Cannot reconfigure.
|
|
|
|
* UP: Device is currently on and visible in userspace.
|
2017-08-14 16:04:58 -06:00
|
|
|
* THROTTLED: Device is being throttled.
|
2017-08-14 16:04:59 -06:00
|
|
|
* CACHE: Device is using a write-back cache.
|
2017-08-14 16:04:53 -06:00
|
|
|
*/
|
|
|
|
enum nullb_device_flags {
|
|
|
|
NULLB_DEV_FL_CONFIGURED = 0,
|
|
|
|
NULLB_DEV_FL_UP = 1,
|
2017-08-14 16:04:58 -06:00
|
|
|
NULLB_DEV_FL_THROTTLED = 2,
|
2017-08-14 16:04:59 -06:00
|
|
|
NULLB_DEV_FL_CACHE = 3,
|
2017-08-14 16:04:53 -06:00
|
|
|
};
|
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
/*
|
|
|
|
* nullb_page is a page in memory for nullb devices.
|
|
|
|
*
|
|
|
|
* @page: The page holding the data.
|
|
|
|
* @bitmap: The bitmap represents which sector in the page has data.
|
|
|
|
* Each bit represents one block size. For example, sector 8
|
|
|
|
* will use the 7th bit
|
2017-08-14 16:04:59 -06:00
|
|
|
* The highest 2 bits of bitmap are for special purpose. LOCK means the cache
|
|
|
|
* page is being flushing to storage. FREE means the cache page is freed and
|
|
|
|
* should be skipped from flushing to storage. Please see
|
|
|
|
* null_make_cache_space
|
2017-08-14 16:04:56 -06:00
|
|
|
*/
|
|
|
|
struct nullb_page {
|
|
|
|
struct page *page;
|
|
|
|
unsigned long bitmap;
|
|
|
|
};
|
2017-08-14 16:04:59 -06:00
|
|
|
#define NULLB_PAGE_LOCK (sizeof(unsigned long) * 8 - 1)
|
|
|
|
#define NULLB_PAGE_FREE (sizeof(unsigned long) * 8 - 2)
|
2017-08-14 16:04:56 -06:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
struct nullb_device {
|
|
|
|
struct nullb *nullb;
|
2017-08-14 16:04:53 -06:00
|
|
|
struct config_item item;
|
2017-08-14 16:04:56 -06:00
|
|
|
struct radix_tree_root data; /* data stored in the disk */
|
2017-08-14 16:04:59 -06:00
|
|
|
struct radix_tree_root cache; /* disk cache data */
|
2017-08-14 16:04:53 -06:00
|
|
|
unsigned long flags; /* device flags */
|
2017-08-14 16:04:59 -06:00
|
|
|
unsigned int curr_cache;
|
2017-08-14 16:05:00 -06:00
|
|
|
struct badblocks badblocks;
|
2017-08-14 16:04:52 -06:00
|
|
|
|
|
|
|
unsigned long size; /* device size in MB */
|
|
|
|
unsigned long completion_nsec; /* time in ns to complete a request */
|
2017-08-14 16:04:59 -06:00
|
|
|
unsigned long cache_size; /* disk cache size in MB */
|
2017-08-14 16:04:52 -06:00
|
|
|
unsigned int submit_queues; /* number of submission queues */
|
|
|
|
unsigned int home_node; /* home node for the device */
|
|
|
|
unsigned int queue_mode; /* block interface */
|
|
|
|
unsigned int blocksize; /* block size */
|
|
|
|
unsigned int irqmode; /* IRQ completion handler */
|
|
|
|
unsigned int hw_queue_depth; /* queue depth */
|
2017-08-14 16:04:54 -06:00
|
|
|
unsigned int index; /* index of the disk, only valid with a disk */
|
2017-08-14 16:04:58 -06:00
|
|
|
unsigned int mbps; /* Bandwidth throttle cap (in MB/s) */
|
2017-08-14 16:04:52 -06:00
|
|
|
bool use_lightnvm; /* register as a LightNVM device */
|
|
|
|
bool blocking; /* blocking blk-mq device */
|
|
|
|
bool use_per_node_hctx; /* use per-node allocation for hardware context */
|
2017-08-14 16:04:54 -06:00
|
|
|
bool power; /* power on/off the device */
|
2017-08-14 16:04:56 -06:00
|
|
|
bool memory_backed; /* if data is stored in memory */
|
2017-08-14 16:04:57 -06:00
|
|
|
bool discard; /* if support discard */
|
2017-08-14 16:04:52 -06:00
|
|
|
};
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
struct nullb {
|
2017-08-14 16:04:52 -06:00
|
|
|
struct nullb_device *dev;
|
2013-10-25 04:52:25 -06:00
|
|
|
struct list_head list;
|
|
|
|
unsigned int index;
|
|
|
|
struct request_queue *q;
|
|
|
|
struct gendisk *disk;
|
2016-09-16 06:25:07 -06:00
|
|
|
struct nvm_dev *ndev;
|
2017-06-20 14:22:01 -06:00
|
|
|
struct blk_mq_tag_set *tag_set;
|
|
|
|
struct blk_mq_tag_set __tag_set;
|
2013-10-25 04:52:25 -06:00
|
|
|
unsigned int queue_depth;
|
2017-08-14 16:04:58 -06:00
|
|
|
atomic_long_t cur_bytes;
|
|
|
|
struct hrtimer bw_timer;
|
2017-08-14 16:04:59 -06:00
|
|
|
unsigned long cache_flush_pos;
|
2013-10-25 04:52:25 -06:00
|
|
|
spinlock_t lock;
|
|
|
|
|
|
|
|
struct nullb_queue *queues;
|
|
|
|
unsigned int nr_queues;
|
2015-11-12 12:25:10 -07:00
|
|
|
char disk_name[DISK_NAME_LEN];
|
2013-10-25 04:52:25 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
static LIST_HEAD(nullb_list);
|
|
|
|
static struct mutex lock;
|
|
|
|
static int null_major;
|
2017-08-14 16:04:55 -06:00
|
|
|
static DEFINE_IDA(nullb_indexes);
|
2015-11-19 04:50:08 -07:00
|
|
|
static struct kmem_cache *ppa_cache;
|
2017-06-20 14:22:01 -06:00
|
|
|
static struct blk_mq_tag_set tag_set;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
enum {
|
|
|
|
NULL_IRQ_NONE = 0,
|
|
|
|
NULL_IRQ_SOFTIRQ = 1,
|
|
|
|
NULL_IRQ_TIMER = 2,
|
2014-02-10 04:24:40 -07:00
|
|
|
};
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2014-02-10 04:24:40 -07:00
|
|
|
enum {
|
2013-10-25 04:52:25 -06:00
|
|
|
NULL_Q_BIO = 0,
|
|
|
|
NULL_Q_RQ = 1,
|
|
|
|
NULL_Q_MQ = 2,
|
|
|
|
};
|
|
|
|
|
2017-09-29 19:49:21 -06:00
|
|
|
static int g_no_sched;
|
|
|
|
module_param_named(no_sched, g_no_sched, int, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(no_sched, "No io scheduler");
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static int g_submit_queues = 1;
|
|
|
|
module_param_named(submit_queues, g_submit_queues, int, S_IRUGO);
|
2013-10-25 04:52:25 -06:00
|
|
|
MODULE_PARM_DESC(submit_queues, "Number of submission queues");
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static int g_home_node = NUMA_NO_NODE;
|
|
|
|
module_param_named(home_node, g_home_node, int, S_IRUGO);
|
2013-10-25 04:52:25 -06:00
|
|
|
MODULE_PARM_DESC(home_node, "Home node for the device");
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static int g_queue_mode = NULL_Q_MQ;
|
2014-11-26 14:45:48 -07:00
|
|
|
|
|
|
|
static int null_param_store_val(const char *str, int *val, int min, int max)
|
|
|
|
{
|
|
|
|
int ret, new_val;
|
|
|
|
|
|
|
|
ret = kstrtoint(str, 10, &new_val);
|
|
|
|
if (ret)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (new_val < min || new_val > max)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
*val = new_val;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
|
|
|
|
{
|
2017-08-14 16:04:52 -06:00
|
|
|
return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
|
2014-11-26 14:45:48 -07:00
|
|
|
}
|
|
|
|
|
2015-05-26 19:39:38 -06:00
|
|
|
static const struct kernel_param_ops null_queue_mode_param_ops = {
|
2014-11-26 14:45:48 -07:00
|
|
|
.set = null_set_queue_mode,
|
|
|
|
.get = param_get_int,
|
|
|
|
};
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, S_IRUGO);
|
2014-06-11 15:13:50 -06:00
|
|
|
MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static int g_gb = 250;
|
|
|
|
module_param_named(gb, g_gb, int, S_IRUGO);
|
2013-10-25 04:52:25 -06:00
|
|
|
MODULE_PARM_DESC(gb, "Size in GB");
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static int g_bs = 512;
|
|
|
|
module_param_named(bs, g_bs, int, S_IRUGO);
|
2013-10-25 04:52:25 -06:00
|
|
|
MODULE_PARM_DESC(bs, "Block size (in bytes)");
|
|
|
|
|
2017-06-20 14:22:01 -06:00
|
|
|
static int nr_devices = 1;
|
2013-10-25 04:52:25 -06:00
|
|
|
module_param(nr_devices, int, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(nr_devices, "Number of devices to register");
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static bool g_use_lightnvm;
|
|
|
|
module_param_named(use_lightnvm, g_use_lightnvm, bool, S_IRUGO);
|
2015-11-12 12:25:10 -07:00
|
|
|
MODULE_PARM_DESC(use_lightnvm, "Register as a LightNVM device");
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static bool g_blocking;
|
|
|
|
module_param_named(blocking, g_blocking, bool, S_IRUGO);
|
2017-03-30 13:44:26 -06:00
|
|
|
MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
|
|
|
|
|
2017-06-20 14:22:01 -06:00
|
|
|
static bool shared_tags;
|
|
|
|
module_param(shared_tags, bool, S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static int g_irqmode = NULL_IRQ_SOFTIRQ;
|
2014-11-26 14:45:48 -07:00
|
|
|
|
|
|
|
static int null_set_irqmode(const char *str, const struct kernel_param *kp)
|
|
|
|
{
|
2017-08-14 16:04:52 -06:00
|
|
|
return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
|
2014-11-26 14:45:48 -07:00
|
|
|
NULL_IRQ_TIMER);
|
|
|
|
}
|
|
|
|
|
2015-05-26 19:39:38 -06:00
|
|
|
static const struct kernel_param_ops null_irqmode_param_ops = {
|
2014-11-26 14:45:48 -07:00
|
|
|
.set = null_set_irqmode,
|
|
|
|
.get = param_get_int,
|
|
|
|
};
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, S_IRUGO);
|
2013-10-25 04:52:25 -06:00
|
|
|
MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static unsigned long g_completion_nsec = 10000;
|
|
|
|
module_param_named(completion_nsec, g_completion_nsec, ulong, S_IRUGO);
|
2013-10-25 04:52:25 -06:00
|
|
|
MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static int g_hw_queue_depth = 64;
|
|
|
|
module_param_named(hw_queue_depth, g_hw_queue_depth, int, S_IRUGO);
|
2013-10-25 04:52:25 -06:00
|
|
|
MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static bool g_use_per_node_hctx;
|
|
|
|
module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, S_IRUGO);
|
2013-12-20 16:11:00 -07:00
|
|
|
MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:53 -06:00
|
|
|
static struct nullb_device *null_alloc_dev(void);
|
|
|
|
static void null_free_dev(struct nullb_device *dev);
|
2017-08-14 16:04:54 -06:00
|
|
|
static void null_del_dev(struct nullb *nullb);
|
|
|
|
static int null_add_dev(struct nullb_device *dev);
|
2017-08-14 16:04:59 -06:00
|
|
|
static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
|
2017-08-14 16:04:53 -06:00
|
|
|
|
|
|
|
static inline struct nullb_device *to_nullb_device(struct config_item *item)
|
|
|
|
{
|
|
|
|
return item ? container_of(item, struct nullb_device, item) : NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
|
|
|
|
{
|
|
|
|
return snprintf(page, PAGE_SIZE, "%u\n", val);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
|
|
|
|
char *page)
|
|
|
|
{
|
|
|
|
return snprintf(page, PAGE_SIZE, "%lu\n", val);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
|
|
|
|
{
|
|
|
|
return snprintf(page, PAGE_SIZE, "%u\n", val);
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t nullb_device_uint_attr_store(unsigned int *val,
|
|
|
|
const char *page, size_t count)
|
|
|
|
{
|
|
|
|
unsigned int tmp;
|
|
|
|
int result;
|
|
|
|
|
|
|
|
result = kstrtouint(page, 0, &tmp);
|
|
|
|
if (result)
|
|
|
|
return result;
|
|
|
|
|
|
|
|
*val = tmp;
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
|
|
|
|
const char *page, size_t count)
|
|
|
|
{
|
|
|
|
int result;
|
|
|
|
unsigned long tmp;
|
|
|
|
|
|
|
|
result = kstrtoul(page, 0, &tmp);
|
|
|
|
if (result)
|
|
|
|
return result;
|
|
|
|
|
|
|
|
*val = tmp;
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
|
|
|
|
size_t count)
|
|
|
|
{
|
|
|
|
bool tmp;
|
|
|
|
int result;
|
|
|
|
|
|
|
|
result = kstrtobool(page, &tmp);
|
|
|
|
if (result)
|
|
|
|
return result;
|
|
|
|
|
|
|
|
*val = tmp;
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* The following macro should only be used with TYPE = {uint, ulong, bool}. */
|
|
|
|
#define NULLB_DEVICE_ATTR(NAME, TYPE) \
|
|
|
|
static ssize_t \
|
|
|
|
nullb_device_##NAME##_show(struct config_item *item, char *page) \
|
|
|
|
{ \
|
|
|
|
return nullb_device_##TYPE##_attr_show( \
|
|
|
|
to_nullb_device(item)->NAME, page); \
|
|
|
|
} \
|
|
|
|
static ssize_t \
|
|
|
|
nullb_device_##NAME##_store(struct config_item *item, const char *page, \
|
|
|
|
size_t count) \
|
|
|
|
{ \
|
|
|
|
if (test_bit(NULLB_DEV_FL_CONFIGURED, &to_nullb_device(item)->flags)) \
|
|
|
|
return -EBUSY; \
|
|
|
|
return nullb_device_##TYPE##_attr_store( \
|
|
|
|
&to_nullb_device(item)->NAME, page, count); \
|
|
|
|
} \
|
|
|
|
CONFIGFS_ATTR(nullb_device_, NAME);
|
|
|
|
|
|
|
|
NULLB_DEVICE_ATTR(size, ulong);
|
|
|
|
NULLB_DEVICE_ATTR(completion_nsec, ulong);
|
|
|
|
NULLB_DEVICE_ATTR(submit_queues, uint);
|
|
|
|
NULLB_DEVICE_ATTR(home_node, uint);
|
|
|
|
NULLB_DEVICE_ATTR(queue_mode, uint);
|
|
|
|
NULLB_DEVICE_ATTR(blocksize, uint);
|
|
|
|
NULLB_DEVICE_ATTR(irqmode, uint);
|
|
|
|
NULLB_DEVICE_ATTR(hw_queue_depth, uint);
|
2017-08-14 16:04:54 -06:00
|
|
|
NULLB_DEVICE_ATTR(index, uint);
|
2017-08-14 16:04:53 -06:00
|
|
|
NULLB_DEVICE_ATTR(use_lightnvm, bool);
|
|
|
|
NULLB_DEVICE_ATTR(blocking, bool);
|
|
|
|
NULLB_DEVICE_ATTR(use_per_node_hctx, bool);
|
2017-08-14 16:04:56 -06:00
|
|
|
NULLB_DEVICE_ATTR(memory_backed, bool);
|
2017-08-14 16:04:57 -06:00
|
|
|
NULLB_DEVICE_ATTR(discard, bool);
|
2017-08-14 16:04:58 -06:00
|
|
|
NULLB_DEVICE_ATTR(mbps, uint);
|
2017-08-14 16:04:59 -06:00
|
|
|
NULLB_DEVICE_ATTR(cache_size, ulong);
|
2017-08-14 16:04:53 -06:00
|
|
|
|
2017-08-14 16:04:54 -06:00
|
|
|
static ssize_t nullb_device_power_show(struct config_item *item, char *page)
|
|
|
|
{
|
|
|
|
return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t nullb_device_power_store(struct config_item *item,
|
|
|
|
const char *page, size_t count)
|
|
|
|
{
|
|
|
|
struct nullb_device *dev = to_nullb_device(item);
|
|
|
|
bool newp = false;
|
|
|
|
ssize_t ret;
|
|
|
|
|
|
|
|
ret = nullb_device_bool_attr_store(&newp, page, count);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (!dev->power && newp) {
|
|
|
|
if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
|
|
|
|
return count;
|
|
|
|
if (null_add_dev(dev)) {
|
|
|
|
clear_bit(NULLB_DEV_FL_UP, &dev->flags);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
|
|
|
|
dev->power = newp;
|
2017-08-28 15:06:31 -06:00
|
|
|
} else if (dev->power && !newp) {
|
2017-08-14 16:04:54 -06:00
|
|
|
mutex_lock(&lock);
|
|
|
|
dev->power = newp;
|
|
|
|
null_del_dev(dev->nullb);
|
|
|
|
mutex_unlock(&lock);
|
|
|
|
clear_bit(NULLB_DEV_FL_UP, &dev->flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
|
|
|
CONFIGFS_ATTR(nullb_device_, power);
|
|
|
|
|
2017-08-14 16:05:00 -06:00
|
|
|
static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
|
|
|
|
{
|
|
|
|
struct nullb_device *t_dev = to_nullb_device(item);
|
|
|
|
|
|
|
|
return badblocks_show(&t_dev->badblocks, page, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t nullb_device_badblocks_store(struct config_item *item,
|
|
|
|
const char *page, size_t count)
|
|
|
|
{
|
|
|
|
struct nullb_device *t_dev = to_nullb_device(item);
|
|
|
|
char *orig, *buf, *tmp;
|
|
|
|
u64 start, end;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
orig = kstrndup(page, count, GFP_KERNEL);
|
|
|
|
if (!orig)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
buf = strstrip(orig);
|
|
|
|
|
|
|
|
ret = -EINVAL;
|
|
|
|
if (buf[0] != '+' && buf[0] != '-')
|
|
|
|
goto out;
|
|
|
|
tmp = strchr(&buf[1], '-');
|
|
|
|
if (!tmp)
|
|
|
|
goto out;
|
|
|
|
*tmp = '\0';
|
|
|
|
ret = kstrtoull(buf + 1, 0, &start);
|
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
ret = kstrtoull(tmp + 1, 0, &end);
|
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
ret = -EINVAL;
|
|
|
|
if (start > end)
|
|
|
|
goto out;
|
|
|
|
/* enable badblocks */
|
|
|
|
cmpxchg(&t_dev->badblocks.shift, -1, 0);
|
|
|
|
if (buf[0] == '+')
|
|
|
|
ret = badblocks_set(&t_dev->badblocks, start,
|
|
|
|
end - start + 1, 1);
|
|
|
|
else
|
|
|
|
ret = badblocks_clear(&t_dev->badblocks, start,
|
|
|
|
end - start + 1);
|
|
|
|
if (ret == 0)
|
|
|
|
ret = count;
|
|
|
|
out:
|
|
|
|
kfree(orig);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
CONFIGFS_ATTR(nullb_device_, badblocks);
|
|
|
|
|
2017-08-14 16:04:53 -06:00
|
|
|
static struct configfs_attribute *nullb_device_attrs[] = {
|
|
|
|
&nullb_device_attr_size,
|
|
|
|
&nullb_device_attr_completion_nsec,
|
|
|
|
&nullb_device_attr_submit_queues,
|
|
|
|
&nullb_device_attr_home_node,
|
|
|
|
&nullb_device_attr_queue_mode,
|
|
|
|
&nullb_device_attr_blocksize,
|
|
|
|
&nullb_device_attr_irqmode,
|
|
|
|
&nullb_device_attr_hw_queue_depth,
|
2017-08-14 16:04:54 -06:00
|
|
|
&nullb_device_attr_index,
|
2017-08-14 16:04:53 -06:00
|
|
|
&nullb_device_attr_use_lightnvm,
|
|
|
|
&nullb_device_attr_blocking,
|
|
|
|
&nullb_device_attr_use_per_node_hctx,
|
2017-08-14 16:04:54 -06:00
|
|
|
&nullb_device_attr_power,
|
2017-08-14 16:04:56 -06:00
|
|
|
&nullb_device_attr_memory_backed,
|
2017-08-14 16:04:57 -06:00
|
|
|
&nullb_device_attr_discard,
|
2017-08-14 16:04:58 -06:00
|
|
|
&nullb_device_attr_mbps,
|
2017-08-14 16:04:59 -06:00
|
|
|
&nullb_device_attr_cache_size,
|
2017-08-14 16:05:00 -06:00
|
|
|
&nullb_device_attr_badblocks,
|
2017-08-14 16:04:53 -06:00
|
|
|
NULL,
|
|
|
|
};
|
|
|
|
|
|
|
|
static void nullb_device_release(struct config_item *item)
|
|
|
|
{
|
2017-08-14 16:04:56 -06:00
|
|
|
struct nullb_device *dev = to_nullb_device(item);
|
|
|
|
|
2017-08-14 16:05:00 -06:00
|
|
|
badblocks_exit(&dev->badblocks);
|
2017-08-14 16:04:59 -06:00
|
|
|
null_free_device_storage(dev, false);
|
2017-08-14 16:04:56 -06:00
|
|
|
null_free_dev(dev);
|
2017-08-14 16:04:53 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct configfs_item_operations nullb_device_ops = {
|
|
|
|
.release = nullb_device_release,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct config_item_type nullb_device_type = {
|
|
|
|
.ct_item_ops = &nullb_device_ops,
|
|
|
|
.ct_attrs = nullb_device_attrs,
|
|
|
|
.ct_owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct
|
|
|
|
config_item *nullb_group_make_item(struct config_group *group, const char *name)
|
|
|
|
{
|
|
|
|
struct nullb_device *dev;
|
|
|
|
|
|
|
|
dev = null_alloc_dev();
|
|
|
|
if (!dev)
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
|
|
|
|
config_item_init_type_name(&dev->item, name, &nullb_device_type);
|
|
|
|
|
|
|
|
return &dev->item;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nullb_group_drop_item(struct config_group *group, struct config_item *item)
|
|
|
|
{
|
2017-08-14 16:04:54 -06:00
|
|
|
struct nullb_device *dev = to_nullb_device(item);
|
|
|
|
|
|
|
|
if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
|
|
|
|
mutex_lock(&lock);
|
|
|
|
dev->power = false;
|
|
|
|
null_del_dev(dev->nullb);
|
|
|
|
mutex_unlock(&lock);
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:53 -06:00
|
|
|
config_item_put(item);
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t memb_group_features_show(struct config_item *item, char *page)
|
|
|
|
{
|
2017-08-14 16:05:00 -06:00
|
|
|
return snprintf(page, PAGE_SIZE, "memory_backed,discard,bandwidth,cache,badblocks\n");
|
2017-08-14 16:04:53 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
CONFIGFS_ATTR_RO(memb_group_, features);
|
|
|
|
|
|
|
|
static struct configfs_attribute *nullb_group_attrs[] = {
|
|
|
|
&memb_group_attr_features,
|
|
|
|
NULL,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct configfs_group_operations nullb_group_ops = {
|
|
|
|
.make_item = nullb_group_make_item,
|
|
|
|
.drop_item = nullb_group_drop_item,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct config_item_type nullb_group_type = {
|
|
|
|
.ct_group_ops = &nullb_group_ops,
|
|
|
|
.ct_attrs = nullb_group_attrs,
|
|
|
|
.ct_owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct configfs_subsystem nullb_subsys = {
|
|
|
|
.su_group = {
|
|
|
|
.cg_item = {
|
|
|
|
.ci_namebuf = "nullb",
|
|
|
|
.ci_type = &nullb_group_type,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
};
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
static inline int null_cache_active(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static struct nullb_device *null_alloc_dev(void)
|
|
|
|
{
|
|
|
|
struct nullb_device *dev;
|
|
|
|
|
|
|
|
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
|
|
|
|
if (!dev)
|
|
|
|
return NULL;
|
2017-08-14 16:04:56 -06:00
|
|
|
INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
|
2017-08-14 16:04:59 -06:00
|
|
|
INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
|
2017-08-14 16:05:00 -06:00
|
|
|
if (badblocks_init(&dev->badblocks, 0)) {
|
|
|
|
kfree(dev);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
dev->size = g_gb * 1024;
|
|
|
|
dev->completion_nsec = g_completion_nsec;
|
|
|
|
dev->submit_queues = g_submit_queues;
|
|
|
|
dev->home_node = g_home_node;
|
|
|
|
dev->queue_mode = g_queue_mode;
|
|
|
|
dev->blocksize = g_bs;
|
|
|
|
dev->irqmode = g_irqmode;
|
|
|
|
dev->hw_queue_depth = g_hw_queue_depth;
|
|
|
|
dev->use_lightnvm = g_use_lightnvm;
|
|
|
|
dev->blocking = g_blocking;
|
|
|
|
dev->use_per_node_hctx = g_use_per_node_hctx;
|
|
|
|
return dev;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_free_dev(struct nullb_device *dev)
|
|
|
|
{
|
|
|
|
kfree(dev);
|
|
|
|
}
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
static void put_tag(struct nullb_queue *nq, unsigned int tag)
|
|
|
|
{
|
|
|
|
clear_bit_unlock(tag, nq->tag_map);
|
|
|
|
|
|
|
|
if (waitqueue_active(&nq->wait))
|
|
|
|
wake_up(&nq->wait);
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned int get_tag(struct nullb_queue *nq)
|
|
|
|
{
|
|
|
|
unsigned int tag;
|
|
|
|
|
|
|
|
do {
|
|
|
|
tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
|
|
|
|
if (tag >= nq->queue_depth)
|
|
|
|
return -1U;
|
|
|
|
} while (test_and_set_bit_lock(tag, nq->tag_map));
|
|
|
|
|
|
|
|
return tag;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void free_cmd(struct nullb_cmd *cmd)
|
|
|
|
{
|
|
|
|
put_tag(cmd->nq, cmd->tag);
|
|
|
|
}
|
|
|
|
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer);
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
|
|
|
|
{
|
|
|
|
struct nullb_cmd *cmd;
|
|
|
|
unsigned int tag;
|
|
|
|
|
|
|
|
tag = get_tag(nq);
|
|
|
|
if (tag != -1U) {
|
|
|
|
cmd = &nq->cmds[tag];
|
|
|
|
cmd->tag = tag;
|
|
|
|
cmd->nq = nq;
|
2017-08-14 16:04:52 -06:00
|
|
|
if (nq->dev->irqmode == NULL_IRQ_TIMER) {
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
|
|
|
|
HRTIMER_MODE_REL);
|
|
|
|
cmd->timer.function = null_cmd_timer_expired;
|
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
return cmd;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
|
|
|
|
{
|
|
|
|
struct nullb_cmd *cmd;
|
|
|
|
DEFINE_WAIT(wait);
|
|
|
|
|
|
|
|
cmd = __alloc_cmd(nq);
|
|
|
|
if (cmd || !can_wait)
|
|
|
|
return cmd;
|
|
|
|
|
|
|
|
do {
|
|
|
|
prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
|
|
|
|
cmd = __alloc_cmd(nq);
|
|
|
|
if (cmd)
|
|
|
|
break;
|
|
|
|
|
|
|
|
io_schedule();
|
|
|
|
} while (1);
|
|
|
|
|
|
|
|
finish_wait(&nq->wait, &wait);
|
|
|
|
return cmd;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void end_cmd(struct nullb_cmd *cmd)
|
|
|
|
{
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
struct request_queue *q = NULL;
|
2017-08-14 16:04:52 -06:00
|
|
|
int queue_mode = cmd->nq->dev->queue_mode;
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
|
2015-12-15 02:56:40 -07:00
|
|
|
if (cmd->rq)
|
|
|
|
q = cmd->rq->q;
|
|
|
|
|
2014-02-10 04:24:40 -07:00
|
|
|
switch (queue_mode) {
|
|
|
|
case NULL_Q_MQ:
|
2017-08-14 16:04:56 -06:00
|
|
|
blk_mq_end_request(cmd->rq, cmd->error);
|
2014-02-10 04:24:40 -07:00
|
|
|
return;
|
|
|
|
case NULL_Q_RQ:
|
|
|
|
INIT_LIST_HEAD(&cmd->rq->queuelist);
|
2017-08-14 16:04:56 -06:00
|
|
|
blk_end_request_all(cmd->rq, cmd->error);
|
2014-02-10 04:24:40 -07:00
|
|
|
break;
|
|
|
|
case NULL_Q_BIO:
|
2017-08-14 16:04:56 -06:00
|
|
|
cmd->bio->bi_status = cmd->error;
|
2015-07-20 07:29:37 -06:00
|
|
|
bio_endio(cmd->bio);
|
2015-12-28 13:02:47 -07:00
|
|
|
break;
|
2014-02-10 04:24:40 -07:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2015-12-28 13:02:47 -07:00
|
|
|
free_cmd(cmd);
|
|
|
|
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
/* Restart queue if needed, as we are freeing a tag */
|
2015-12-28 13:02:47 -07:00
|
|
|
if (queue_mode == NULL_Q_RQ && blk_queue_stopped(q)) {
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(q->queue_lock, flags);
|
2015-12-28 13:02:47 -07:00
|
|
|
blk_start_queue_async(q);
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
spin_unlock_irqrestore(q->queue_lock, flags);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
null_blk: guarantee device restart in all irq modes
In single-queue (block layer) mode,the function null_rq_prep_fn stops
the device if alloc_cmd fails. Then, once stopped, the device must be
restarted on the next command completion, so that the request(s) for
which alloc_cmd failed can be requeued. Otherwise the device hangs.
Unfortunately, device restart is currently performed only for delayed
completions, i.e., in irqmode==2. This fact causes hangs, for the
above reasons, with the other irqmodes in combination with single-queue
block layer.
This commits addresses this issue by making sure that, if stopped, the
device is properly restarted for all irqmodes on completions.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna AVanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:18 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
end_cmd(container_of(timer, struct nullb_cmd, timer));
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
return HRTIMER_NORESTART;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_cmd_end_timer(struct nullb_cmd *cmd)
|
|
|
|
{
|
2017-08-14 16:04:52 -06:00
|
|
|
ktime_t kt = cmd->nq->dev->completion_nsec;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void null_softirq_done_fn(struct request *rq)
|
|
|
|
{
|
2017-08-14 16:04:52 -06:00
|
|
|
struct nullb *nullb = rq->q->queuedata;
|
|
|
|
|
|
|
|
if (nullb->dev->queue_mode == NULL_Q_MQ)
|
2014-06-16 11:40:25 -06:00
|
|
|
end_cmd(blk_mq_rq_to_pdu(rq));
|
|
|
|
else
|
|
|
|
end_cmd(rq->special);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
static struct nullb_page *null_alloc_page(gfp_t gfp_flags)
|
|
|
|
{
|
|
|
|
struct nullb_page *t_page;
|
|
|
|
|
|
|
|
t_page = kmalloc(sizeof(struct nullb_page), gfp_flags);
|
|
|
|
if (!t_page)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
t_page->page = alloc_pages(gfp_flags, 0);
|
|
|
|
if (!t_page->page)
|
|
|
|
goto out_freepage;
|
|
|
|
|
|
|
|
t_page->bitmap = 0;
|
|
|
|
return t_page;
|
|
|
|
out_freepage:
|
|
|
|
kfree(t_page);
|
|
|
|
out:
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_free_page(struct nullb_page *t_page)
|
|
|
|
{
|
2017-08-14 16:04:59 -06:00
|
|
|
__set_bit(NULLB_PAGE_FREE, &t_page->bitmap);
|
|
|
|
if (test_bit(NULLB_PAGE_LOCK, &t_page->bitmap))
|
|
|
|
return;
|
2017-08-14 16:04:56 -06:00
|
|
|
__free_page(t_page->page);
|
|
|
|
kfree(t_page);
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
static void null_free_sector(struct nullb *nullb, sector_t sector,
|
|
|
|
bool is_cache)
|
2017-08-14 16:04:56 -06:00
|
|
|
{
|
|
|
|
unsigned int sector_bit;
|
|
|
|
u64 idx;
|
|
|
|
struct nullb_page *t_page, *ret;
|
|
|
|
struct radix_tree_root *root;
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
|
2017-08-14 16:04:56 -06:00
|
|
|
idx = sector >> PAGE_SECTORS_SHIFT;
|
|
|
|
sector_bit = (sector & SECTOR_MASK);
|
|
|
|
|
|
|
|
t_page = radix_tree_lookup(root, idx);
|
|
|
|
if (t_page) {
|
|
|
|
__clear_bit(sector_bit, &t_page->bitmap);
|
|
|
|
|
|
|
|
if (!t_page->bitmap) {
|
|
|
|
ret = radix_tree_delete_item(root, idx, t_page);
|
|
|
|
WARN_ON(ret != t_page);
|
|
|
|
null_free_page(ret);
|
2017-08-14 16:04:59 -06:00
|
|
|
if (is_cache)
|
|
|
|
nullb->dev->curr_cache -= PAGE_SIZE;
|
2017-08-14 16:04:56 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
|
2017-08-14 16:04:59 -06:00
|
|
|
struct nullb_page *t_page, bool is_cache)
|
2017-08-14 16:04:56 -06:00
|
|
|
{
|
|
|
|
struct radix_tree_root *root;
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
|
2017-08-14 16:04:56 -06:00
|
|
|
|
|
|
|
if (radix_tree_insert(root, idx, t_page)) {
|
|
|
|
null_free_page(t_page);
|
|
|
|
t_page = radix_tree_lookup(root, idx);
|
|
|
|
WARN_ON(!t_page || t_page->page->index != idx);
|
2017-08-14 16:04:59 -06:00
|
|
|
} else if (is_cache)
|
|
|
|
nullb->dev->curr_cache += PAGE_SIZE;
|
2017-08-14 16:04:56 -06:00
|
|
|
|
|
|
|
return t_page;
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
|
2017-08-14 16:04:56 -06:00
|
|
|
{
|
|
|
|
unsigned long pos = 0;
|
|
|
|
int nr_pages;
|
|
|
|
struct nullb_page *ret, *t_pages[FREE_BATCH];
|
|
|
|
struct radix_tree_root *root;
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
root = is_cache ? &dev->cache : &dev->data;
|
2017-08-14 16:04:56 -06:00
|
|
|
|
|
|
|
do {
|
|
|
|
int i;
|
|
|
|
|
|
|
|
nr_pages = radix_tree_gang_lookup(root,
|
|
|
|
(void **)t_pages, pos, FREE_BATCH);
|
|
|
|
|
|
|
|
for (i = 0; i < nr_pages; i++) {
|
|
|
|
pos = t_pages[i]->page->index;
|
|
|
|
ret = radix_tree_delete_item(root, pos, t_pages[i]);
|
|
|
|
WARN_ON(ret != t_pages[i]);
|
|
|
|
null_free_page(ret);
|
|
|
|
}
|
|
|
|
|
|
|
|
pos++;
|
|
|
|
} while (nr_pages == FREE_BATCH);
|
2017-08-14 16:04:59 -06:00
|
|
|
|
|
|
|
if (is_cache)
|
|
|
|
dev->curr_cache = 0;
|
2017-08-14 16:04:56 -06:00
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
static struct nullb_page *__null_lookup_page(struct nullb *nullb,
|
|
|
|
sector_t sector, bool for_write, bool is_cache)
|
2017-08-14 16:04:56 -06:00
|
|
|
{
|
|
|
|
unsigned int sector_bit;
|
|
|
|
u64 idx;
|
|
|
|
struct nullb_page *t_page;
|
2017-08-14 16:04:59 -06:00
|
|
|
struct radix_tree_root *root;
|
2017-08-14 16:04:56 -06:00
|
|
|
|
|
|
|
idx = sector >> PAGE_SECTORS_SHIFT;
|
|
|
|
sector_bit = (sector & SECTOR_MASK);
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
|
|
|
|
t_page = radix_tree_lookup(root, idx);
|
2017-08-14 16:04:56 -06:00
|
|
|
WARN_ON(t_page && t_page->page->index != idx);
|
|
|
|
|
|
|
|
if (t_page && (for_write || test_bit(sector_bit, &t_page->bitmap)))
|
|
|
|
return t_page;
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
static struct nullb_page *null_lookup_page(struct nullb *nullb,
|
|
|
|
sector_t sector, bool for_write, bool ignore_cache)
|
|
|
|
{
|
|
|
|
struct nullb_page *page = NULL;
|
|
|
|
|
|
|
|
if (!ignore_cache)
|
|
|
|
page = __null_lookup_page(nullb, sector, for_write, true);
|
|
|
|
if (page)
|
|
|
|
return page;
|
|
|
|
return __null_lookup_page(nullb, sector, for_write, false);
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
static struct nullb_page *null_insert_page(struct nullb *nullb,
|
2017-08-14 16:04:59 -06:00
|
|
|
sector_t sector, bool ignore_cache)
|
2017-08-14 16:04:56 -06:00
|
|
|
{
|
|
|
|
u64 idx;
|
|
|
|
struct nullb_page *t_page;
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
t_page = null_lookup_page(nullb, sector, true, ignore_cache);
|
2017-08-14 16:04:56 -06:00
|
|
|
if (t_page)
|
|
|
|
return t_page;
|
|
|
|
|
|
|
|
spin_unlock_irq(&nullb->lock);
|
|
|
|
|
|
|
|
t_page = null_alloc_page(GFP_NOIO);
|
|
|
|
if (!t_page)
|
|
|
|
goto out_lock;
|
|
|
|
|
|
|
|
if (radix_tree_preload(GFP_NOIO))
|
|
|
|
goto out_freepage;
|
|
|
|
|
|
|
|
spin_lock_irq(&nullb->lock);
|
|
|
|
idx = sector >> PAGE_SECTORS_SHIFT;
|
|
|
|
t_page->page->index = idx;
|
2017-08-14 16:04:59 -06:00
|
|
|
t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
|
2017-08-14 16:04:56 -06:00
|
|
|
radix_tree_preload_end();
|
|
|
|
|
|
|
|
return t_page;
|
|
|
|
out_freepage:
|
|
|
|
null_free_page(t_page);
|
|
|
|
out_lock:
|
|
|
|
spin_lock_irq(&nullb->lock);
|
2017-08-14 16:04:59 -06:00
|
|
|
return null_lookup_page(nullb, sector, true, ignore_cache);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
unsigned int offset;
|
|
|
|
u64 idx;
|
|
|
|
struct nullb_page *t_page, *ret;
|
|
|
|
void *dst, *src;
|
|
|
|
|
|
|
|
idx = c_page->page->index;
|
|
|
|
|
|
|
|
t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
|
|
|
|
|
|
|
|
__clear_bit(NULLB_PAGE_LOCK, &c_page->bitmap);
|
|
|
|
if (test_bit(NULLB_PAGE_FREE, &c_page->bitmap)) {
|
|
|
|
null_free_page(c_page);
|
|
|
|
if (t_page && t_page->bitmap == 0) {
|
|
|
|
ret = radix_tree_delete_item(&nullb->dev->data,
|
|
|
|
idx, t_page);
|
|
|
|
null_free_page(t_page);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!t_page)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
src = kmap_atomic(c_page->page);
|
|
|
|
dst = kmap_atomic(t_page->page);
|
|
|
|
|
|
|
|
for (i = 0; i < PAGE_SECTORS;
|
|
|
|
i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
|
|
|
|
if (test_bit(i, &c_page->bitmap)) {
|
|
|
|
offset = (i << SECTOR_SHIFT);
|
|
|
|
memcpy(dst + offset, src + offset,
|
|
|
|
nullb->dev->blocksize);
|
|
|
|
__set_bit(i, &t_page->bitmap);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
kunmap_atomic(dst);
|
|
|
|
kunmap_atomic(src);
|
|
|
|
|
|
|
|
ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
|
|
|
|
null_free_page(ret);
|
|
|
|
nullb->dev->curr_cache -= PAGE_SIZE;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int null_make_cache_space(struct nullb *nullb, unsigned long n)
|
2013-10-25 04:52:25 -06:00
|
|
|
{
|
2017-08-14 16:04:59 -06:00
|
|
|
int i, err, nr_pages;
|
|
|
|
struct nullb_page *c_pages[FREE_BATCH];
|
|
|
|
unsigned long flushed = 0, one_round;
|
|
|
|
|
|
|
|
again:
|
|
|
|
if ((nullb->dev->cache_size * 1024 * 1024) >
|
|
|
|
nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
|
|
|
|
(void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
|
|
|
|
/*
|
|
|
|
* nullb_flush_cache_page could unlock before using the c_pages. To
|
|
|
|
* avoid race, we don't allow page free
|
|
|
|
*/
|
|
|
|
for (i = 0; i < nr_pages; i++) {
|
|
|
|
nullb->cache_flush_pos = c_pages[i]->page->index;
|
|
|
|
/*
|
|
|
|
* We found the page which is being flushed to disk by other
|
|
|
|
* threads
|
|
|
|
*/
|
|
|
|
if (test_bit(NULLB_PAGE_LOCK, &c_pages[i]->bitmap))
|
|
|
|
c_pages[i] = NULL;
|
|
|
|
else
|
|
|
|
__set_bit(NULLB_PAGE_LOCK, &c_pages[i]->bitmap);
|
|
|
|
}
|
|
|
|
|
|
|
|
one_round = 0;
|
|
|
|
for (i = 0; i < nr_pages; i++) {
|
|
|
|
if (c_pages[i] == NULL)
|
|
|
|
continue;
|
|
|
|
err = null_flush_cache_page(nullb, c_pages[i]);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
one_round++;
|
|
|
|
}
|
|
|
|
flushed += one_round << PAGE_SHIFT;
|
|
|
|
|
|
|
|
if (n > flushed) {
|
|
|
|
if (nr_pages == 0)
|
|
|
|
nullb->cache_flush_pos = 0;
|
|
|
|
if (one_round == 0) {
|
|
|
|
/* give other threads a chance */
|
|
|
|
spin_unlock_irq(&nullb->lock);
|
|
|
|
spin_lock_irq(&nullb->lock);
|
|
|
|
}
|
|
|
|
goto again;
|
|
|
|
}
|
|
|
|
return 0;
|
2017-08-14 16:04:56 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int copy_to_nullb(struct nullb *nullb, struct page *source,
|
2017-08-14 16:04:59 -06:00
|
|
|
unsigned int off, sector_t sector, size_t n, bool is_fua)
|
2017-08-14 16:04:56 -06:00
|
|
|
{
|
|
|
|
size_t temp, count = 0;
|
|
|
|
unsigned int offset;
|
|
|
|
struct nullb_page *t_page;
|
|
|
|
void *dst, *src;
|
|
|
|
|
|
|
|
while (count < n) {
|
|
|
|
temp = min_t(size_t, nullb->dev->blocksize, n - count);
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
if (null_cache_active(nullb) && !is_fua)
|
|
|
|
null_make_cache_space(nullb, PAGE_SIZE);
|
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
|
2017-08-14 16:04:59 -06:00
|
|
|
t_page = null_insert_page(nullb, sector,
|
|
|
|
!null_cache_active(nullb) || is_fua);
|
2017-08-14 16:04:56 -06:00
|
|
|
if (!t_page)
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
src = kmap_atomic(source);
|
|
|
|
dst = kmap_atomic(t_page->page);
|
|
|
|
memcpy(dst + offset, src + off + count, temp);
|
|
|
|
kunmap_atomic(dst);
|
|
|
|
kunmap_atomic(src);
|
|
|
|
|
|
|
|
__set_bit(sector & SECTOR_MASK, &t_page->bitmap);
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
if (is_fua)
|
|
|
|
null_free_sector(nullb, sector, true);
|
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
count += temp;
|
|
|
|
sector += temp >> SECTOR_SHIFT;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int copy_from_nullb(struct nullb *nullb, struct page *dest,
|
|
|
|
unsigned int off, sector_t sector, size_t n)
|
|
|
|
{
|
|
|
|
size_t temp, count = 0;
|
|
|
|
unsigned int offset;
|
|
|
|
struct nullb_page *t_page;
|
|
|
|
void *dst, *src;
|
|
|
|
|
|
|
|
while (count < n) {
|
|
|
|
temp = min_t(size_t, nullb->dev->blocksize, n - count);
|
|
|
|
|
|
|
|
offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
|
2017-08-14 16:04:59 -06:00
|
|
|
t_page = null_lookup_page(nullb, sector, false,
|
|
|
|
!null_cache_active(nullb));
|
2017-08-14 16:04:56 -06:00
|
|
|
|
|
|
|
dst = kmap_atomic(dest);
|
|
|
|
if (!t_page) {
|
|
|
|
memset(dst + off + count, 0, temp);
|
|
|
|
goto next;
|
|
|
|
}
|
|
|
|
src = kmap_atomic(t_page->page);
|
|
|
|
memcpy(dst + off + count, src + offset, temp);
|
|
|
|
kunmap_atomic(src);
|
|
|
|
next:
|
|
|
|
kunmap_atomic(dst);
|
|
|
|
|
|
|
|
count += temp;
|
|
|
|
sector += temp >> SECTOR_SHIFT;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:57 -06:00
|
|
|
static void null_handle_discard(struct nullb *nullb, sector_t sector, size_t n)
|
|
|
|
{
|
|
|
|
size_t temp;
|
|
|
|
|
|
|
|
spin_lock_irq(&nullb->lock);
|
|
|
|
while (n > 0) {
|
|
|
|
temp = min_t(size_t, n, nullb->dev->blocksize);
|
2017-08-14 16:04:59 -06:00
|
|
|
null_free_sector(nullb, sector, false);
|
|
|
|
if (null_cache_active(nullb))
|
|
|
|
null_free_sector(nullb, sector, true);
|
2017-08-14 16:04:57 -06:00
|
|
|
sector += temp >> SECTOR_SHIFT;
|
|
|
|
n -= temp;
|
|
|
|
}
|
|
|
|
spin_unlock_irq(&nullb->lock);
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
static int null_handle_flush(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (!null_cache_active(nullb))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
spin_lock_irq(&nullb->lock);
|
|
|
|
while (true) {
|
|
|
|
err = null_make_cache_space(nullb,
|
|
|
|
nullb->dev->cache_size * 1024 * 1024);
|
|
|
|
if (err || nullb->dev->curr_cache == 0)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
WARN_ON(!radix_tree_empty(&nullb->dev->cache));
|
|
|
|
spin_unlock_irq(&nullb->lock);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
static int null_transfer(struct nullb *nullb, struct page *page,
|
2017-08-14 16:04:59 -06:00
|
|
|
unsigned int len, unsigned int off, bool is_write, sector_t sector,
|
|
|
|
bool is_fua)
|
2013-10-25 04:52:25 -06:00
|
|
|
{
|
2017-08-14 16:04:56 -06:00
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
if (!is_write) {
|
|
|
|
err = copy_from_nullb(nullb, page, off, sector, len);
|
|
|
|
flush_dcache_page(page);
|
|
|
|
} else {
|
|
|
|
flush_dcache_page(page);
|
2017-08-14 16:04:59 -06:00
|
|
|
err = copy_to_nullb(nullb, page, off, sector, len, is_fua);
|
2017-08-14 16:04:56 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int null_handle_rq(struct nullb_cmd *cmd)
|
|
|
|
{
|
|
|
|
struct request *rq = cmd->rq;
|
|
|
|
struct nullb *nullb = cmd->nq->dev->nullb;
|
|
|
|
int err;
|
|
|
|
unsigned int len;
|
|
|
|
sector_t sector;
|
|
|
|
struct req_iterator iter;
|
|
|
|
struct bio_vec bvec;
|
|
|
|
|
|
|
|
sector = blk_rq_pos(rq);
|
|
|
|
|
2017-08-14 16:04:57 -06:00
|
|
|
if (req_op(rq) == REQ_OP_DISCARD) {
|
|
|
|
null_handle_discard(nullb, sector, blk_rq_bytes(rq));
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
spin_lock_irq(&nullb->lock);
|
|
|
|
rq_for_each_segment(bvec, rq, iter) {
|
|
|
|
len = bvec.bv_len;
|
|
|
|
err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
|
2017-08-14 16:04:59 -06:00
|
|
|
op_is_write(req_op(rq)), sector,
|
|
|
|
req_op(rq) & REQ_FUA);
|
2017-08-14 16:04:56 -06:00
|
|
|
if (err) {
|
|
|
|
spin_unlock_irq(&nullb->lock);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
sector += len >> SECTOR_SHIFT;
|
|
|
|
}
|
|
|
|
spin_unlock_irq(&nullb->lock);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int null_handle_bio(struct nullb_cmd *cmd)
|
|
|
|
{
|
|
|
|
struct bio *bio = cmd->bio;
|
|
|
|
struct nullb *nullb = cmd->nq->dev->nullb;
|
|
|
|
int err;
|
|
|
|
unsigned int len;
|
|
|
|
sector_t sector;
|
|
|
|
struct bio_vec bvec;
|
|
|
|
struct bvec_iter iter;
|
|
|
|
|
|
|
|
sector = bio->bi_iter.bi_sector;
|
|
|
|
|
2017-08-14 16:04:57 -06:00
|
|
|
if (bio_op(bio) == REQ_OP_DISCARD) {
|
|
|
|
null_handle_discard(nullb, sector,
|
|
|
|
bio_sectors(bio) << SECTOR_SHIFT);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
spin_lock_irq(&nullb->lock);
|
|
|
|
bio_for_each_segment(bvec, bio, iter) {
|
|
|
|
len = bvec.bv_len;
|
|
|
|
err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
|
2017-08-14 16:04:59 -06:00
|
|
|
op_is_write(bio_op(bio)), sector,
|
|
|
|
bio_op(bio) & REQ_FUA);
|
2017-08-14 16:04:56 -06:00
|
|
|
if (err) {
|
|
|
|
spin_unlock_irq(&nullb->lock);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
sector += len >> SECTOR_SHIFT;
|
|
|
|
}
|
|
|
|
spin_unlock_irq(&nullb->lock);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:58 -06:00
|
|
|
static void null_stop_queue(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
struct request_queue *q = nullb->q;
|
|
|
|
|
|
|
|
if (nullb->dev->queue_mode == NULL_Q_MQ)
|
|
|
|
blk_mq_stop_hw_queues(q);
|
|
|
|
else {
|
|
|
|
spin_lock_irq(q->queue_lock);
|
|
|
|
blk_stop_queue(q);
|
|
|
|
spin_unlock_irq(q->queue_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_restart_queue_async(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
struct request_queue *q = nullb->q;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
if (nullb->dev->queue_mode == NULL_Q_MQ)
|
|
|
|
blk_mq_start_stopped_hw_queues(q, true);
|
|
|
|
else {
|
|
|
|
spin_lock_irqsave(q->queue_lock, flags);
|
|
|
|
blk_start_queue_async(q);
|
|
|
|
spin_unlock_irqrestore(q->queue_lock, flags);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
static blk_status_t null_handle_cmd(struct nullb_cmd *cmd)
|
|
|
|
{
|
|
|
|
struct nullb_device *dev = cmd->nq->dev;
|
2017-08-14 16:04:58 -06:00
|
|
|
struct nullb *nullb = dev->nullb;
|
2017-08-14 16:04:56 -06:00
|
|
|
int err = 0;
|
|
|
|
|
2017-08-14 16:04:58 -06:00
|
|
|
if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
|
|
|
|
struct request *rq = cmd->rq;
|
|
|
|
|
|
|
|
if (!hrtimer_active(&nullb->bw_timer))
|
|
|
|
hrtimer_restart(&nullb->bw_timer);
|
|
|
|
|
|
|
|
if (atomic_long_sub_return(blk_rq_bytes(rq),
|
|
|
|
&nullb->cur_bytes) < 0) {
|
|
|
|
null_stop_queue(nullb);
|
|
|
|
/* race with timer */
|
|
|
|
if (atomic_long_read(&nullb->cur_bytes) > 0)
|
|
|
|
null_restart_queue_async(nullb);
|
|
|
|
if (dev->queue_mode == NULL_Q_RQ) {
|
|
|
|
struct request_queue *q = nullb->q;
|
|
|
|
|
|
|
|
spin_lock_irq(q->queue_lock);
|
|
|
|
rq->rq_flags |= RQF_DONTPREP;
|
|
|
|
blk_requeue_request(q, rq);
|
|
|
|
spin_unlock_irq(q->queue_lock);
|
|
|
|
return BLK_STS_OK;
|
|
|
|
} else
|
|
|
|
/* requeue request */
|
|
|
|
return BLK_STS_RESOURCE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:05:00 -06:00
|
|
|
if (nullb->dev->badblocks.shift != -1) {
|
|
|
|
int bad_sectors;
|
|
|
|
sector_t sector, size, first_bad;
|
|
|
|
bool is_flush = true;
|
|
|
|
|
|
|
|
if (dev->queue_mode == NULL_Q_BIO &&
|
|
|
|
bio_op(cmd->bio) != REQ_OP_FLUSH) {
|
|
|
|
is_flush = false;
|
|
|
|
sector = cmd->bio->bi_iter.bi_sector;
|
|
|
|
size = bio_sectors(cmd->bio);
|
|
|
|
}
|
|
|
|
if (dev->queue_mode != NULL_Q_BIO &&
|
|
|
|
req_op(cmd->rq) != REQ_OP_FLUSH) {
|
|
|
|
is_flush = false;
|
|
|
|
sector = blk_rq_pos(cmd->rq);
|
|
|
|
size = blk_rq_sectors(cmd->rq);
|
|
|
|
}
|
|
|
|
if (!is_flush && badblocks_check(&nullb->dev->badblocks, sector,
|
|
|
|
size, &first_bad, &bad_sectors)) {
|
|
|
|
cmd->error = BLK_STS_IOERR;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
if (dev->memory_backed) {
|
2017-08-14 16:04:59 -06:00
|
|
|
if (dev->queue_mode == NULL_Q_BIO) {
|
|
|
|
if (bio_op(cmd->bio) == REQ_OP_FLUSH)
|
|
|
|
err = null_handle_flush(nullb);
|
|
|
|
else
|
|
|
|
err = null_handle_bio(cmd);
|
|
|
|
} else {
|
|
|
|
if (req_op(cmd->rq) == REQ_OP_FLUSH)
|
|
|
|
err = null_handle_flush(nullb);
|
|
|
|
else
|
|
|
|
err = null_handle_rq(cmd);
|
|
|
|
}
|
2017-08-14 16:04:56 -06:00
|
|
|
}
|
|
|
|
cmd->error = errno_to_blk_status(err);
|
2017-08-14 16:05:00 -06:00
|
|
|
out:
|
2013-10-25 04:52:25 -06:00
|
|
|
/* Complete IO by inline, softirq or timer */
|
2017-08-14 16:04:56 -06:00
|
|
|
switch (dev->irqmode) {
|
2013-10-25 04:52:25 -06:00
|
|
|
case NULL_IRQ_SOFTIRQ:
|
2017-08-14 16:04:56 -06:00
|
|
|
switch (dev->queue_mode) {
|
2014-02-10 04:24:40 -07:00
|
|
|
case NULL_Q_MQ:
|
2017-04-20 08:03:09 -06:00
|
|
|
blk_mq_complete_request(cmd->rq);
|
2014-02-10 04:24:40 -07:00
|
|
|
break;
|
|
|
|
case NULL_Q_RQ:
|
|
|
|
blk_complete_request(cmd->rq);
|
|
|
|
break;
|
|
|
|
case NULL_Q_BIO:
|
|
|
|
/*
|
|
|
|
* XXX: no proper submitting cpu information available.
|
|
|
|
*/
|
|
|
|
end_cmd(cmd);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case NULL_IRQ_NONE:
|
2013-10-25 04:52:25 -06:00
|
|
|
end_cmd(cmd);
|
|
|
|
break;
|
|
|
|
case NULL_IRQ_TIMER:
|
|
|
|
null_cmd_end_timer(cmd);
|
|
|
|
break;
|
|
|
|
}
|
2017-08-14 16:04:56 -06:00
|
|
|
return BLK_STS_OK;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:58 -06:00
|
|
|
static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
|
|
|
|
ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
|
|
|
|
unsigned int mbps = nullb->dev->mbps;
|
|
|
|
|
|
|
|
if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
|
|
|
|
return HRTIMER_NORESTART;
|
|
|
|
|
|
|
|
atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
|
|
|
|
null_restart_queue_async(nullb);
|
|
|
|
|
|
|
|
hrtimer_forward_now(&nullb->bw_timer, timer_interval);
|
|
|
|
|
|
|
|
return HRTIMER_RESTART;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nullb_setup_bwtimer(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
|
|
|
|
|
|
|
|
hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
|
|
|
nullb->bw_timer.function = nullb_bwtimer_fn;
|
|
|
|
atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
|
|
|
|
hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
int index = 0;
|
|
|
|
|
|
|
|
if (nullb->nr_queues != 1)
|
|
|
|
index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
|
|
|
|
|
|
|
|
return &nullb->queues[index];
|
|
|
|
}
|
|
|
|
|
2015-11-05 10:41:16 -07:00
|
|
|
static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
|
2013-10-25 04:52:25 -06:00
|
|
|
{
|
|
|
|
struct nullb *nullb = q->queuedata;
|
|
|
|
struct nullb_queue *nq = nullb_to_queue(nullb);
|
|
|
|
struct nullb_cmd *cmd;
|
|
|
|
|
|
|
|
cmd = alloc_cmd(nq, 1);
|
|
|
|
cmd->bio = bio;
|
|
|
|
|
|
|
|
null_handle_cmd(cmd);
|
2015-11-05 10:41:16 -07:00
|
|
|
return BLK_QC_T_NONE;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int null_rq_prep_fn(struct request_queue *q, struct request *req)
|
|
|
|
{
|
|
|
|
struct nullb *nullb = q->queuedata;
|
|
|
|
struct nullb_queue *nq = nullb_to_queue(nullb);
|
|
|
|
struct nullb_cmd *cmd;
|
|
|
|
|
|
|
|
cmd = alloc_cmd(nq, 0);
|
|
|
|
if (cmd) {
|
|
|
|
cmd->rq = req;
|
|
|
|
req->special = cmd;
|
|
|
|
return BLKPREP_OK;
|
|
|
|
}
|
2015-06-01 17:35:10 -06:00
|
|
|
blk_stop_queue(q);
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
return BLKPREP_DEFER;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_request_fn(struct request_queue *q)
|
|
|
|
{
|
|
|
|
struct request *rq;
|
|
|
|
|
|
|
|
while ((rq = blk_fetch_request(q)) != NULL) {
|
|
|
|
struct nullb_cmd *cmd = rq->special;
|
|
|
|
|
|
|
|
spin_unlock_irq(q->queue_lock);
|
|
|
|
null_handle_cmd(cmd);
|
|
|
|
spin_lock_irq(q->queue_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-06-03 01:38:05 -06:00
|
|
|
static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
|
2014-10-29 11:14:52 -06:00
|
|
|
const struct blk_mq_queue_data *bd)
|
2013-10-25 04:52:25 -06:00
|
|
|
{
|
2014-10-29 11:14:52 -06:00
|
|
|
struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
|
2017-08-14 16:04:52 -06:00
|
|
|
struct nullb_queue *nq = hctx->driver_data;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-03-30 13:44:26 -06:00
|
|
|
might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (nq->dev->irqmode == NULL_IRQ_TIMER) {
|
null_blk: set a separate timer for each command
For the Timer IRQ mode (i.e., when command completions are delayed),
there is one timer for each CPU. Each of these timers
. has a completion queue associated with it, containing all the
command completions to be executed when the timer fires;
. is set, and a new completion-to-execute is inserted into its
completion queue, every time the dispatch code for a new command
happens to be executed on the CPU related to the timer.
This implies that, if the dispatch of a new command happens to be
executed on a CPU whose timer has already been set, but has not yet
fired, then the timer is set again, to the completion time of the
newly arrived command. When the timer eventually fires, all its queued
completions are executed.
This way of handling delayed command completions entails the following
problem: if more than one command completion is inserted into the
queue of a timer before the timer fires, then the expiration time for
the timer is moved forward every time each of these completions is
enqueued. As a consequence, only the last completion enqueued enjoys a
correct execution time, while all previous completions are unjustly
delayed until the last completion is executed (and at that time they
are executed all together).
Specifically, if all the above completions are enqueued almost at the
same time, then the problem is negligible. On the opposite end, if
every completion is enqueued a while after the previous completion was
enqueued (in the extreme case, it is enqueued only right before the
timer would have expired), then every enqueued completion, except for
the last one, experiences an inflated delay, proportional to the number
of completions enqueued after it. In the end, commands, and thus I/O
requests, may be completed at an arbitrarily lower rate than the
desired one.
This commit addresses this issue by replacing per-CPU timers with
per-command timers, i.e., by associating an individual timer with each
command.
Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
Signed-off-by: Arianna Avanzini <avanzini@google.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-12-01 03:48:17 -07:00
|
|
|
hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
|
|
|
cmd->timer.function = null_cmd_timer_expired;
|
|
|
|
}
|
2014-10-29 11:14:52 -06:00
|
|
|
cmd->rq = bd->rq;
|
2017-08-14 16:04:52 -06:00
|
|
|
cmd->nq = nq;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2014-10-29 11:14:52 -06:00
|
|
|
blk_mq_start_request(bd->rq);
|
2014-09-13 17:40:09 -06:00
|
|
|
|
2017-08-14 16:04:56 -06:00
|
|
|
return null_handle_cmd(cmd);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
2017-03-30 14:39:16 -06:00
|
|
|
static const struct blk_mq_ops null_mq_ops = {
|
2013-10-25 04:52:25 -06:00
|
|
|
.queue_rq = null_queue_rq,
|
2014-02-10 04:24:40 -07:00
|
|
|
.complete = null_softirq_done_fn,
|
2013-10-25 04:52:25 -06:00
|
|
|
};
|
|
|
|
|
2015-08-31 06:17:18 -06:00
|
|
|
static void cleanup_queue(struct nullb_queue *nq)
|
|
|
|
{
|
|
|
|
kfree(nq->tag_map);
|
|
|
|
kfree(nq->cmds);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void cleanup_queues(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < nullb->nr_queues; i++)
|
|
|
|
cleanup_queue(&nullb->queues[i]);
|
|
|
|
|
|
|
|
kfree(nullb->queues);
|
|
|
|
}
|
|
|
|
|
2015-11-12 12:25:10 -07:00
|
|
|
#ifdef CONFIG_NVM
|
|
|
|
|
2017-06-03 01:38:04 -06:00
|
|
|
static void null_lnvm_end_io(struct request *rq, blk_status_t status)
|
2015-11-12 12:25:10 -07:00
|
|
|
{
|
|
|
|
struct nvm_rq *rqd = rq->end_io_data;
|
|
|
|
|
2017-06-03 01:38:04 -06:00
|
|
|
/* XXX: lighnvm core seems to expect NVM_RSP_* values here.. */
|
|
|
|
rqd->error = status ? -EIO : 0;
|
2017-01-31 05:17:17 -07:00
|
|
|
nvm_end_io(rqd);
|
2015-11-12 12:25:10 -07:00
|
|
|
|
|
|
|
blk_put_request(rq);
|
|
|
|
}
|
|
|
|
|
2015-12-06 03:25:48 -07:00
|
|
|
static int null_lnvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
|
2015-11-12 12:25:10 -07:00
|
|
|
{
|
2015-12-06 03:25:48 -07:00
|
|
|
struct request_queue *q = dev->q;
|
2015-11-12 12:25:10 -07:00
|
|
|
struct request *rq;
|
|
|
|
struct bio *bio = rqd->bio;
|
|
|
|
|
2017-01-31 08:57:31 -07:00
|
|
|
rq = blk_mq_alloc_request(q,
|
|
|
|
op_is_write(bio_op(bio)) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
|
2015-11-12 12:25:10 -07:00
|
|
|
if (IS_ERR(rq))
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2017-04-19 15:01:25 -06:00
|
|
|
blk_init_request_from_bio(rq, bio);
|
2015-11-12 12:25:10 -07:00
|
|
|
|
|
|
|
rq->end_io_data = rqd;
|
|
|
|
|
|
|
|
blk_execute_rq_nowait(q, NULL, rq, 0, null_lnvm_end_io);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-12-06 03:25:48 -07:00
|
|
|
static int null_lnvm_id(struct nvm_dev *dev, struct nvm_id *id)
|
2015-11-12 12:25:10 -07:00
|
|
|
{
|
2017-08-14 16:04:52 -06:00
|
|
|
struct nullb *nullb = dev->q->queuedata;
|
|
|
|
sector_t size = (sector_t)nullb->dev->size * 1024 * 1024ULL;
|
2015-11-19 04:50:09 -07:00
|
|
|
sector_t blksize;
|
2015-11-12 12:25:10 -07:00
|
|
|
struct nvm_id_group *grp;
|
|
|
|
|
|
|
|
id->ver_id = 0x1;
|
|
|
|
id->vmnt = 0;
|
2016-02-04 07:13:27 -07:00
|
|
|
id->cap = 0x2;
|
2015-11-12 12:25:10 -07:00
|
|
|
id->dom = 0x1;
|
2015-11-19 04:50:09 -07:00
|
|
|
|
|
|
|
id->ppaf.blk_offset = 0;
|
|
|
|
id->ppaf.blk_len = 16;
|
|
|
|
id->ppaf.pg_offset = 16;
|
|
|
|
id->ppaf.pg_len = 16;
|
|
|
|
id->ppaf.sect_offset = 32;
|
|
|
|
id->ppaf.sect_len = 8;
|
|
|
|
id->ppaf.pln_offset = 40;
|
|
|
|
id->ppaf.pln_len = 8;
|
|
|
|
id->ppaf.lun_offset = 48;
|
|
|
|
id->ppaf.lun_len = 8;
|
|
|
|
id->ppaf.ch_offset = 56;
|
|
|
|
id->ppaf.ch_len = 8;
|
2015-11-12 12:25:10 -07:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
sector_div(size, nullb->dev->blocksize); /* convert size to pages */
|
2016-01-13 15:04:08 -07:00
|
|
|
size >>= 8; /* concert size to pgs pr blk */
|
2017-01-31 05:17:15 -07:00
|
|
|
grp = &id->grp;
|
2015-11-12 12:25:10 -07:00
|
|
|
grp->mtype = 0;
|
2015-11-19 04:50:09 -07:00
|
|
|
grp->fmtype = 0;
|
2015-11-12 12:25:10 -07:00
|
|
|
grp->num_ch = 1;
|
|
|
|
grp->num_pg = 256;
|
2015-11-19 04:50:09 -07:00
|
|
|
blksize = size;
|
2016-01-13 15:04:08 -07:00
|
|
|
size >>= 16;
|
2015-11-19 04:50:09 -07:00
|
|
|
grp->num_lun = size + 1;
|
2016-01-13 15:04:08 -07:00
|
|
|
sector_div(blksize, grp->num_lun);
|
2015-11-19 04:50:09 -07:00
|
|
|
grp->num_blk = blksize;
|
|
|
|
grp->num_pln = 1;
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
grp->fpg_sz = nullb->dev->blocksize;
|
|
|
|
grp->csecs = nullb->dev->blocksize;
|
2015-11-12 12:25:10 -07:00
|
|
|
grp->trdt = 25000;
|
|
|
|
grp->trdm = 25000;
|
|
|
|
grp->tprt = 500000;
|
|
|
|
grp->tprm = 500000;
|
|
|
|
grp->tbet = 1500000;
|
|
|
|
grp->tbem = 1500000;
|
|
|
|
grp->mpos = 0x010101; /* single plane rwe */
|
2017-08-14 16:04:52 -06:00
|
|
|
grp->cpar = nullb->dev->hw_queue_depth;
|
2015-11-12 12:25:10 -07:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-12-06 03:25:48 -07:00
|
|
|
static void *null_lnvm_create_dma_pool(struct nvm_dev *dev, char *name)
|
2015-11-12 12:25:10 -07:00
|
|
|
{
|
|
|
|
mempool_t *virtmem_pool;
|
|
|
|
|
2015-11-19 04:50:08 -07:00
|
|
|
virtmem_pool = mempool_create_slab_pool(64, ppa_cache);
|
2015-11-12 12:25:10 -07:00
|
|
|
if (!virtmem_pool) {
|
|
|
|
pr_err("null_blk: Unable to create virtual memory pool\n");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return virtmem_pool;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_lnvm_destroy_dma_pool(void *pool)
|
|
|
|
{
|
|
|
|
mempool_destroy(pool);
|
|
|
|
}
|
|
|
|
|
2015-12-06 03:25:48 -07:00
|
|
|
static void *null_lnvm_dev_dma_alloc(struct nvm_dev *dev, void *pool,
|
2015-11-12 12:25:10 -07:00
|
|
|
gfp_t mem_flags, dma_addr_t *dma_handler)
|
|
|
|
{
|
|
|
|
return mempool_alloc(pool, mem_flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_lnvm_dev_dma_free(void *pool, void *entry,
|
|
|
|
dma_addr_t dma_handler)
|
|
|
|
{
|
|
|
|
mempool_free(entry, pool);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct nvm_dev_ops null_lnvm_dev_ops = {
|
|
|
|
.identity = null_lnvm_id,
|
|
|
|
.submit_io = null_lnvm_submit_io,
|
|
|
|
|
|
|
|
.create_dma_pool = null_lnvm_create_dma_pool,
|
|
|
|
.destroy_dma_pool = null_lnvm_destroy_dma_pool,
|
|
|
|
.dev_dma_alloc = null_lnvm_dev_dma_alloc,
|
|
|
|
.dev_dma_free = null_lnvm_dev_dma_free,
|
|
|
|
|
|
|
|
/* Simulate nvme protocol restriction */
|
|
|
|
.max_phys_sect = 64,
|
|
|
|
};
|
2016-09-16 06:25:05 -06:00
|
|
|
|
|
|
|
static int null_nvm_register(struct nullb *nullb)
|
|
|
|
{
|
2016-09-16 06:25:07 -06:00
|
|
|
struct nvm_dev *dev;
|
|
|
|
int rv;
|
|
|
|
|
|
|
|
dev = nvm_alloc_dev(0);
|
|
|
|
if (!dev)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
dev->q = nullb->q;
|
|
|
|
memcpy(dev->name, nullb->disk_name, DISK_NAME_LEN);
|
|
|
|
dev->ops = &null_lnvm_dev_ops;
|
|
|
|
|
|
|
|
rv = nvm_register(dev);
|
|
|
|
if (rv) {
|
|
|
|
kfree(dev);
|
|
|
|
return rv;
|
|
|
|
}
|
|
|
|
nullb->ndev = dev;
|
|
|
|
return 0;
|
2016-09-16 06:25:05 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void null_nvm_unregister(struct nullb *nullb)
|
|
|
|
{
|
2016-09-16 06:25:07 -06:00
|
|
|
nvm_unregister(nullb->ndev);
|
2016-09-16 06:25:05 -06:00
|
|
|
}
|
2015-11-12 12:25:10 -07:00
|
|
|
#else
|
2016-09-16 06:25:05 -06:00
|
|
|
static int null_nvm_register(struct nullb *nullb)
|
|
|
|
{
|
2016-11-16 08:26:11 -07:00
|
|
|
pr_err("null_blk: CONFIG_NVM needs to be enabled for LightNVM\n");
|
2016-09-16 06:25:05 -06:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
static void null_nvm_unregister(struct nullb *nullb) {}
|
2015-11-12 12:25:10 -07:00
|
|
|
#endif /* CONFIG_NVM */
|
|
|
|
|
2016-09-16 06:25:05 -06:00
|
|
|
static void null_del_dev(struct nullb *nullb)
|
|
|
|
{
|
2017-08-14 16:04:52 -06:00
|
|
|
struct nullb_device *dev = nullb->dev;
|
|
|
|
|
2017-08-14 16:04:55 -06:00
|
|
|
ida_simple_remove(&nullb_indexes, nullb->index);
|
|
|
|
|
2016-09-16 06:25:05 -06:00
|
|
|
list_del_init(&nullb->list);
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (dev->use_lightnvm)
|
2016-09-16 06:25:05 -06:00
|
|
|
null_nvm_unregister(nullb);
|
|
|
|
else
|
|
|
|
del_gendisk(nullb->disk);
|
2017-08-14 16:04:58 -06:00
|
|
|
|
|
|
|
if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
|
|
|
|
hrtimer_cancel(&nullb->bw_timer);
|
|
|
|
atomic_long_set(&nullb->cur_bytes, LONG_MAX);
|
|
|
|
null_restart_queue_async(nullb);
|
|
|
|
}
|
|
|
|
|
2016-09-16 06:25:05 -06:00
|
|
|
blk_cleanup_queue(nullb->q);
|
2017-08-14 16:04:52 -06:00
|
|
|
if (dev->queue_mode == NULL_Q_MQ &&
|
|
|
|
nullb->tag_set == &nullb->__tag_set)
|
2017-06-20 14:22:01 -06:00
|
|
|
blk_mq_free_tag_set(nullb->tag_set);
|
2017-08-14 16:04:52 -06:00
|
|
|
if (!dev->use_lightnvm)
|
2016-09-16 06:25:05 -06:00
|
|
|
put_disk(nullb->disk);
|
|
|
|
cleanup_queues(nullb);
|
2017-08-14 16:04:59 -06:00
|
|
|
if (null_cache_active(nullb))
|
|
|
|
null_free_device_storage(nullb->dev, true);
|
2016-09-16 06:25:05 -06:00
|
|
|
kfree(nullb);
|
2017-08-14 16:04:52 -06:00
|
|
|
dev->nullb = NULL;
|
2016-09-16 06:25:05 -06:00
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:57 -06:00
|
|
|
static void null_config_discard(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
if (nullb->dev->discard == false)
|
|
|
|
return;
|
|
|
|
nullb->q->limits.discard_granularity = nullb->dev->blocksize;
|
|
|
|
nullb->q->limits.discard_alignment = nullb->dev->blocksize;
|
|
|
|
blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9);
|
|
|
|
queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, nullb->q);
|
2016-09-16 06:25:05 -06:00
|
|
|
}
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
static int null_open(struct block_device *bdev, fmode_t mode)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void null_release(struct gendisk *disk, fmode_t mode)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct block_device_operations null_fops = {
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.open = null_open,
|
|
|
|
.release = null_release,
|
|
|
|
};
|
|
|
|
|
2017-06-20 14:22:01 -06:00
|
|
|
static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
|
|
|
|
{
|
|
|
|
BUG_ON(!nullb);
|
|
|
|
BUG_ON(!nq);
|
|
|
|
|
|
|
|
init_waitqueue_head(&nq->wait);
|
|
|
|
nq->queue_depth = nullb->queue_depth;
|
2017-08-14 16:04:52 -06:00
|
|
|
nq->dev = nullb->dev;
|
2017-06-20 14:22:01 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void null_init_queues(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
struct request_queue *q = nullb->q;
|
|
|
|
struct blk_mq_hw_ctx *hctx;
|
|
|
|
struct nullb_queue *nq;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
|
|
|
if (!hctx->nr_ctx || !hctx->tags)
|
|
|
|
continue;
|
|
|
|
nq = &nullb->queues[i];
|
|
|
|
hctx->driver_data = nq;
|
|
|
|
null_init_queue(nullb, nq);
|
|
|
|
nullb->nr_queues++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
static int setup_commands(struct nullb_queue *nq)
|
|
|
|
{
|
|
|
|
struct nullb_cmd *cmd;
|
|
|
|
int i, tag_size;
|
|
|
|
|
|
|
|
nq->cmds = kzalloc(nq->queue_depth * sizeof(*cmd), GFP_KERNEL);
|
|
|
|
if (!nq->cmds)
|
2013-12-18 05:41:43 -07:00
|
|
|
return -ENOMEM;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
|
|
|
|
nq->tag_map = kzalloc(tag_size * sizeof(unsigned long), GFP_KERNEL);
|
|
|
|
if (!nq->tag_map) {
|
|
|
|
kfree(nq->cmds);
|
2013-12-18 05:41:43 -07:00
|
|
|
return -ENOMEM;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < nq->queue_depth; i++) {
|
|
|
|
cmd = &nq->cmds[i];
|
|
|
|
INIT_LIST_HEAD(&cmd->list);
|
|
|
|
cmd->ll_list.next = NULL;
|
|
|
|
cmd->tag = -1U;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int setup_queues(struct nullb *nullb)
|
|
|
|
{
|
2017-08-14 16:04:52 -06:00
|
|
|
nullb->queues = kzalloc(nullb->dev->submit_queues *
|
|
|
|
sizeof(struct nullb_queue), GFP_KERNEL);
|
2013-10-25 04:52:25 -06:00
|
|
|
if (!nullb->queues)
|
2013-12-18 05:41:43 -07:00
|
|
|
return -ENOMEM;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
nullb->nr_queues = 0;
|
2017-08-14 16:04:52 -06:00
|
|
|
nullb->queue_depth = nullb->dev->hw_queue_depth;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2013-12-18 05:41:43 -07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int init_driver_queues(struct nullb *nullb)
|
|
|
|
{
|
|
|
|
struct nullb_queue *nq;
|
|
|
|
int i, ret = 0;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
for (i = 0; i < nullb->dev->submit_queues; i++) {
|
2013-10-25 04:52:25 -06:00
|
|
|
nq = &nullb->queues[i];
|
2013-12-18 05:41:43 -07:00
|
|
|
|
|
|
|
null_init_queue(nullb, nq);
|
|
|
|
|
|
|
|
ret = setup_commands(nq);
|
|
|
|
if (ret)
|
2014-10-22 07:34:21 -06:00
|
|
|
return ret;
|
2013-10-25 04:52:25 -06:00
|
|
|
nullb->nr_queues++;
|
|
|
|
}
|
2013-12-18 05:41:43 -07:00
|
|
|
return 0;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
2016-09-16 06:25:05 -06:00
|
|
|
static int null_gendisk_register(struct nullb *nullb)
|
2013-10-25 04:52:25 -06:00
|
|
|
{
|
|
|
|
struct gendisk *disk;
|
|
|
|
sector_t size;
|
2016-09-16 06:25:05 -06:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node);
|
2016-09-16 06:25:05 -06:00
|
|
|
if (!disk)
|
|
|
|
return -ENOMEM;
|
2017-08-14 16:04:52 -06:00
|
|
|
size = (sector_t)nullb->dev->size * 1024 * 1024ULL;
|
2016-09-16 06:25:05 -06:00
|
|
|
set_capacity(disk, size >> 9);
|
|
|
|
|
|
|
|
disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
|
|
|
|
disk->major = null_major;
|
|
|
|
disk->first_minor = nullb->index;
|
|
|
|
disk->fops = &null_fops;
|
|
|
|
disk->private_data = nullb;
|
|
|
|
disk->queue = nullb->q;
|
|
|
|
strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
|
|
|
|
|
|
|
|
add_disk(disk);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
|
2017-06-20 14:22:01 -06:00
|
|
|
{
|
|
|
|
set->ops = &null_mq_ops;
|
2017-08-14 16:04:52 -06:00
|
|
|
set->nr_hw_queues = nullb ? nullb->dev->submit_queues :
|
|
|
|
g_submit_queues;
|
|
|
|
set->queue_depth = nullb ? nullb->dev->hw_queue_depth :
|
|
|
|
g_hw_queue_depth;
|
|
|
|
set->numa_node = nullb ? nullb->dev->home_node : g_home_node;
|
2017-06-20 14:22:01 -06:00
|
|
|
set->cmd_size = sizeof(struct nullb_cmd);
|
|
|
|
set->flags = BLK_MQ_F_SHOULD_MERGE;
|
2017-09-29 19:49:21 -06:00
|
|
|
if (g_no_sched)
|
|
|
|
set->flags |= BLK_MQ_F_NO_SCHED;
|
2017-06-20 14:22:01 -06:00
|
|
|
set->driver_data = NULL;
|
|
|
|
|
2017-08-25 14:46:25 -06:00
|
|
|
if ((nullb && nullb->dev->blocking) || g_blocking)
|
2017-06-20 14:22:01 -06:00
|
|
|
set->flags |= BLK_MQ_F_BLOCKING;
|
|
|
|
|
|
|
|
return blk_mq_alloc_tag_set(set);
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:54 -06:00
|
|
|
static void null_validate_conf(struct nullb_device *dev)
|
|
|
|
{
|
|
|
|
dev->blocksize = round_down(dev->blocksize, 512);
|
|
|
|
dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
|
|
|
|
if (dev->use_lightnvm && dev->blocksize != 4096)
|
|
|
|
dev->blocksize = 4096;
|
|
|
|
|
|
|
|
if (dev->use_lightnvm && dev->queue_mode != NULL_Q_MQ)
|
|
|
|
dev->queue_mode = NULL_Q_MQ;
|
|
|
|
|
|
|
|
if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) {
|
|
|
|
if (dev->submit_queues != nr_online_nodes)
|
|
|
|
dev->submit_queues = nr_online_nodes;
|
|
|
|
} else if (dev->submit_queues > nr_cpu_ids)
|
|
|
|
dev->submit_queues = nr_cpu_ids;
|
|
|
|
else if (dev->submit_queues == 0)
|
|
|
|
dev->submit_queues = 1;
|
|
|
|
|
|
|
|
dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);
|
|
|
|
dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
|
2017-08-14 16:04:56 -06:00
|
|
|
|
|
|
|
/* Do memory allocation, so set blocking */
|
|
|
|
if (dev->memory_backed)
|
|
|
|
dev->blocking = true;
|
2017-08-14 16:04:59 -06:00
|
|
|
else /* cache is meaningless */
|
|
|
|
dev->cache_size = 0;
|
|
|
|
dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
|
|
|
|
dev->cache_size);
|
2017-08-14 16:04:58 -06:00
|
|
|
dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
|
|
|
|
/* can not stop a queue */
|
|
|
|
if (dev->queue_mode == NULL_Q_BIO)
|
|
|
|
dev->mbps = 0;
|
2017-08-14 16:04:54 -06:00
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
static int null_add_dev(struct nullb_device *dev)
|
2016-09-16 06:25:05 -06:00
|
|
|
{
|
|
|
|
struct nullb *nullb;
|
2014-09-02 10:38:49 -06:00
|
|
|
int rv;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:54 -06:00
|
|
|
null_validate_conf(dev);
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
|
2014-09-02 10:38:49 -06:00
|
|
|
if (!nullb) {
|
|
|
|
rv = -ENOMEM;
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out;
|
2014-09-02 10:38:49 -06:00
|
|
|
}
|
2017-08-14 16:04:52 -06:00
|
|
|
nullb->dev = dev;
|
|
|
|
dev->nullb = nullb;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
spin_lock_init(&nullb->lock);
|
|
|
|
|
2014-09-02 10:38:49 -06:00
|
|
|
rv = setup_queues(nullb);
|
|
|
|
if (rv)
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out_free_nullb;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (dev->queue_mode == NULL_Q_MQ) {
|
2017-06-20 14:22:01 -06:00
|
|
|
if (shared_tags) {
|
|
|
|
nullb->tag_set = &tag_set;
|
|
|
|
rv = 0;
|
|
|
|
} else {
|
|
|
|
nullb->tag_set = &nullb->__tag_set;
|
2017-08-14 16:04:52 -06:00
|
|
|
rv = null_init_tag_set(nullb, nullb->tag_set);
|
2017-06-20 14:22:01 -06:00
|
|
|
}
|
|
|
|
|
2014-09-02 10:38:49 -06:00
|
|
|
if (rv)
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out_cleanup_queues;
|
|
|
|
|
2017-06-20 14:22:01 -06:00
|
|
|
nullb->q = blk_mq_init_queue(nullb->tag_set);
|
2015-01-02 07:25:27 -07:00
|
|
|
if (IS_ERR(nullb->q)) {
|
2014-09-02 10:38:49 -06:00
|
|
|
rv = -ENOMEM;
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out_cleanup_tags;
|
2014-09-02 10:38:49 -06:00
|
|
|
}
|
2017-06-20 14:22:01 -06:00
|
|
|
null_init_queues(nullb);
|
2017-08-14 16:04:52 -06:00
|
|
|
} else if (dev->queue_mode == NULL_Q_BIO) {
|
|
|
|
nullb->q = blk_alloc_queue_node(GFP_KERNEL, dev->home_node);
|
2014-09-02 10:38:49 -06:00
|
|
|
if (!nullb->q) {
|
|
|
|
rv = -ENOMEM;
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out_cleanup_queues;
|
2014-09-02 10:38:49 -06:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
blk_queue_make_request(nullb->q, null_queue_bio);
|
2014-10-22 07:34:21 -06:00
|
|
|
rv = init_driver_queues(nullb);
|
|
|
|
if (rv)
|
|
|
|
goto out_cleanup_blk_queue;
|
2013-10-25 04:52:25 -06:00
|
|
|
} else {
|
2017-08-14 16:04:52 -06:00
|
|
|
nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock,
|
|
|
|
dev->home_node);
|
2014-09-02 10:38:49 -06:00
|
|
|
if (!nullb->q) {
|
|
|
|
rv = -ENOMEM;
|
2014-04-15 14:14:00 -06:00
|
|
|
goto out_cleanup_queues;
|
2014-09-02 10:38:49 -06:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
blk_queue_prep_rq(nullb->q, null_rq_prep_fn);
|
2014-04-15 14:14:00 -06:00
|
|
|
blk_queue_softirq_done(nullb->q, null_softirq_done_fn);
|
2014-10-22 07:34:21 -06:00
|
|
|
rv = init_driver_queues(nullb);
|
|
|
|
if (rv)
|
|
|
|
goto out_cleanup_blk_queue;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:58 -06:00
|
|
|
if (dev->mbps) {
|
|
|
|
set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
|
|
|
|
nullb_setup_bwtimer(nullb);
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
if (dev->cache_size > 0) {
|
|
|
|
set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
|
|
|
|
blk_queue_write_cache(nullb->q, true, true);
|
|
|
|
blk_queue_flush_queueable(nullb->q, true);
|
|
|
|
}
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
nullb->q->queuedata = nullb;
|
|
|
|
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, nullb->q);
|
2014-10-04 10:55:32 -06:00
|
|
|
queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, nullb->q);
|
2013-10-25 04:52:25 -06:00
|
|
|
|
|
|
|
mutex_lock(&lock);
|
2017-08-14 16:04:55 -06:00
|
|
|
nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);
|
2017-08-14 16:04:54 -06:00
|
|
|
dev->index = nullb->index;
|
2013-10-25 04:52:25 -06:00
|
|
|
mutex_unlock(&lock);
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
blk_queue_logical_block_size(nullb->q, dev->blocksize);
|
|
|
|
blk_queue_physical_block_size(nullb->q, dev->blocksize);
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:57 -06:00
|
|
|
null_config_discard(nullb);
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2015-11-12 12:25:10 -07:00
|
|
|
sprintf(nullb->disk_name, "nullb%d", nullb->index);
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (dev->use_lightnvm)
|
2016-09-16 06:25:05 -06:00
|
|
|
rv = null_nvm_register(nullb);
|
|
|
|
else
|
|
|
|
rv = null_gendisk_register(nullb);
|
2015-11-12 12:25:10 -07:00
|
|
|
|
2016-09-16 06:25:05 -06:00
|
|
|
if (rv)
|
|
|
|
goto out_cleanup_blk_queue;
|
2016-02-11 06:49:13 -07:00
|
|
|
|
|
|
|
mutex_lock(&lock);
|
|
|
|
list_add_tail(&nullb->list, &nullb_list);
|
|
|
|
mutex_unlock(&lock);
|
2016-03-04 09:27:04 -07:00
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
return 0;
|
2014-04-15 14:14:00 -06:00
|
|
|
out_cleanup_blk_queue:
|
|
|
|
blk_cleanup_queue(nullb->q);
|
|
|
|
out_cleanup_tags:
|
2017-08-14 16:04:52 -06:00
|
|
|
if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
|
2017-06-20 14:22:01 -06:00
|
|
|
blk_mq_free_tag_set(nullb->tag_set);
|
2014-04-15 14:14:00 -06:00
|
|
|
out_cleanup_queues:
|
|
|
|
cleanup_queues(nullb);
|
|
|
|
out_free_nullb:
|
|
|
|
kfree(nullb);
|
|
|
|
out:
|
2014-09-02 10:38:49 -06:00
|
|
|
return rv;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int __init null_init(void)
|
|
|
|
{
|
2015-12-08 13:47:34 -07:00
|
|
|
int ret = 0;
|
2013-10-25 04:52:25 -06:00
|
|
|
unsigned int i;
|
2015-12-08 13:47:34 -07:00
|
|
|
struct nullb *nullb;
|
2017-08-14 16:04:52 -06:00
|
|
|
struct nullb_device *dev;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:59 -06:00
|
|
|
/* check for nullb_page.bitmap */
|
|
|
|
if (sizeof(unsigned long) * 8 - 2 < (PAGE_SIZE >> SECTOR_SHIFT))
|
|
|
|
return -EINVAL;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (g_bs > PAGE_SIZE) {
|
2014-01-21 04:29:59 -07:00
|
|
|
pr_warn("null_blk: invalid block size\n");
|
|
|
|
pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE);
|
2017-08-14 16:04:52 -06:00
|
|
|
g_bs = PAGE_SIZE;
|
2014-01-21 04:29:59 -07:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (g_use_lightnvm && g_bs != 4096) {
|
2015-11-19 04:50:08 -07:00
|
|
|
pr_warn("null_blk: LightNVM only supports 4k block size\n");
|
|
|
|
pr_warn("null_blk: defaults block size to 4k\n");
|
2017-08-14 16:04:52 -06:00
|
|
|
g_bs = 4096;
|
2015-11-19 04:50:08 -07:00
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (g_use_lightnvm && g_queue_mode != NULL_Q_MQ) {
|
2015-11-12 12:25:10 -07:00
|
|
|
pr_warn("null_blk: LightNVM only supported for blk-mq\n");
|
|
|
|
pr_warn("null_blk: defaults queue mode to blk-mq\n");
|
2017-08-14 16:04:52 -06:00
|
|
|
g_queue_mode = NULL_Q_MQ;
|
2015-11-12 12:25:10 -07:00
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
|
|
|
|
if (g_submit_queues != nr_online_nodes) {
|
2017-08-02 10:26:39 -06:00
|
|
|
pr_warn("null_blk: submit_queues param is set to %u.\n",
|
2013-12-18 05:41:44 -07:00
|
|
|
nr_online_nodes);
|
2017-08-14 16:04:52 -06:00
|
|
|
g_submit_queues = nr_online_nodes;
|
2013-12-20 16:11:01 -07:00
|
|
|
}
|
2017-08-14 16:04:52 -06:00
|
|
|
} else if (g_submit_queues > nr_cpu_ids)
|
|
|
|
g_submit_queues = nr_cpu_ids;
|
|
|
|
else if (g_submit_queues <= 0)
|
|
|
|
g_submit_queues = 1;
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (g_queue_mode == NULL_Q_MQ && shared_tags) {
|
|
|
|
ret = null_init_tag_set(NULL, &tag_set);
|
2017-07-06 09:00:07 -06:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2017-08-14 16:04:53 -06:00
|
|
|
config_group_init(&nullb_subsys.su_group);
|
|
|
|
mutex_init(&nullb_subsys.su_mutex);
|
|
|
|
|
|
|
|
ret = configfs_register_subsystem(&nullb_subsys);
|
|
|
|
if (ret)
|
|
|
|
goto err_tagset;
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
mutex_init(&lock);
|
|
|
|
|
|
|
|
null_major = register_blkdev(0, "nullb");
|
2017-07-06 09:00:07 -06:00
|
|
|
if (null_major < 0) {
|
|
|
|
ret = null_major;
|
2017-08-14 16:04:53 -06:00
|
|
|
goto err_conf;
|
2017-07-06 09:00:07 -06:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (g_use_lightnvm) {
|
2015-11-19 04:50:08 -07:00
|
|
|
ppa_cache = kmem_cache_create("ppa_cache", 64 * sizeof(u64),
|
|
|
|
0, 0, NULL);
|
|
|
|
if (!ppa_cache) {
|
|
|
|
pr_err("null_blk: unable to create ppa cache\n");
|
2015-12-08 13:47:34 -07:00
|
|
|
ret = -ENOMEM;
|
|
|
|
goto err_ppa;
|
2015-11-19 04:50:08 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
for (i = 0; i < nr_devices; i++) {
|
2017-08-14 16:04:52 -06:00
|
|
|
dev = null_alloc_dev();
|
2017-10-17 06:11:46 -06:00
|
|
|
if (!dev) {
|
|
|
|
ret = -ENOMEM;
|
2017-08-14 16:04:52 -06:00
|
|
|
goto err_dev;
|
2017-10-17 06:11:46 -06:00
|
|
|
}
|
2017-08-14 16:04:52 -06:00
|
|
|
ret = null_add_dev(dev);
|
|
|
|
if (ret) {
|
|
|
|
null_free_dev(dev);
|
2015-12-08 13:47:34 -07:00
|
|
|
goto err_dev;
|
2017-08-14 16:04:52 -06:00
|
|
|
}
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
pr_info("null: module loaded\n");
|
|
|
|
return 0;
|
2015-12-08 13:47:34 -07:00
|
|
|
|
|
|
|
err_dev:
|
|
|
|
while (!list_empty(&nullb_list)) {
|
|
|
|
nullb = list_entry(nullb_list.next, struct nullb, list);
|
2017-08-14 16:04:52 -06:00
|
|
|
dev = nullb->dev;
|
2015-12-08 13:47:34 -07:00
|
|
|
null_del_dev(nullb);
|
2017-08-14 16:04:52 -06:00
|
|
|
null_free_dev(dev);
|
2015-12-08 13:47:34 -07:00
|
|
|
}
|
2015-11-19 04:50:08 -07:00
|
|
|
kmem_cache_destroy(ppa_cache);
|
2015-12-08 13:47:34 -07:00
|
|
|
err_ppa:
|
|
|
|
unregister_blkdev(null_major, "nullb");
|
2017-08-14 16:04:53 -06:00
|
|
|
err_conf:
|
|
|
|
configfs_unregister_subsystem(&nullb_subsys);
|
2017-07-06 09:00:07 -06:00
|
|
|
err_tagset:
|
2017-08-14 16:04:52 -06:00
|
|
|
if (g_queue_mode == NULL_Q_MQ && shared_tags)
|
2017-07-06 09:00:07 -06:00
|
|
|
blk_mq_free_tag_set(&tag_set);
|
2015-12-08 13:47:34 -07:00
|
|
|
return ret;
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void __exit null_exit(void)
|
|
|
|
{
|
|
|
|
struct nullb *nullb;
|
|
|
|
|
2017-08-14 16:04:53 -06:00
|
|
|
configfs_unregister_subsystem(&nullb_subsys);
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
unregister_blkdev(null_major, "nullb");
|
|
|
|
|
|
|
|
mutex_lock(&lock);
|
|
|
|
while (!list_empty(&nullb_list)) {
|
2017-08-14 16:04:52 -06:00
|
|
|
struct nullb_device *dev;
|
|
|
|
|
2013-10-25 04:52:25 -06:00
|
|
|
nullb = list_entry(nullb_list.next, struct nullb, list);
|
2017-08-14 16:04:52 -06:00
|
|
|
dev = nullb->dev;
|
2013-10-25 04:52:25 -06:00
|
|
|
null_del_dev(nullb);
|
2017-08-14 16:04:52 -06:00
|
|
|
null_free_dev(dev);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
mutex_unlock(&lock);
|
2015-11-19 04:50:08 -07:00
|
|
|
|
2017-08-14 16:04:52 -06:00
|
|
|
if (g_queue_mode == NULL_Q_MQ && shared_tags)
|
2017-06-20 14:22:01 -06:00
|
|
|
blk_mq_free_tag_set(&tag_set);
|
|
|
|
|
2015-11-19 04:50:08 -07:00
|
|
|
kmem_cache_destroy(ppa_cache);
|
2013-10-25 04:52:25 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
module_init(null_init);
|
|
|
|
module_exit(null_exit);
|
|
|
|
|
2017-08-25 12:53:15 -06:00
|
|
|
MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
|
2013-10-25 04:52:25 -06:00
|
|
|
MODULE_LICENSE("GPL");
|