nvme: simplify completion handling
Now that all commands are executed as block layer requests we can remove the internal completion in the NVMe driver. Note that we can simply call blk_mq_complete_request to abort commands as the block layer will protect against double copletions internally. Signed-off-by: Christoph Hellwig <hch@lst.de> Acked-by: Keith Busch <keith.busch@intel.com> Signed-off-by: Jens Axboe <axboe@fb.com>hifive-unleashed-5.1
Christoph Hellwig
Jens Axboe
parent
adf68f21c1
commit
aae239e191
|
|
@ -199,15 +199,11 @@ static inline void _nvme_check_size(void)
|
|||
BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
|
||||
}
|
||||
|
||||
typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
|
||||
struct nvme_completion *);
|
||||
|
||||
struct nvme_cmd_info {
|
||||
nvme_completion_fn fn;
|
||||
void *ctx;
|
||||
int aborted;
|
||||
struct nvme_queue *nvmeq;
|
||||
struct nvme_iod iod[0];
|
||||
struct nvme_iod *iod;
|
||||
struct nvme_iod __iod;
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
@ -302,15 +298,6 @@ static int nvme_init_request(void *data, struct request *req,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void nvme_set_info(struct nvme_cmd_info *cmd, void *ctx,
|
||||
nvme_completion_fn handler)
|
||||
{
|
||||
cmd->fn = handler;
|
||||
cmd->ctx = ctx;
|
||||
cmd->aborted = 0;
|
||||
blk_mq_start_request(blk_mq_rq_from_pdu(cmd));
|
||||
}
|
||||
|
||||
static void *iod_get_private(struct nvme_iod *iod)
|
||||
{
|
||||
return (void *) (iod->private & ~0x1UL);
|
||||
|
|
@ -324,44 +311,6 @@ static bool iod_should_kfree(struct nvme_iod *iod)
|
|||
return (iod->private & NVME_INT_MASK) == 0;
|
||||
}
|
||||
|
||||
/* Special values must be less than 0x1000 */
|
||||
#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA)
|
||||
#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE)
|
||||
#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE)
|
||||
#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE)
|
||||
|
||||
static void special_completion(struct nvme_queue *nvmeq, void *ctx,
|
||||
struct nvme_completion *cqe)
|
||||
{
|
||||
if (ctx == CMD_CTX_CANCELLED)
|
||||
return;
|
||||
if (ctx == CMD_CTX_COMPLETED) {
|
||||
dev_warn(nvmeq->q_dmadev,
|
||||
"completed id %d twice on queue %d\n",
|
||||
cqe->command_id, le16_to_cpup(&cqe->sq_id));
|
||||
return;
|
||||
}
|
||||
if (ctx == CMD_CTX_INVALID) {
|
||||
dev_warn(nvmeq->q_dmadev,
|
||||
"invalid id %d completed on queue %d\n",
|
||||
cqe->command_id, le16_to_cpup(&cqe->sq_id));
|
||||
return;
|
||||
}
|
||||
dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
|
||||
}
|
||||
|
||||
static void *cancel_cmd_info(struct nvme_cmd_info *cmd, nvme_completion_fn *fn)
|
||||
{
|
||||
void *ctx;
|
||||
|
||||
if (fn)
|
||||
*fn = cmd->fn;
|
||||
ctx = cmd->ctx;
|
||||
cmd->fn = special_completion;
|
||||
cmd->ctx = CMD_CTX_CANCELLED;
|
||||
return ctx;
|
||||
}
|
||||
|
||||
static void nvme_complete_async_event(struct nvme_dev *dev,
|
||||
struct nvme_completion *cqe)
|
||||
{
|
||||
|
|
@ -382,34 +331,6 @@ static void nvme_complete_async_event(struct nvme_dev *dev,
|
|||
}
|
||||
}
|
||||
|
||||
static inline struct nvme_cmd_info *get_cmd_from_tag(struct nvme_queue *nvmeq,
|
||||
unsigned int tag)
|
||||
{
|
||||
struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, tag);
|
||||
|
||||
return blk_mq_rq_to_pdu(req);
|
||||
}
|
||||
|
||||
/*
|
||||
* Called with local interrupts disabled and the q_lock held. May not sleep.
|
||||
*/
|
||||
static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag,
|
||||
nvme_completion_fn *fn)
|
||||
{
|
||||
struct nvme_cmd_info *cmd = get_cmd_from_tag(nvmeq, tag);
|
||||
void *ctx;
|
||||
if (tag >= nvmeq->q_depth) {
|
||||
*fn = special_completion;
|
||||
return CMD_CTX_INVALID;
|
||||
}
|
||||
if (fn)
|
||||
*fn = cmd->fn;
|
||||
ctx = cmd->ctx;
|
||||
cmd->fn = special_completion;
|
||||
cmd->ctx = CMD_CTX_COMPLETED;
|
||||
return ctx;
|
||||
}
|
||||
|
||||
/**
|
||||
* __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
|
||||
* @nvmeq: The queue to use
|
||||
|
|
@ -473,7 +394,7 @@ static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
|
|||
size <= NVME_INT_BYTES(dev)) {
|
||||
struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);
|
||||
|
||||
iod = cmd->iod;
|
||||
iod = &cmd->__iod;
|
||||
iod_init(iod, size, rq->nr_phys_segments,
|
||||
(unsigned long) rq | NVME_INT_MASK);
|
||||
return iod;
|
||||
|
|
@ -570,12 +491,11 @@ static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
|
|||
}
|
||||
#endif
|
||||
|
||||
static void req_completion(struct nvme_queue *nvmeq, void *ctx,
|
||||
struct nvme_completion *cqe)
|
||||
static void req_completion(struct nvme_queue *nvmeq, struct nvme_completion *cqe)
|
||||
{
|
||||
struct nvme_iod *iod = ctx;
|
||||
struct request *req = iod_get_private(iod);
|
||||
struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
|
||||
struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
|
||||
struct nvme_iod *iod = cmd_rq->iod;
|
||||
u16 status = le16_to_cpup(&cqe->status) >> 1;
|
||||
int error = 0;
|
||||
|
||||
|
|
@ -586,14 +506,10 @@ static void req_completion(struct nvme_queue *nvmeq, void *ctx,
|
|||
return;
|
||||
}
|
||||
|
||||
if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
|
||||
if (cmd_rq->ctx == CMD_CTX_CANCELLED)
|
||||
error = NVME_SC_CANCELLED;
|
||||
else
|
||||
error = status;
|
||||
} else {
|
||||
if (req->cmd_type == REQ_TYPE_DRV_PRIV)
|
||||
error = status;
|
||||
else
|
||||
error = nvme_error_status(status);
|
||||
}
|
||||
}
|
||||
|
||||
if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
|
||||
|
|
@ -836,8 +752,10 @@ static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
|
|||
if (ret)
|
||||
goto out;
|
||||
|
||||
cmd->iod = iod;
|
||||
cmd->aborted = 0;
|
||||
cmnd.common.command_id = req->tag;
|
||||
nvme_set_info(cmd, iod, req_completion);
|
||||
blk_mq_start_request(req);
|
||||
|
||||
spin_lock_irq(&nvmeq->q_lock);
|
||||
__nvme_submit_cmd(nvmeq, &cmnd);
|
||||
|
|
@ -857,8 +775,6 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
|
|||
phase = nvmeq->cq_phase;
|
||||
|
||||
for (;;) {
|
||||
void *ctx;
|
||||
nvme_completion_fn fn;
|
||||
struct nvme_completion cqe = nvmeq->cqes[head];
|
||||
u16 status = le16_to_cpu(cqe.status);
|
||||
|
||||
|
|
@ -873,6 +789,13 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
|
|||
if (tag && *tag == cqe.command_id)
|
||||
*tag = -1;
|
||||
|
||||
if (unlikely(cqe.command_id >= nvmeq->q_depth)) {
|
||||
dev_warn(nvmeq->q_dmadev,
|
||||
"invalid id %d completed on queue %d\n",
|
||||
cqe.command_id, le16_to_cpu(cqe.sq_id));
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* AEN requests are special as they don't time out and can
|
||||
* survive any kind of queue freeze and often don't respond to
|
||||
|
|
@ -885,8 +808,7 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
|
|||
continue;
|
||||
}
|
||||
|
||||
ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
|
||||
fn(nvmeq, ctx, &cqe);
|
||||
req_completion(nvmeq, &cqe);
|
||||
}
|
||||
|
||||
/* If the controller ignores the cq head doorbell and continuously
|
||||
|
|
@ -1125,29 +1047,18 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
|
|||
static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved)
|
||||
{
|
||||
struct nvme_queue *nvmeq = data;
|
||||
void *ctx;
|
||||
nvme_completion_fn fn;
|
||||
struct nvme_cmd_info *cmd;
|
||||
struct nvme_completion cqe;
|
||||
int status;
|
||||
|
||||
if (!blk_mq_request_started(req))
|
||||
return;
|
||||
|
||||
cmd = blk_mq_rq_to_pdu(req);
|
||||
|
||||
if (cmd->ctx == CMD_CTX_CANCELLED)
|
||||
return;
|
||||
dev_warn(nvmeq->q_dmadev,
|
||||
"Cancelling I/O %d QID %d\n", req->tag, nvmeq->qid);
|
||||
|
||||
status = NVME_SC_CANCELLED;
|
||||
if (blk_queue_dying(req->q))
|
||||
cqe.status = cpu_to_le16((NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1);
|
||||
else
|
||||
cqe.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);
|
||||
|
||||
|
||||
dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n",
|
||||
req->tag, nvmeq->qid);
|
||||
ctx = cancel_cmd_info(cmd, &fn);
|
||||
fn(nvmeq, ctx, &cqe);
|
||||
status |= NVME_SC_DNR;
|
||||
blk_mq_complete_request(req, status);
|
||||
}
|
||||
|
||||
static void nvme_free_queue(struct nvme_queue *nvmeq)
|
||||
|
|
|
|||
Loading…
Reference in New Issue