From 09d7583294aada625349b6f80f1e1c730b5a5208 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Mon, 17 Sep 2018 17:09:49 -0700 Subject: [PATCH] vhost/scsi: Use common handling code in request queue handler Change the request queue handler to use common handling routines same as the control queue handler. Signed-off-by: Bijan Mottahedeh Signed-off-by: Michael S. Tsirkin --- drivers/vhost/scsi.c | 485 ++++++++++++++++++++----------------------- 1 file changed, 226 insertions(+), 259 deletions(-) diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 4cd03a1d7f21..50dffe83714c 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -813,256 +813,6 @@ vhost_scsi_send_bad_target(struct vhost_scsi *vs, pr_err("Faulted on virtio_scsi_cmd_resp\n"); } -static void -vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq) -{ - struct vhost_scsi_tpg **vs_tpg, *tpg; - struct virtio_scsi_cmd_req v_req; - struct virtio_scsi_cmd_req_pi v_req_pi; - struct vhost_scsi_cmd *cmd; - struct iov_iter out_iter, in_iter, prot_iter, data_iter; - u64 tag; - u32 exp_data_len, data_direction; - unsigned int out = 0, in = 0; - int head, ret, prot_bytes; - size_t req_size, rsp_size = sizeof(struct virtio_scsi_cmd_resp); - size_t out_size, in_size; - u16 lun; - u8 *target, *lunp, task_attr; - bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI); - void *req, *cdb; - - mutex_lock(&vq->mutex); - /* - * We can handle the vq only after the endpoint is setup by calling the - * VHOST_SCSI_SET_ENDPOINT ioctl. - */ - vs_tpg = vq->private_data; - if (!vs_tpg) - goto out; - - vhost_disable_notify(&vs->dev, vq); - - for (;;) { - head = vhost_get_vq_desc(vq, vq->iov, - ARRAY_SIZE(vq->iov), &out, &in, - NULL, NULL); - pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n", - head, out, in); - /* On error, stop handling until the next kick. */ - if (unlikely(head < 0)) - break; - /* Nothing new? Wait for eventfd to tell us they refilled. */ - if (head == vq->num) { - if (unlikely(vhost_enable_notify(&vs->dev, vq))) { - vhost_disable_notify(&vs->dev, vq); - continue; - } - break; - } - /* - * Check for a sane response buffer so we can report early - * errors back to the guest. - */ - if (unlikely(vq->iov[out].iov_len < rsp_size)) { - vq_err(vq, "Expecting at least virtio_scsi_cmd_resp" - " size, got %zu bytes\n", vq->iov[out].iov_len); - break; - } - /* - * Setup pointers and values based upon different virtio-scsi - * request header if T10_PI is enabled in KVM guest. - */ - if (t10_pi) { - req = &v_req_pi; - req_size = sizeof(v_req_pi); - lunp = &v_req_pi.lun[0]; - target = &v_req_pi.lun[1]; - } else { - req = &v_req; - req_size = sizeof(v_req); - lunp = &v_req.lun[0]; - target = &v_req.lun[1]; - } - /* - * FIXME: Not correct for BIDI operation - */ - out_size = iov_length(vq->iov, out); - in_size = iov_length(&vq->iov[out], in); - - /* - * Copy over the virtio-scsi request header, which for a - * ANY_LAYOUT enabled guest may span multiple iovecs, or a - * single iovec may contain both the header + outgoing - * WRITE payloads. - * - * copy_from_iter() will advance out_iter, so that it will - * point at the start of the outgoing WRITE payload, if - * DMA_TO_DEVICE is set. - */ - iov_iter_init(&out_iter, WRITE, vq->iov, out, out_size); - - if (unlikely(!copy_from_iter_full(req, req_size, &out_iter))) { - vq_err(vq, "Faulted on copy_from_iter\n"); - vhost_scsi_send_bad_target(vs, vq, head, out); - continue; - } - /* virtio-scsi spec requires byte 0 of the lun to be 1 */ - if (unlikely(*lunp != 1)) { - vq_err(vq, "Illegal virtio-scsi lun: %u\n", *lunp); - vhost_scsi_send_bad_target(vs, vq, head, out); - continue; - } - - tpg = READ_ONCE(vs_tpg[*target]); - if (unlikely(!tpg)) { - /* Target does not exist, fail the request */ - vhost_scsi_send_bad_target(vs, vq, head, out); - continue; - } - /* - * Determine data_direction by calculating the total outgoing - * iovec sizes + incoming iovec sizes vs. virtio-scsi request + - * response headers respectively. - * - * For DMA_TO_DEVICE this is out_iter, which is already pointing - * to the right place. - * - * For DMA_FROM_DEVICE, the iovec will be just past the end - * of the virtio-scsi response header in either the same - * or immediately following iovec. - * - * Any associated T10_PI bytes for the outgoing / incoming - * payloads are included in calculation of exp_data_len here. - */ - prot_bytes = 0; - - if (out_size > req_size) { - data_direction = DMA_TO_DEVICE; - exp_data_len = out_size - req_size; - data_iter = out_iter; - } else if (in_size > rsp_size) { - data_direction = DMA_FROM_DEVICE; - exp_data_len = in_size - rsp_size; - - iov_iter_init(&in_iter, READ, &vq->iov[out], in, - rsp_size + exp_data_len); - iov_iter_advance(&in_iter, rsp_size); - data_iter = in_iter; - } else { - data_direction = DMA_NONE; - exp_data_len = 0; - } - /* - * If T10_PI header + payload is present, setup prot_iter values - * and recalculate data_iter for vhost_scsi_mapal() mapping to - * host scatterlists via get_user_pages_fast(). - */ - if (t10_pi) { - if (v_req_pi.pi_bytesout) { - if (data_direction != DMA_TO_DEVICE) { - vq_err(vq, "Received non zero pi_bytesout," - " but wrong data_direction\n"); - vhost_scsi_send_bad_target(vs, vq, head, out); - continue; - } - prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout); - } else if (v_req_pi.pi_bytesin) { - if (data_direction != DMA_FROM_DEVICE) { - vq_err(vq, "Received non zero pi_bytesin," - " but wrong data_direction\n"); - vhost_scsi_send_bad_target(vs, vq, head, out); - continue; - } - prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin); - } - /* - * Set prot_iter to data_iter and truncate it to - * prot_bytes, and advance data_iter past any - * preceeding prot_bytes that may be present. - * - * Also fix up the exp_data_len to reflect only the - * actual data payload length. - */ - if (prot_bytes) { - exp_data_len -= prot_bytes; - prot_iter = data_iter; - iov_iter_truncate(&prot_iter, prot_bytes); - iov_iter_advance(&data_iter, prot_bytes); - } - tag = vhost64_to_cpu(vq, v_req_pi.tag); - task_attr = v_req_pi.task_attr; - cdb = &v_req_pi.cdb[0]; - lun = ((v_req_pi.lun[2] << 8) | v_req_pi.lun[3]) & 0x3FFF; - } else { - tag = vhost64_to_cpu(vq, v_req.tag); - task_attr = v_req.task_attr; - cdb = &v_req.cdb[0]; - lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF; - } - /* - * Check that the received CDB size does not exceeded our - * hardcoded max for vhost-scsi, then get a pre-allocated - * cmd descriptor for the new virtio-scsi tag. - * - * TODO what if cdb was too small for varlen cdb header? - */ - if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) { - vq_err(vq, "Received SCSI CDB with command_size: %d that" - " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", - scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE); - vhost_scsi_send_bad_target(vs, vq, head, out); - continue; - } - cmd = vhost_scsi_get_tag(vq, tpg, cdb, tag, lun, task_attr, - exp_data_len + prot_bytes, - data_direction); - if (IS_ERR(cmd)) { - vq_err(vq, "vhost_scsi_get_tag failed %ld\n", - PTR_ERR(cmd)); - vhost_scsi_send_bad_target(vs, vq, head, out); - continue; - } - cmd->tvc_vhost = vs; - cmd->tvc_vq = vq; - cmd->tvc_resp_iov = vq->iov[out]; - cmd->tvc_in_iovs = in; - - pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n", - cmd->tvc_cdb[0], cmd->tvc_lun); - pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:" - " %d\n", cmd, exp_data_len, prot_bytes, data_direction); - - if (data_direction != DMA_NONE) { - ret = vhost_scsi_mapal(cmd, - prot_bytes, &prot_iter, - exp_data_len, &data_iter); - if (unlikely(ret)) { - vq_err(vq, "Failed to map iov to sgl\n"); - vhost_scsi_release_cmd(&cmd->tvc_se_cmd); - vhost_scsi_send_bad_target(vs, vq, head, out); - continue; - } - } - /* - * Save the descriptor from vhost_get_vq_desc() to be used to - * complete the virtio-scsi request in TCM callback context via - * vhost_scsi_queue_data_in() and vhost_scsi_queue_status() - */ - cmd->tvc_vq_desc = head; - /* - * Dispatch cmd descriptor for cmwq execution in process - * context provided by vhost_scsi_workqueue. This also ensures - * cmd is executed on the same kworker CPU as this vhost - * thread to gain positive L2 cache locality effects. - */ - INIT_WORK(&cmd->work, vhost_scsi_submission_work); - queue_work(vhost_scsi_workqueue, &cmd->work); - } -out: - mutex_unlock(&vq->mutex); -} - static int vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc) @@ -1091,6 +841,7 @@ vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq, /* * Get the size of request and response buffers. + * FIXME: Not correct for BIDI operation */ vc->out_size = iov_length(vq->iov, vc->out); vc->in_size = iov_length(&vq->iov[vc->out], vc->in); @@ -1137,20 +888,20 @@ vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc, int ret = -EIO; if (unlikely(!copy_from_iter_full(vc->req, vc->req_size, - &vc->out_iter))) + &vc->out_iter))) { vq_err(vq, "Faulted on copy_from_iter\n"); - else if (unlikely(*vc->lunp != 1)) + } else if (unlikely(*vc->lunp != 1)) { /* virtio-scsi spec requires byte 0 of the lun to be 1 */ vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp); - else { + } else { struct vhost_scsi_tpg **vs_tpg, *tpg; vs_tpg = vq->private_data; /* validated at handler entry */ tpg = READ_ONCE(vs_tpg[*vc->target]); - if (unlikely(!tpg)) + if (unlikely(!tpg)) { vq_err(vq, "Target 0x%x does not exist\n", *vc->target); - else { + } else { if (tpgp) *tpgp = tpg; ret = 0; @@ -1161,9 +912,225 @@ vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc, } static void -vhost_scsi_send_tmf_resp(struct vhost_scsi *vs, - struct vhost_virtqueue *vq, - struct vhost_scsi_ctx *vc) +vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq) +{ + struct vhost_scsi_tpg **vs_tpg, *tpg; + struct virtio_scsi_cmd_req v_req; + struct virtio_scsi_cmd_req_pi v_req_pi; + struct vhost_scsi_ctx vc; + struct vhost_scsi_cmd *cmd; + struct iov_iter in_iter, prot_iter, data_iter; + u64 tag; + u32 exp_data_len, data_direction; + int ret, prot_bytes; + u16 lun; + u8 task_attr; + bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI); + void *cdb; + + mutex_lock(&vq->mutex); + /* + * We can handle the vq only after the endpoint is setup by calling the + * VHOST_SCSI_SET_ENDPOINT ioctl. + */ + vs_tpg = vq->private_data; + if (!vs_tpg) + goto out; + + memset(&vc, 0, sizeof(vc)); + vc.rsp_size = sizeof(struct virtio_scsi_cmd_resp); + + vhost_disable_notify(&vs->dev, vq); + + for (;;) { + ret = vhost_scsi_get_desc(vs, vq, &vc); + if (ret) + goto err; + + /* + * Setup pointers and values based upon different virtio-scsi + * request header if T10_PI is enabled in KVM guest. + */ + if (t10_pi) { + vc.req = &v_req_pi; + vc.req_size = sizeof(v_req_pi); + vc.lunp = &v_req_pi.lun[0]; + vc.target = &v_req_pi.lun[1]; + } else { + vc.req = &v_req; + vc.req_size = sizeof(v_req); + vc.lunp = &v_req.lun[0]; + vc.target = &v_req.lun[1]; + } + + /* + * Validate the size of request and response buffers. + * Check for a sane response buffer so we can report + * early errors back to the guest. + */ + ret = vhost_scsi_chk_size(vq, &vc); + if (ret) + goto err; + + ret = vhost_scsi_get_req(vq, &vc, &tpg); + if (ret) + goto err; + + ret = -EIO; /* bad target on any error from here on */ + + /* + * Determine data_direction by calculating the total outgoing + * iovec sizes + incoming iovec sizes vs. virtio-scsi request + + * response headers respectively. + * + * For DMA_TO_DEVICE this is out_iter, which is already pointing + * to the right place. + * + * For DMA_FROM_DEVICE, the iovec will be just past the end + * of the virtio-scsi response header in either the same + * or immediately following iovec. + * + * Any associated T10_PI bytes for the outgoing / incoming + * payloads are included in calculation of exp_data_len here. + */ + prot_bytes = 0; + + if (vc.out_size > vc.req_size) { + data_direction = DMA_TO_DEVICE; + exp_data_len = vc.out_size - vc.req_size; + data_iter = vc.out_iter; + } else if (vc.in_size > vc.rsp_size) { + data_direction = DMA_FROM_DEVICE; + exp_data_len = vc.in_size - vc.rsp_size; + + iov_iter_init(&in_iter, READ, &vq->iov[vc.out], vc.in, + vc.rsp_size + exp_data_len); + iov_iter_advance(&in_iter, vc.rsp_size); + data_iter = in_iter; + } else { + data_direction = DMA_NONE; + exp_data_len = 0; + } + /* + * If T10_PI header + payload is present, setup prot_iter values + * and recalculate data_iter for vhost_scsi_mapal() mapping to + * host scatterlists via get_user_pages_fast(). + */ + if (t10_pi) { + if (v_req_pi.pi_bytesout) { + if (data_direction != DMA_TO_DEVICE) { + vq_err(vq, "Received non zero pi_bytesout," + " but wrong data_direction\n"); + goto err; + } + prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout); + } else if (v_req_pi.pi_bytesin) { + if (data_direction != DMA_FROM_DEVICE) { + vq_err(vq, "Received non zero pi_bytesin," + " but wrong data_direction\n"); + goto err; + } + prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin); + } + /* + * Set prot_iter to data_iter and truncate it to + * prot_bytes, and advance data_iter past any + * preceeding prot_bytes that may be present. + * + * Also fix up the exp_data_len to reflect only the + * actual data payload length. + */ + if (prot_bytes) { + exp_data_len -= prot_bytes; + prot_iter = data_iter; + iov_iter_truncate(&prot_iter, prot_bytes); + iov_iter_advance(&data_iter, prot_bytes); + } + tag = vhost64_to_cpu(vq, v_req_pi.tag); + task_attr = v_req_pi.task_attr; + cdb = &v_req_pi.cdb[0]; + lun = ((v_req_pi.lun[2] << 8) | v_req_pi.lun[3]) & 0x3FFF; + } else { + tag = vhost64_to_cpu(vq, v_req.tag); + task_attr = v_req.task_attr; + cdb = &v_req.cdb[0]; + lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF; + } + /* + * Check that the received CDB size does not exceeded our + * hardcoded max for vhost-scsi, then get a pre-allocated + * cmd descriptor for the new virtio-scsi tag. + * + * TODO what if cdb was too small for varlen cdb header? + */ + if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) { + vq_err(vq, "Received SCSI CDB with command_size: %d that" + " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", + scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE); + goto err; + } + cmd = vhost_scsi_get_tag(vq, tpg, cdb, tag, lun, task_attr, + exp_data_len + prot_bytes, + data_direction); + if (IS_ERR(cmd)) { + vq_err(vq, "vhost_scsi_get_tag failed %ld\n", + PTR_ERR(cmd)); + goto err; + } + cmd->tvc_vhost = vs; + cmd->tvc_vq = vq; + cmd->tvc_resp_iov = vq->iov[vc.out]; + cmd->tvc_in_iovs = vc.in; + + pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n", + cmd->tvc_cdb[0], cmd->tvc_lun); + pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:" + " %d\n", cmd, exp_data_len, prot_bytes, data_direction); + + if (data_direction != DMA_NONE) { + if (unlikely(vhost_scsi_mapal(cmd, prot_bytes, + &prot_iter, exp_data_len, + &data_iter))) { + vq_err(vq, "Failed to map iov to sgl\n"); + vhost_scsi_release_cmd(&cmd->tvc_se_cmd); + goto err; + } + } + /* + * Save the descriptor from vhost_get_vq_desc() to be used to + * complete the virtio-scsi request in TCM callback context via + * vhost_scsi_queue_data_in() and vhost_scsi_queue_status() + */ + cmd->tvc_vq_desc = vc.head; + /* + * Dispatch cmd descriptor for cmwq execution in process + * context provided by vhost_scsi_workqueue. This also ensures + * cmd is executed on the same kworker CPU as this vhost + * thread to gain positive L2 cache locality effects. + */ + INIT_WORK(&cmd->work, vhost_scsi_submission_work); + queue_work(vhost_scsi_workqueue, &cmd->work); + ret = 0; +err: + /* + * ENXIO: No more requests, or read error, wait for next kick + * EINVAL: Invalid response buffer, drop the request + * EIO: Respond with bad target + * EAGAIN: Pending request + */ + if (ret == -ENXIO) + break; + else if (ret == -EIO) + vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out); + } +out: + mutex_unlock(&vq->mutex); +} + +static void +vhost_scsi_send_tmf_reject(struct vhost_scsi *vs, + struct vhost_virtqueue *vq, + struct vhost_scsi_ctx *vc) { struct virtio_scsi_ctrl_tmf_resp __user *resp; struct virtio_scsi_ctrl_tmf_resp rsp; @@ -1289,7 +1256,7 @@ vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq) goto err; if (v_req.type == VIRTIO_SCSI_T_TMF) - vhost_scsi_send_tmf_resp(vs, vq, &vc); + vhost_scsi_send_tmf_reject(vs, vq, &vc); else vhost_scsi_send_an_resp(vs, vq, &vc); err: