From 963395269c758641e1cb7208f3bdce6824ea608d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 11 Jun 2020 08:44:49 +0200 Subject: [PATCH] blk-mq: factor out a blk_mq_complete_need_ipi helper Add a helper to decide if we can complete locally or need an IPI. Reviewed-by: Daniel Wagner Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 95125bfe779b..961635b40999 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -654,6 +654,24 @@ static void __blk_mq_complete_request_remote(void *data) __blk_mq_complete_request(data); } +static inline bool blk_mq_complete_need_ipi(struct request *rq) +{ + int cpu = raw_smp_processor_id(); + + if (!IS_ENABLED(CONFIG_SMP) || + !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) + return false; + + /* same CPU or cache domain? Complete locally */ + if (cpu == rq->mq_ctx->cpu || + (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) && + cpus_share_cache(cpu, rq->mq_ctx->cpu))) + return false; + + /* don't try to IPI to an offline CPU */ + return cpu_online(rq->mq_ctx->cpu); +} + /** * blk_mq_complete_request - end I/O on a request * @rq: the request being processed @@ -663,11 +681,6 @@ static void __blk_mq_complete_request_remote(void *data) **/ void blk_mq_complete_request(struct request *rq) { - struct blk_mq_ctx *ctx = rq->mq_ctx; - struct request_queue *q = rq->q; - bool shared = false; - int cpu; - WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); /* @@ -675,25 +688,15 @@ void blk_mq_complete_request(struct request *rq) * to redirect the completion. */ if (rq->cmd_flags & REQ_HIPRI) { - q->mq_ops->complete(rq); + rq->q->mq_ops->complete(rq); return; } - if (!IS_ENABLED(CONFIG_SMP) || - !test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags)) { - __blk_mq_complete_request(rq); - return; - } - - cpu = raw_smp_processor_id(); - if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags)) - shared = cpus_share_cache(cpu, ctx->cpu); - - if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { + if (blk_mq_complete_need_ipi(rq)) { rq->csd.func = __blk_mq_complete_request_remote; rq->csd.info = rq; rq->csd.flags = 0; - smp_call_function_single_async(ctx->cpu, &rq->csd); + smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd); } else { __blk_mq_complete_request(rq); }