From f083f557186d77de5f14bf90aec1bfe2503cf513 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:10:31 -0700 Subject: [PATCH 01/44] dmaengine: Add helper function to prep for error reporting Dmaengine does not provide a way to pass back the result from a DMA transaction through the callback function. We are adding dmaengine helper function in order to prep for a mechanism that allow result status and other information through the callback. The initial conversion will make the existing driver use these new helper functions but retain the original behavior of the code. However, the helper functions paves a way towards adding the result parameter through callback. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/dmaengine.h | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/drivers/dma/dmaengine.h b/drivers/dma/dmaengine.h index 17f983a4e9ba..94a4379c7639 100644 --- a/drivers/dma/dmaengine.h +++ b/drivers/dma/dmaengine.h @@ -86,4 +86,76 @@ static inline void dma_set_residue(struct dma_tx_state *state, u32 residue) state->residue = residue; } +struct dmaengine_desc_callback { + dma_async_tx_callback callback; + void *callback_param; +}; + +/** + * dmaengine_desc_get_callback - get the passed in callback function + * @tx: tx descriptor + * @cb: temp struct to hold the callback info + * + * Fill the passed in cb struct with what's available in the passed in + * tx descriptor struct + * No locking is required. + */ +static inline void +dmaengine_desc_get_callback(struct dma_async_tx_descriptor *tx, + struct dmaengine_desc_callback *cb) +{ + cb->callback = tx->callback; + cb->callback_param = tx->callback_param; +} + +/** + * dmaengine_desc_callback_invoke - call the callback function in cb struct + * @cb: temp struct that is holding the callback info + * @result: dummy pointer for now + * + * Call the callback function provided in the cb struct with the parameter + * in the cb struct. + * Locking is dependent on the driver. + */ +static inline void +dmaengine_desc_callback_invoke(struct dmaengine_desc_callback *cb, + const void *result) +{ + if (cb->callback) + cb->callback(cb->callback_param); +} + +/** + * dmaengine_desc_get_callback_invoke - get the callback in tx descriptor and + * then immediately call the callback. + * @tx: dma async tx descriptor + * @result: dummy pointer for now + * + * Call dmaengine_desc_get_callback() and dmaengine_desc_callback_invoke() + * in a single function since no work is necessary in between for the driver. + * Locking is dependent on the driver. + */ +static inline void +dmaengine_desc_get_callback_invoke(struct dma_async_tx_descriptor *tx, + const void *result) +{ + struct dmaengine_desc_callback cb; + + dmaengine_desc_get_callback(tx, &cb); + dmaengine_desc_callback_invoke(&cb, result); +} + +/** + * dmaengine_desc_callback_valid - verify the callback is valid in cb + * @cb: callback info struct + * + * Return a bool that verifies whether callback in cb is valid or not. + * No locking is required. + */ +static inline bool +dmaengine_desc_callback_valid(struct dmaengine_desc_callback *cb) +{ + return (cb->callback) ? true : false; +} + #endif From dff232dab98154903970c1e6b36f47c9defbd0a2 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:10:37 -0700 Subject: [PATCH 02/44] dmaengine: at_hdmac: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Cc: Nicolas Ferre Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 53d22eb73b56..a4c8f80db29d 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -473,15 +473,11 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) /* for cyclic transfers, * no need to replay callback function while stopping */ if (!atc_chan_is_cyclic(atchan)) { - dma_async_tx_callback callback = txd->callback; - void *param = txd->callback_param; - /* * The API requires that no submissions are done from a * callback, so we don't need to drop the lock here */ - if (callback) - callback(param); + dmaengine_desc_get_callback_invoke(txd, NULL); } dma_run_dependencies(txd); @@ -598,15 +594,12 @@ static void atc_handle_cyclic(struct at_dma_chan *atchan) { struct at_desc *first = atc_first_active(atchan); struct dma_async_tx_descriptor *txd = &first->txd; - dma_async_tx_callback callback = txd->callback; - void *param = txd->callback_param; dev_vdbg(chan2dev(&atchan->chan_common), "new cyclic period llp 0x%08x\n", channel_readl(atchan, DSCR)); - if (callback) - callback(param); + dmaengine_desc_get_callback_invoke(txd, NULL); } /*-- IRQ & Tasklet ---------------------------------------------------*/ From a1d4eaaf4021a3a0c1299f254faf7b50ca9ddbfa Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:10:42 -0700 Subject: [PATCH 03/44] dmaengine: at_xdmac: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Cc: Ludovic Desroches Signed-off-by: Vinod Koul --- drivers/dma/at_xdmac.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index e434ffe7bc5c..2badc57a7f31 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -1572,8 +1572,8 @@ static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan) desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); txd = &desc->tx_dma_desc; - if (txd->callback && (txd->flags & DMA_PREP_INTERRUPT)) - txd->callback(txd->callback_param); + if (txd->flags & DMA_PREP_INTERRUPT) + dmaengine_desc_get_callback_invoke(txd, NULL); } static void at_xdmac_tasklet(unsigned long data) @@ -1616,8 +1616,8 @@ static void at_xdmac_tasklet(unsigned long data) if (!at_xdmac_chan_is_cyclic(atchan)) { dma_cookie_complete(txd); - if (txd->callback && (txd->flags & DMA_PREP_INTERRUPT)) - txd->callback(txd->callback_param); + if (txd->flags & DMA_PREP_INTERRUPT) + dmaengine_desc_get_callback_invoke(txd, NULL); } dma_run_dependencies(txd); From 3ab553d9f5ade52fc72a49b66bb5c6e356c2ba42 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:10:48 -0700 Subject: [PATCH 04/44] dmaengine: coh901318: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Cc: Linus Walleij Signed-off-by: Vinod Koul --- drivers/dma/coh901318.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c index e4acd63e42aa..81d29f0262b8 100644 --- a/drivers/dma/coh901318.c +++ b/drivers/dma/coh901318.c @@ -1888,8 +1888,7 @@ static void dma_tasklet(unsigned long data) struct coh901318_chan *cohc = (struct coh901318_chan *) data; struct coh901318_desc *cohd_fin; unsigned long flags; - dma_async_tx_callback callback; - void *callback_param; + struct dmaengine_desc_callback cb; dev_vdbg(COHC_2_DEV(cohc), "[%s] chan_id %d" " nbr_active_done %ld\n", __func__, @@ -1904,8 +1903,7 @@ static void dma_tasklet(unsigned long data) goto err; /* locate callback to client */ - callback = cohd_fin->desc.callback; - callback_param = cohd_fin->desc.callback_param; + dmaengine_desc_get_callback(&cohd_fin->desc, &cb); /* sign this job as completed on the channel */ dma_cookie_complete(&cohd_fin->desc); @@ -1920,8 +1918,7 @@ static void dma_tasklet(unsigned long data) spin_unlock_irqrestore(&cohc->lock, flags); /* Call the callback when we're done */ - if (callback) - callback(callback_param); + dmaengine_desc_callback_invoke(&cb, NULL); spin_lock_irqsave(&cohc->lock, flags); From b310a619ab3fc78350e1dab4af0312af99c60b39 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:10:54 -0700 Subject: [PATCH 05/44] dmaengine: cppi41: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/cppi41.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c index 4b2317426c8e..3b4c842b5720 100644 --- a/drivers/dma/cppi41.c +++ b/drivers/dma/cppi41.c @@ -331,7 +331,7 @@ static irqreturn_t cppi41_irq(int irq, void *data) c->residue = pd_trans_len(c->desc->pd6) - len; dma_cookie_complete(&c->txd); - c->txd.callback(c->txd.callback_param); + dmaengine_desc_get_callback_invoke(&c->txd, NULL); } } return IRQ_HANDLED; From 577ef92512a5baf90233ab5d0dd6dd7dca33ee96 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:00 -0700 Subject: [PATCH 06/44] dmaengine: dw: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Cc: Viresh Kumar Signed-off-by: Vinod Koul --- drivers/dma/dw/core.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index edf053f73a49..12eedd457193 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -270,20 +270,19 @@ static void dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc, bool callback_required) { - dma_async_tx_callback callback = NULL; - void *param = NULL; struct dma_async_tx_descriptor *txd = &desc->txd; struct dw_desc *child; unsigned long flags; + struct dmaengine_desc_callback cb; dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie); spin_lock_irqsave(&dwc->lock, flags); dma_cookie_complete(txd); - if (callback_required) { - callback = txd->callback; - param = txd->callback_param; - } + if (callback_required) + dmaengine_desc_get_callback(txd, &cb); + else + memset(&cb, 0, sizeof(cb)); /* async_tx_ack */ list_for_each_entry(child, &desc->tx_list, desc_node) @@ -292,8 +291,7 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc, dwc_desc_put(dwc, desc); spin_unlock_irqrestore(&dwc->lock, flags); - if (callback) - callback(param); + dmaengine_desc_callback_invoke(&cb, NULL); } static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc) From dac86a148a8f363993dfe62931e0fb08d6108136 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:05 -0700 Subject: [PATCH 07/44] dmaengine: ep93xx_dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/ep93xx_dma.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c index 21f08cc3352b..aba0b381cb8c 100644 --- a/drivers/dma/ep93xx_dma.c +++ b/drivers/dma/ep93xx_dma.c @@ -739,10 +739,10 @@ static void ep93xx_dma_tasklet(unsigned long data) { struct ep93xx_dma_chan *edmac = (struct ep93xx_dma_chan *)data; struct ep93xx_dma_desc *desc, *d; - dma_async_tx_callback callback = NULL; - void *callback_param = NULL; + struct dmaengine_desc_callback cb; LIST_HEAD(list); + memset(&cb, 0, sizeof(cb)); spin_lock_irq(&edmac->lock); /* * If dma_terminate_all() was called before we get to run, the active @@ -757,8 +757,7 @@ static void ep93xx_dma_tasklet(unsigned long data) dma_cookie_complete(&desc->txd); list_splice_init(&edmac->active, &list); } - callback = desc->txd.callback; - callback_param = desc->txd.callback_param; + dmaengine_desc_get_callback(&desc->txd, &cb); } spin_unlock_irq(&edmac->lock); @@ -771,8 +770,7 @@ static void ep93xx_dma_tasklet(unsigned long data) ep93xx_dma_desc_put(edmac, desc); } - if (callback) - callback(callback_param); + dmaengine_desc_callback_invoke(&cb, NULL); } static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id) From 1595c3e1bf4dc7d75d8623795040fc71d340ebe3 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:11 -0700 Subject: [PATCH 08/44] dmaengine: fsl_raid: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/fsl_raid.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index aad167eaaee8..35d017a50502 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -134,16 +134,8 @@ static void fsl_re_issue_pending(struct dma_chan *chan) static void fsl_re_desc_done(struct fsl_re_desc *desc) { - dma_async_tx_callback callback; - void *callback_param; - dma_cookie_complete(&desc->async_tx); - - callback = desc->async_tx.callback; - callback_param = desc->async_tx.callback_param; - if (callback) - callback(callback_param); - + dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); dma_descriptor_unmap(&desc->async_tx); } From af1a5a5114e0f3646dbe1be0d42ec1a3a373223f Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:17 -0700 Subject: [PATCH 09/44] dmaengine: fsldma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Cc: Li Yang Signed-off-by: Vinod Koul --- drivers/dma/fsldma.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index 911b7177eb50..ef808665aeca 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -517,11 +517,7 @@ static dma_cookie_t fsldma_run_tx_complete_actions(struct fsldma_chan *chan, ret = txd->cookie; /* Run the link descriptor callback function */ - if (txd->callback) { - chan_dbg(chan, "LD %p callback\n", desc); - txd->callback(txd->callback_param); - } - + dmaengine_desc_get_callback_invoke(txd, NULL); dma_descriptor_unmap(txd); } From be5af2855af50edb7e81a99bc3e8c725839c6eff Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:22 -0700 Subject: [PATCH 10/44] dmaengine: imx-dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/imx-dma.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c index a960608c0a4d..ab0fb804fb1e 100644 --- a/drivers/dma/imx-dma.c +++ b/drivers/dma/imx-dma.c @@ -663,9 +663,7 @@ static void imxdma_tasklet(unsigned long data) out: spin_unlock_irqrestore(&imxdma->lock, flags); - if (desc->desc.callback) - desc->desc.callback(desc->desc.callback_param); - + dmaengine_desc_get_callback_invoke(&desc->desc, NULL); } static int imxdma_terminate_all(struct dma_chan *chan) From 48dc77e2d4fc2e5d85da6e6892f228a75272d040 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:28 -0700 Subject: [PATCH 11/44] dmaengine: imx-sdma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 03ec76fc22ff..624facb6c8f4 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -650,8 +650,7 @@ static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event) static void sdma_handle_channel_loop(struct sdma_channel *sdmac) { - if (sdmac->desc.callback) - sdmac->desc.callback(sdmac->desc.callback_param); + dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL); } static void sdma_update_channel_loop(struct sdma_channel *sdmac) @@ -701,8 +700,8 @@ static void mxc_sdma_handle_channel_normal(struct sdma_channel *sdmac) sdmac->status = DMA_COMPLETE; dma_cookie_complete(&sdmac->desc); - if (sdmac->desc.callback) - sdmac->desc.callback(sdmac->desc.callback_param); + + dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL); } static void sdma_tasklet(unsigned long data) From 63992864a2a55026fb11d1c9c686d348b205ce1f Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:33 -0700 Subject: [PATCH 12/44] dmaengine: ioatdma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/ioat/dma.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index bd09961443b1..6499de4b8e79 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -570,10 +570,8 @@ static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) if (tx->cookie) { dma_cookie_complete(tx); dma_descriptor_unmap(tx); - if (tx->callback) { - tx->callback(tx->callback_param); - tx->callback = NULL; - } + dmaengine_desc_get_callback_invoke(tx, NULL); + tx->callback = NULL; } if (tx->phys == phys_complete) @@ -707,10 +705,8 @@ static void ioat_eh(struct ioatdma_chan *ioat_chan) if (tx->cookie) { dma_cookie_complete(tx); dma_descriptor_unmap(tx); - if (tx->callback) { - tx->callback(tx->callback_param); - tx->callback = NULL; - } + dmaengine_desc_get_callback_invoke(tx, NULL); + tx->callback = NULL; } } From db89e3c87715164abd7e2185a162499da1cdaa55 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:39 -0700 Subject: [PATCH 13/44] dmaengine: iop-adma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/iop-adma.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c index f039cfadf17b..a410657f7bcd 100644 --- a/drivers/dma/iop-adma.c +++ b/drivers/dma/iop-adma.c @@ -71,8 +71,7 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, /* call the callback (must not sleep or submit new * operations to this channel) */ - if (tx->callback) - tx->callback(tx->callback_param); + dmaengine_desc_get_callback_invoke(tx, NULL); dma_descriptor_unmap(tx); if (desc->group_head) From 80a7d64325e79b811dd26254785323ccb1337fd5 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:45 -0700 Subject: [PATCH 14/44] dmaengine: ipu: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/ipu/ipu_idmac.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/drivers/dma/ipu/ipu_idmac.c b/drivers/dma/ipu/ipu_idmac.c index b54f62de9232..ed76044ce4b9 100644 --- a/drivers/dma/ipu/ipu_idmac.c +++ b/drivers/dma/ipu/ipu_idmac.c @@ -1160,11 +1160,10 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id) struct scatterlist **sg, *sgnext, *sgnew = NULL; /* Next transfer descriptor */ struct idmac_tx_desc *desc, *descnew; - dma_async_tx_callback callback; - void *callback_param; bool done = false; u32 ready0, ready1, curbuf, err; unsigned long flags; + struct dmaengine_desc_callback cb; /* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */ @@ -1278,12 +1277,12 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id) if (likely(sgnew) && ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) { - callback = descnew->txd.callback; - callback_param = descnew->txd.callback_param; + dmaengine_desc_get_callback(&descnew->txd, &cb); + list_del_init(&descnew->list); spin_unlock(&ichan->lock); - if (callback) - callback(callback_param); + + dmaengine_desc_callback_invoke(&cb, NULL); spin_lock(&ichan->lock); } @@ -1292,13 +1291,12 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id) if (done) dma_cookie_complete(&desc->txd); - callback = desc->txd.callback; - callback_param = desc->txd.callback_param; + dmaengine_desc_get_callback(&desc->txd, &cb); spin_unlock(&ichan->lock); - if (done && (desc->txd.flags & DMA_PREP_INTERRUPT) && callback) - callback(callback_param); + if (done && (desc->txd.flags & DMA_PREP_INTERRUPT)) + dmaengine_desc_callback_invoke(&cb, NULL); return IRQ_HANDLED; } From 7a883acd3932fa3c24fef524d118b5784abb1c5e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:50 -0700 Subject: [PATCH 15/44] dmaengine: mic_x100_dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Cc: Sudeep Dutt Signed-off-by: Vinod Koul --- drivers/dma/mic_x100_dma.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/dma/mic_x100_dma.c b/drivers/dma/mic_x100_dma.c index 1502b24b7c7d..818255844a3c 100644 --- a/drivers/dma/mic_x100_dma.c +++ b/drivers/dma/mic_x100_dma.c @@ -104,10 +104,8 @@ static void mic_dma_cleanup(struct mic_dma_chan *ch) tx = &ch->tx_array[last_tail]; if (tx->cookie) { dma_cookie_complete(tx); - if (tx->callback) { - tx->callback(tx->callback_param); - tx->callback = NULL; - } + dmaengine_desc_get_callback_invoke(tx, NULL); + tx->callback = NULL; } last_tail = mic_dma_hw_ring_inc(last_tail); } From 9c1e511cc6675f3e2102e4a69a5d6eda18cfacc1 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:11:56 -0700 Subject: [PATCH 16/44] dmaengine: mmp_pdma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/mmp_pdma.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index f4b25fb0d040..eb3a1f42ab06 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -864,19 +864,15 @@ static void dma_do_tasklet(unsigned long data) struct mmp_pdma_desc_sw *desc, *_desc; LIST_HEAD(chain_cleanup); unsigned long flags; + struct dmaengine_desc_callback cb; if (chan->cyclic_first) { - dma_async_tx_callback cb = NULL; - void *cb_data = NULL; - spin_lock_irqsave(&chan->desc_lock, flags); desc = chan->cyclic_first; - cb = desc->async_tx.callback; - cb_data = desc->async_tx.callback_param; + dmaengine_desc_get_callback(&desc->async_tx, &cb); spin_unlock_irqrestore(&chan->desc_lock, flags); - if (cb) - cb(cb_data); + dmaengine_desc_callback_invoke(&cb, NULL); return; } @@ -921,8 +917,8 @@ static void dma_do_tasklet(unsigned long data) /* Remove from the list of transactions */ list_del(&desc->node); /* Run the link descriptor callback function */ - if (txd->callback) - txd->callback(txd->callback_param); + dmaengine_desc_get_callback(txd, &cb); + dmaengine_desc_callback_invoke(&cb, NULL); dma_pool_free(chan->desc_pool, desc, txd->phys); } From 81141bac700568afec626abda493310720077043 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:12:01 -0700 Subject: [PATCH 17/44] dmaengine: mmp_tdma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/mmp_tdma.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c index b3441f57a364..0e933849e80f 100644 --- a/drivers/dma/mmp_tdma.c +++ b/drivers/dma/mmp_tdma.c @@ -349,9 +349,7 @@ static void dma_do_tasklet(unsigned long data) { struct mmp_tdma_chan *tdmac = (struct mmp_tdma_chan *)data; - if (tdmac->desc.callback) - tdmac->desc.callback(tdmac->desc.callback_param); - + dmaengine_desc_get_callback_invoke(&tdmac->desc, NULL); } static void mmp_tdma_free_descriptor(struct mmp_tdma_chan *tdmac) From ad34636885bee29e473bea4e2540fe5fb8efb564 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:12:07 -0700 Subject: [PATCH 18/44] dmaengine: mpc512x_dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/mpc512x_dma.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c index fa86592c7ae1..cacb78e34136 100644 --- a/drivers/dma/mpc512x_dma.c +++ b/drivers/dma/mpc512x_dma.c @@ -411,8 +411,7 @@ static void mpc_dma_process_completed(struct mpc_dma *mdma) list_for_each_entry(mdesc, &list, node) { desc = &mdesc->desc; - if (desc->callback) - desc->callback(desc->callback_param); + dmaengine_desc_get_callback_invoke(desc, NULL); last_cookie = desc->cookie; dma_run_dependencies(desc); From ee7681a48063111c5bcb0385809ec2be90eabd70 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:12:13 -0700 Subject: [PATCH 19/44] dmaengine: mv_xor: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/mv_xor.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index f4c9f98ec35e..f8b5e7424b3a 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -209,10 +209,7 @@ mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc, /* call the callback (must not sleep or submit new * operations to this channel) */ - if (desc->async_tx.callback) - desc->async_tx.callback( - desc->async_tx.callback_param); - + dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); dma_descriptor_unmap(&desc->async_tx); } From 064370c6a1d97dda4b8827c02782587023c5c227 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:12:18 -0700 Subject: [PATCH 20/44] dmaengine: mxs-dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/mxs-dma.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index 60de35251da5..50e64e113ffb 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -326,8 +326,7 @@ static void mxs_dma_tasklet(unsigned long data) { struct mxs_dma_chan *mxs_chan = (struct mxs_dma_chan *) data; - if (mxs_chan->desc.callback) - mxs_chan->desc.callback(mxs_chan->desc.callback_param); + dmaengine_desc_get_callback_invoke(&mxs_chan->desc, NULL); } static int mxs_dma_irq_to_chan(struct mxs_dma_engine *mxs_dma, int irq) From 0024b2ac374490de46bfd35c43002c7800ad14ed Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:12:24 -0700 Subject: [PATCH 21/44] dmaengine: nbpfaxi: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/nbpfaxi.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c index 08c45c185549..09de71519d37 100644 --- a/drivers/dma/nbpfaxi.c +++ b/drivers/dma/nbpfaxi.c @@ -1102,8 +1102,7 @@ static void nbpf_chan_tasklet(unsigned long data) { struct nbpf_channel *chan = (struct nbpf_channel *)data; struct nbpf_desc *desc, *tmp; - dma_async_tx_callback callback; - void *param; + struct dmaengine_desc_callback cb; while (!list_empty(&chan->done)) { bool found = false, must_put, recycling = false; @@ -1151,14 +1150,12 @@ static void nbpf_chan_tasklet(unsigned long data) must_put = false; } - callback = desc->async_tx.callback; - param = desc->async_tx.callback_param; + dmaengine_desc_get_callback(&desc->async_tx, &cb); /* ack and callback completed descriptor */ spin_unlock_irq(&chan->lock); - if (callback) - callback(param); + dmaengine_desc_callback_invoke(&cb, NULL); if (must_put) nbpf_desc_put(desc); From 5c066f7d01917ed76f2b8d395d7da3070ff16455 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:12:29 -0700 Subject: [PATCH 22/44] dmaengine: pch_dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/pch_dma.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index 113605f6fe20..df95727dc2fb 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -357,14 +357,13 @@ static void pdc_chain_complete(struct pch_dma_chan *pd_chan, struct pch_dma_desc *desc) { struct dma_async_tx_descriptor *txd = &desc->txd; - dma_async_tx_callback callback = txd->callback; - void *param = txd->callback_param; + struct dmaengine_desc_callback cb; + dmaengine_desc_get_callback(txd, &cb); list_splice_init(&desc->tx_list, &pd_chan->free_list); list_move(&desc->desc_node, &pd_chan->free_list); - if (callback) - callback(param); + dmaengine_desc_callback_invoke(&cb, NULL); } static void pdc_complete_all(struct pch_dma_chan *pd_chan) From f08462c650a7e2ec5b68adef94505e1c34fdb309 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:12:35 -0700 Subject: [PATCH 23/44] dmaengine: pl330: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/pl330.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 4fc3ffbd5ca0..1ecd4674aa23 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -2039,14 +2039,12 @@ static void pl330_tasklet(unsigned long data) } while (!list_empty(&pch->completed_list)) { - dma_async_tx_callback callback; - void *callback_param; + struct dmaengine_desc_callback cb; desc = list_first_entry(&pch->completed_list, struct dma_pl330_desc, node); - callback = desc->txd.callback; - callback_param = desc->txd.callback_param; + dmaengine_desc_get_callback(&desc->txd, &cb); if (pch->cyclic) { desc->status = PREP; @@ -2064,9 +2062,9 @@ static void pl330_tasklet(unsigned long data) dma_descriptor_unmap(&desc->txd); - if (callback) { + if (dmaengine_desc_callback_valid(&cb)) { spin_unlock_irqrestore(&pch->lock, flags); - callback(callback_param); + dmaengine_desc_callback_invoke(&cb, NULL); spin_lock_irqsave(&pch->lock, flags); } } From 44967bf73352ccc0ea9346fc1c7dd805bc2279b5 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:12:40 -0700 Subject: [PATCH 24/44] dmaengine: ppc4xx_adma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/ppc4xx/adma.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index da3688b94bdc..140f3ed429f4 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -1485,10 +1485,7 @@ static dma_cookie_t ppc440spe_adma_run_tx_complete_actions( /* call the callback (must not sleep or submit new * operations to this channel) */ - if (desc->async_tx.callback) - desc->async_tx.callback( - desc->async_tx.callback_param); - + dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); dma_descriptor_unmap(&desc->async_tx); } From 5ade6683e916df6a00a9747ccc40191fff83a064 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:12:47 -0700 Subject: [PATCH 25/44] dmaengine: qcom_hidma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Acked-by: Sinan Kaya Signed-off-by: Vinod Koul --- drivers/dma/qcom/hidma.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index b2374cd91e45..1197fbf8f30e 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -132,8 +132,8 @@ static void hidma_process_completed(struct hidma_chan *mchan) spin_unlock_irqrestore(&mchan->lock, irqflags); llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); - if (desc->callback && (llstat == DMA_COMPLETE)) - desc->callback(desc->callback_param); + if (llstat == DMA_COMPLETE) + dmaengine_desc_get_callback_invoke(desc, NULL); last_cookie = desc->cookie; dma_run_dependencies(desc); @@ -413,14 +413,9 @@ static int hidma_terminate_channel(struct dma_chan *chan) /* return all user requests */ list_for_each_entry_safe(mdesc, tmp, &list, node) { struct dma_async_tx_descriptor *txd = &mdesc->desc; - dma_async_tx_callback callback = mdesc->desc.callback; - void *param = mdesc->desc.callback_param; dma_descriptor_unmap(txd); - - if (callback) - callback(param); - + dmaengine_desc_get_callback_invoke(txd, NULL); dma_run_dependencies(txd); /* move myself to free_list */ From 964b2fd88bf5fffe2997663b04952f2d109f3b54 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:12:53 -0700 Subject: [PATCH 26/44] dmaengine: sh_rcar-dmac: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/sh/rcar-dmac.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 0dd953884d1d..d1defa4646ba 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -1389,21 +1389,18 @@ static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev) { struct rcar_dmac_chan *chan = dev; struct rcar_dmac_desc *desc; + struct dmaengine_desc_callback cb; spin_lock_irq(&chan->lock); /* For cyclic transfers notify the user after every chunk. */ if (chan->desc.running && chan->desc.running->cyclic) { - dma_async_tx_callback callback; - void *callback_param; - desc = chan->desc.running; - callback = desc->async_tx.callback; - callback_param = desc->async_tx.callback_param; + dmaengine_desc_get_callback(&desc->async_tx, &cb); - if (callback) { + if (dmaengine_desc_callback_valid(&cb)) { spin_unlock_irq(&chan->lock); - callback(callback_param); + dmaengine_desc_callback_invoke(&cb, NULL); spin_lock_irq(&chan->lock); } } @@ -1418,14 +1415,15 @@ static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev) dma_cookie_complete(&desc->async_tx); list_del(&desc->node); - if (desc->async_tx.callback) { + dmaengine_desc_get_callback(&desc->async_tx, &cb); + if (dmaengine_desc_callback_valid(&cb)) { spin_unlock_irq(&chan->lock); /* * We own the only reference to this descriptor, we can * safely dereference it without holding the channel * lock. */ - desc->async_tx.callback(desc->async_tx.callback_param); + dmaengine_desc_callback_invoke(&cb, NULL); spin_lock_irq(&chan->lock); } From b8bdebb98a56c953808388295670f79b88313fd1 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:13:05 -0700 Subject: [PATCH 27/44] dmaengine: sirf-dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/sirf-dma.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c index d8bc3f2a71db..a96e4a480de5 100644 --- a/drivers/dma/sirf-dma.c +++ b/drivers/dma/sirf-dma.c @@ -360,9 +360,7 @@ static void sirfsoc_dma_process_completed(struct sirfsoc_dma *sdma) list_for_each_entry(sdesc, &list, node) { desc = &sdesc->desc; - if (desc->callback) - desc->callback(desc->callback_param); - + dmaengine_desc_get_callback_invoke(desc, NULL); last_cookie = desc->cookie; dma_run_dependencies(desc); } @@ -388,8 +386,7 @@ static void sirfsoc_dma_process_completed(struct sirfsoc_dma *sdma) desc = &sdesc->desc; while (happened_cyclic != schan->completed_cyclic) { - if (desc->callback) - desc->callback(desc->callback_param); + dmaengine_desc_get_callback_invoke(desc, NULL); schan->completed_cyclic++; } } From 3a315d5d4b3ba18062e8c5f0cfa5373768cb91d6 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:13:10 -0700 Subject: [PATCH 28/44] dmaengine: ste_dma40: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Cc: Linus Walleij Signed-off-by: Vinod Koul --- drivers/dma/ste_dma40.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 8b18e44a02d5..73203ac83734 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -1596,8 +1596,7 @@ static void dma_tasklet(unsigned long data) struct d40_desc *d40d; unsigned long flags; bool callback_active; - dma_async_tx_callback callback; - void *callback_param; + struct dmaengine_desc_callback cb; spin_lock_irqsave(&d40c->lock, flags); @@ -1624,8 +1623,7 @@ static void dma_tasklet(unsigned long data) /* Callback to client */ callback_active = !!(d40d->txd.flags & DMA_PREP_INTERRUPT); - callback = d40d->txd.callback; - callback_param = d40d->txd.callback_param; + dmaengine_desc_get_callback(&d40d->txd, &cb); if (!d40d->cyclic) { if (async_tx_test_ack(&d40d->txd)) { @@ -1646,8 +1644,8 @@ static void dma_tasklet(unsigned long data) spin_unlock_irqrestore(&d40c->lock, flags); - if (callback_active && callback) - callback(callback_param); + if (callback_active) + dmaengine_desc_callback_invoke(&cb, NULL); return; From 370c0446af5e3c8ddefbb753a1df84c278fff6d0 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:13:16 -0700 Subject: [PATCH 29/44] dmaengine: tegra20-apb-dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Cc: Laxman Dewangan Signed-off-by: Vinod Koul --- drivers/dma/tegra20-apb-dma.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index 6ab9eb98588a..3722b9d8d9fe 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -655,8 +655,7 @@ static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, static void tegra_dma_tasklet(unsigned long data) { struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data; - dma_async_tx_callback callback = NULL; - void *callback_param = NULL; + struct dmaengine_desc_callback cb; struct tegra_dma_desc *dma_desc; unsigned long flags; int cb_count; @@ -666,13 +665,12 @@ static void tegra_dma_tasklet(unsigned long data) dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), cb_node); list_del(&dma_desc->cb_node); - callback = dma_desc->txd.callback; - callback_param = dma_desc->txd.callback_param; + dmaengine_desc_get_callback(&dma_desc->txd, &cb); cb_count = dma_desc->cb_count; dma_desc->cb_count = 0; spin_unlock_irqrestore(&tdc->lock, flags); - while (cb_count-- && callback) - callback(callback_param); + while (cb_count--) + dmaengine_desc_callback_invoke(&cb, NULL); spin_lock_irqsave(&tdc->lock, flags); } spin_unlock_irqrestore(&tdc->lock, flags); From a06a5bb908248cdc4f696ae5b6334e3b2e150fbc Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:13:22 -0700 Subject: [PATCH 30/44] dmaengine: timb_dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/timb_dma.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c index e82745aa42a8..896bafb7a532 100644 --- a/drivers/dma/timb_dma.c +++ b/drivers/dma/timb_dma.c @@ -226,8 +226,7 @@ static void __td_start_dma(struct timb_dma_chan *td_chan) static void __td_finish(struct timb_dma_chan *td_chan) { - dma_async_tx_callback callback; - void *param; + struct dmaengine_desc_callback cb; struct dma_async_tx_descriptor *txd; struct timb_dma_desc *td_desc; @@ -252,8 +251,7 @@ static void __td_finish(struct timb_dma_chan *td_chan) dma_cookie_complete(txd); td_chan->ongoing = false; - callback = txd->callback; - param = txd->callback_param; + dmaengine_desc_get_callback(txd, &cb); list_move(&td_desc->desc_node, &td_chan->free_list); @@ -262,8 +260,7 @@ static void __td_finish(struct timb_dma_chan *td_chan) * The API requires that no submissions are done from a * callback, so we don't need to drop the lock here */ - if (callback) - callback(param); + dmaengine_desc_callback_invoke(&cb, NULL); } static u32 __td_ier_mask(struct timb_dma *td) From d254c8d0a725f566dc83d4dda81980c63d7f3838 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:13:28 -0700 Subject: [PATCH 31/44] dmaengine: txx9dmac: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/txx9dmac.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c index 7632290e7c14..4d8c7b9078fd 100644 --- a/drivers/dma/txx9dmac.c +++ b/drivers/dma/txx9dmac.c @@ -403,16 +403,14 @@ static void txx9dmac_descriptor_complete(struct txx9dmac_chan *dc, struct txx9dmac_desc *desc) { - dma_async_tx_callback callback; - void *param; + struct dmaengine_desc_callback cb; struct dma_async_tx_descriptor *txd = &desc->txd; dev_vdbg(chan2dev(&dc->chan), "descriptor %u %p complete\n", txd->cookie, desc); dma_cookie_complete(txd); - callback = txd->callback; - param = txd->callback_param; + dmaengine_desc_get_callback(txd, &cb); txx9dmac_sync_desc_for_cpu(dc, desc); list_splice_init(&desc->tx_list, &dc->free_list); @@ -423,8 +421,7 @@ txx9dmac_descriptor_complete(struct txx9dmac_chan *dc, * The API requires that no submissions are done from a * callback, so we don't need to drop the lock here */ - if (callback) - callback(param); + dmaengine_desc_callback_invoke(&cb, NULL); dma_run_dependencies(txd); } From 4f03ac6a2da050591e6e78921cacdff1546418c3 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:13:33 -0700 Subject: [PATCH 32/44] dmaengine: virt-dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/virt-dma.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/drivers/dma/virt-dma.c b/drivers/dma/virt-dma.c index a35c211857dd..e47fc9b0944f 100644 --- a/drivers/dma/virt-dma.c +++ b/drivers/dma/virt-dma.c @@ -87,8 +87,7 @@ static void vchan_complete(unsigned long arg) { struct virt_dma_chan *vc = (struct virt_dma_chan *)arg; struct virt_dma_desc *vd; - dma_async_tx_callback cb = NULL; - void *cb_data = NULL; + struct dmaengine_desc_callback cb; LIST_HEAD(head); spin_lock_irq(&vc->lock); @@ -96,18 +95,17 @@ static void vchan_complete(unsigned long arg) vd = vc->cyclic; if (vd) { vc->cyclic = NULL; - cb = vd->tx.callback; - cb_data = vd->tx.callback_param; + dmaengine_desc_get_callback(&vd->tx, &cb); + } else { + memset(&cb, 0, sizeof(cb)); } spin_unlock_irq(&vc->lock); - if (cb) - cb(cb_data); + dmaengine_desc_callback_invoke(&cb, NULL); while (!list_empty(&head)) { vd = list_first_entry(&head, struct virt_dma_desc, node); - cb = vd->tx.callback; - cb_data = vd->tx.callback_param; + dmaengine_desc_get_callback(&vd->tx, &cb); list_del(&vd->node); if (dmaengine_desc_test_reuse(&vd->tx)) @@ -115,8 +113,7 @@ static void vchan_complete(unsigned long arg) else vc->desc_free(vd); - if (cb) - cb(cb_data); + dmaengine_desc_callback_invoke(&cb, NULL); } } From b1f884a5ff23534c3808926aeb62187be55e751e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:13:39 -0700 Subject: [PATCH 33/44] dmaengine: xgene-dma: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/xgene-dma.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c index 9cb93c5b655d..d66ed11baaec 100644 --- a/drivers/dma/xgene-dma.c +++ b/drivers/dma/xgene-dma.c @@ -608,8 +608,7 @@ static void xgene_dma_run_tx_complete_actions(struct xgene_dma_chan *chan, dma_cookie_complete(tx); /* Run the link descriptor callback function */ - if (tx->callback) - tx->callback(tx->callback_param); + dmaengine_desc_get_callback_invoke(tx, NULL); dma_descriptor_unmap(tx); From f067025bc676ba8d18fba5f959598339e39b86db Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:13:50 -0700 Subject: [PATCH 34/44] dmaengine: add support to provide error result from a DMA transation Adding a new callback that will provide the error result for a transaction. The result is allocated on the stack and the callback should create a copy if it wishes to retain the information after exiting. The result parameter is now defined and takes over the dummy void pointer we placed in the helper functions previously. dmaengine drivers should start converting to the new "callback_result" callback in order to receive transaction results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/dmaengine.h | 22 +++++++++++++++++----- include/linux/dmaengine.h | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/drivers/dma/dmaengine.h b/drivers/dma/dmaengine.h index 94a4379c7639..882ff9448c3b 100644 --- a/drivers/dma/dmaengine.h +++ b/drivers/dma/dmaengine.h @@ -88,6 +88,7 @@ static inline void dma_set_residue(struct dma_tx_state *state, u32 residue) struct dmaengine_desc_callback { dma_async_tx_callback callback; + dma_async_tx_callback_result callback_result; void *callback_param; }; @@ -105,13 +106,14 @@ dmaengine_desc_get_callback(struct dma_async_tx_descriptor *tx, struct dmaengine_desc_callback *cb) { cb->callback = tx->callback; + cb->callback_result = tx->callback_result; cb->callback_param = tx->callback_param; } /** * dmaengine_desc_callback_invoke - call the callback function in cb struct * @cb: temp struct that is holding the callback info - * @result: dummy pointer for now + * @result: transaction result * * Call the callback function provided in the cb struct with the parameter * in the cb struct. @@ -119,17 +121,27 @@ dmaengine_desc_get_callback(struct dma_async_tx_descriptor *tx, */ static inline void dmaengine_desc_callback_invoke(struct dmaengine_desc_callback *cb, - const void *result) + const struct dmaengine_result *result) { - if (cb->callback) + struct dmaengine_result dummy_result = { + .result = DMA_TRANS_NOERROR, + .residue = 0 + }; + + if (cb->callback_result) { + if (!result) + result = &dummy_result; + cb->callback_result(cb->callback_param, result); + } else if (cb->callback) { cb->callback(cb->callback_param); + } } /** * dmaengine_desc_get_callback_invoke - get the callback in tx descriptor and * then immediately call the callback. * @tx: dma async tx descriptor - * @result: dummy pointer for now + * @result: transaction result * * Call dmaengine_desc_get_callback() and dmaengine_desc_callback_invoke() * in a single function since no work is necessary in between for the driver. @@ -137,7 +149,7 @@ dmaengine_desc_callback_invoke(struct dmaengine_desc_callback *cb, */ static inline void dmaengine_desc_get_callback_invoke(struct dma_async_tx_descriptor *tx, - const void *result) + const struct dmaengine_result *result) { struct dmaengine_desc_callback cb; diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 30de0197263a..cc535a478bae 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -441,6 +441,21 @@ typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param); typedef void (*dma_async_tx_callback)(void *dma_async_param); +enum dmaengine_tx_result { + DMA_TRANS_NOERROR = 0, /* SUCCESS */ + DMA_TRANS_READ_FAILED, /* Source DMA read failed */ + DMA_TRANS_WRITE_FAILED, /* Destination DMA write failed */ + DMA_TRANS_ABORTED, /* Op never submitted / aborted */ +}; + +struct dmaengine_result { + enum dmaengine_tx_result result; + u32 residue; +}; + +typedef void (*dma_async_tx_callback_result)(void *dma_async_param, + const struct dmaengine_result *result); + struct dmaengine_unmap_data { u8 map_cnt; u8 to_cnt; @@ -478,6 +493,7 @@ struct dma_async_tx_descriptor { dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx); int (*desc_free)(struct dma_async_tx_descriptor *tx); dma_async_tx_callback callback; + dma_async_tx_callback_result callback_result; void *callback_param; struct dmaengine_unmap_data *unmap; #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH From 9546d4cdc8445acdea415f70a330bbfbd016a0f0 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:13:55 -0700 Subject: [PATCH 35/44] dmaengine: ioatdma: Add error handling to ioat driver Adding error handling to the ioatdma driver so that when a read/write error occurs the error results are reported back and all the remaining descriptors are aborted. This utilizes the new dmaengine callback function that allows reporting of results. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/ioat/dma.c | 140 +++++++++++++++++++++++++++++++---- drivers/dma/ioat/registers.h | 2 + 2 files changed, 126 insertions(+), 16 deletions(-) diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 6499de4b8e79..251f8be639c2 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -568,10 +568,14 @@ static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) tx = &desc->txd; if (tx->cookie) { + struct dmaengine_result res; + dma_cookie_complete(tx); dma_descriptor_unmap(tx); + res.result = DMA_TRANS_NOERROR; dmaengine_desc_get_callback_invoke(tx, NULL); tx->callback = NULL; + tx->callback_result = NULL; } if (tx->phys == phys_complete) @@ -620,7 +624,8 @@ static void ioat_cleanup(struct ioatdma_chan *ioat_chan) if (is_ioat_halted(*ioat_chan->completion)) { u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); - if (chanerr & IOAT_CHANERR_HANDLE_MASK) { + if (chanerr & + (IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) { mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); ioat_eh(ioat_chan); } @@ -650,6 +655,61 @@ static void ioat_restart_channel(struct ioatdma_chan *ioat_chan) __ioat_restart_chan(ioat_chan); } + +static void ioat_abort_descs(struct ioatdma_chan *ioat_chan) +{ + struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; + struct ioat_ring_ent *desc; + u16 active; + int idx = ioat_chan->tail, i; + + /* + * We assume that the failed descriptor has been processed. + * Now we are just returning all the remaining submitted + * descriptors to abort. + */ + active = ioat_ring_active(ioat_chan); + + /* we skip the failed descriptor that tail points to */ + for (i = 1; i < active; i++) { + struct dma_async_tx_descriptor *tx; + + smp_read_barrier_depends(); + prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1)); + desc = ioat_get_ring_ent(ioat_chan, idx + i); + + tx = &desc->txd; + if (tx->cookie) { + struct dmaengine_result res; + + dma_cookie_complete(tx); + dma_descriptor_unmap(tx); + res.result = DMA_TRANS_ABORTED; + dmaengine_desc_get_callback_invoke(tx, &res); + tx->callback = NULL; + tx->callback_result = NULL; + } + + /* skip extended descriptors */ + if (desc_has_ext(desc)) { + WARN_ON(i + 1 >= active); + i++; + } + + /* cleanup super extended descriptors */ + if (desc->sed) { + ioat_free_sed(ioat_dma, desc->sed); + desc->sed = NULL; + } + } + + smp_mb(); /* finish all descriptor reads before incrementing tail */ + ioat_chan->tail = idx + active; + + desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail); + ioat_chan->last_completion = *ioat_chan->completion = desc->txd.phys; +} + static void ioat_eh(struct ioatdma_chan *ioat_chan) { struct pci_dev *pdev = to_pdev(ioat_chan); @@ -660,6 +720,8 @@ static void ioat_eh(struct ioatdma_chan *ioat_chan) u32 err_handled = 0; u32 chanerr_int; u32 chanerr; + bool abort = false; + struct dmaengine_result res; /* cleanup so tail points to descriptor that caused the error */ if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) @@ -695,28 +757,50 @@ static void ioat_eh(struct ioatdma_chan *ioat_chan) break; } + if (chanerr & IOAT_CHANERR_RECOVER_MASK) { + if (chanerr & IOAT_CHANERR_READ_DATA_ERR) { + res.result = DMA_TRANS_READ_FAILED; + err_handled |= IOAT_CHANERR_READ_DATA_ERR; + } else if (chanerr & IOAT_CHANERR_WRITE_DATA_ERR) { + res.result = DMA_TRANS_WRITE_FAILED; + err_handled |= IOAT_CHANERR_WRITE_DATA_ERR; + } + + abort = true; + } else + res.result = DMA_TRANS_NOERROR; + /* fault on unhandled error or spurious halt */ if (chanerr ^ err_handled || chanerr == 0) { dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n", __func__, chanerr, err_handled); BUG(); - } else { /* cleanup the faulty descriptor */ - tx = &desc->txd; - if (tx->cookie) { - dma_cookie_complete(tx); - dma_descriptor_unmap(tx); - dmaengine_desc_get_callback_invoke(tx, NULL); - tx->callback = NULL; - } } - writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); - pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int); + /* cleanup the faulty descriptor since we are continuing */ + tx = &desc->txd; + if (tx->cookie) { + dma_cookie_complete(tx); + dma_descriptor_unmap(tx); + dmaengine_desc_get_callback_invoke(tx, &res); + tx->callback = NULL; + tx->callback_result = NULL; + } /* mark faulting descriptor as complete */ *ioat_chan->completion = desc->txd.phys; spin_lock_bh(&ioat_chan->prep_lock); + /* we need abort all descriptors */ + if (abort) { + ioat_abort_descs(ioat_chan); + /* clean up the channel, we could be in weird state */ + ioat_reset_hw(ioat_chan); + } + + writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); + pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int); + ioat_restart_channel(ioat_chan); spin_unlock_bh(&ioat_chan->prep_lock); } @@ -749,10 +833,25 @@ void ioat_timer_event(unsigned long data) chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n", __func__, chanerr); - if (test_bit(IOAT_RUN, &ioat_chan->state)) - BUG_ON(is_ioat_bug(chanerr)); - else /* we never got off the ground */ - return; + if (test_bit(IOAT_RUN, &ioat_chan->state)) { + spin_lock_bh(&ioat_chan->cleanup_lock); + spin_lock_bh(&ioat_chan->prep_lock); + set_bit(IOAT_CHAN_DOWN, &ioat_chan->state); + spin_unlock_bh(&ioat_chan->prep_lock); + + ioat_abort_descs(ioat_chan); + dev_warn(to_dev(ioat_chan), "Reset channel...\n"); + ioat_reset_hw(ioat_chan); + dev_warn(to_dev(ioat_chan), "Restart channel...\n"); + ioat_restart_channel(ioat_chan); + + spin_lock_bh(&ioat_chan->prep_lock); + clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state); + spin_unlock_bh(&ioat_chan->prep_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); + } + + return; } spin_lock_bh(&ioat_chan->cleanup_lock); @@ -776,14 +875,23 @@ void ioat_timer_event(unsigned long data) u32 chanerr; chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); - dev_warn(to_dev(ioat_chan), "Restarting channel...\n"); dev_warn(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n", status, chanerr); dev_warn(to_dev(ioat_chan), "Active descriptors: %d\n", ioat_ring_active(ioat_chan)); spin_lock_bh(&ioat_chan->prep_lock); + set_bit(IOAT_CHAN_DOWN, &ioat_chan->state); + spin_unlock_bh(&ioat_chan->prep_lock); + + ioat_abort_descs(ioat_chan); + dev_warn(to_dev(ioat_chan), "Resetting channel...\n"); + ioat_reset_hw(ioat_chan); + dev_warn(to_dev(ioat_chan), "Restarting channel...\n"); ioat_restart_channel(ioat_chan); + + spin_lock_bh(&ioat_chan->prep_lock); + clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state); spin_unlock_bh(&ioat_chan->prep_lock); spin_unlock_bh(&ioat_chan->cleanup_lock); return; diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index 70534981a49b..48fa4cf9f64a 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -240,6 +240,8 @@ #define IOAT_CHANERR_DESCRIPTOR_COUNT_ERR 0x40000 #define IOAT_CHANERR_HANDLE_MASK (IOAT_CHANERR_XOR_P_OR_CRC_ERR | IOAT_CHANERR_XOR_Q_ERR) +#define IOAT_CHANERR_RECOVER_MASK (IOAT_CHANERR_READ_DATA_ERR | \ + IOAT_CHANERR_WRITE_DATA_ERR) #define IOAT_CHANERR_MASK_OFFSET 0x2C /* 32-bit Channel Error Register */ From aed681d1dc72914d448e44a99e1dc89baa32d25c Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:14:01 -0700 Subject: [PATCH 36/44] dmaengine: ioatdma: add error strings to chanerr output Provide a mechanism to translate CHANERR bits to English strings in order to allow user to report more concise errors. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- drivers/dma/ioat/dma.c | 65 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 251f8be639c2..49386ce04bf5 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -38,8 +38,54 @@ #include "../dmaengine.h" +static char *chanerr_str[] = { + "DMA Transfer Destination Address Error", + "Next Descriptor Address Error", + "Descriptor Error", + "Chan Address Value Error", + "CHANCMD Error", + "Chipset Uncorrectable Data Integrity Error", + "DMA Uncorrectable Data Integrity Error", + "Read Data Error", + "Write Data Error", + "Descriptor Control Error", + "Descriptor Transfer Size Error", + "Completion Address Error", + "Interrupt Configuration Error", + "Super extended descriptor Address Error", + "Unaffiliated Error", + "CRC or XOR P Error", + "XOR Q Error", + "Descriptor Count Error", + "DIF All F detect Error", + "Guard Tag verification Error", + "Application Tag verification Error", + "Reference Tag verification Error", + "Bundle Bit Error", + "Result DIF All F detect Error", + "Result Guard Tag verification Error", + "Result Application Tag verification Error", + "Result Reference Tag verification Error", + NULL +}; + static void ioat_eh(struct ioatdma_chan *ioat_chan); +static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr) +{ + int i; + + for (i = 0; i < 32; i++) { + if ((chanerr >> i) & 1) { + if (chanerr_str[i]) { + dev_err(to_dev(ioat_chan), "Err(%d): %s\n", + i, chanerr_str[i]); + } else + break; + } + } +} + /** * ioat_dma_do_interrupt - handler used for single vector interrupt mode * @irq: interrupt id @@ -774,6 +820,11 @@ static void ioat_eh(struct ioatdma_chan *ioat_chan) if (chanerr ^ err_handled || chanerr == 0) { dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n", __func__, chanerr, err_handled); + dev_err(to_dev(ioat_chan), "Errors handled:\n"); + ioat_print_chanerrs(ioat_chan, err_handled); + dev_err(to_dev(ioat_chan), "Errors not handled:\n"); + ioat_print_chanerrs(ioat_chan, (chanerr & ~err_handled)); + BUG(); } @@ -833,6 +884,9 @@ void ioat_timer_event(unsigned long data) chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n", __func__, chanerr); + dev_err(to_dev(ioat_chan), "Errors:\n"); + ioat_print_chanerrs(ioat_chan, chanerr); + if (test_bit(IOAT_RUN, &ioat_chan->state)) { spin_lock_bh(&ioat_chan->cleanup_lock); spin_lock_bh(&ioat_chan->prep_lock); @@ -875,10 +929,13 @@ void ioat_timer_event(unsigned long data) u32 chanerr; chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); - dev_warn(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n", - status, chanerr); - dev_warn(to_dev(ioat_chan), "Active descriptors: %d\n", - ioat_ring_active(ioat_chan)); + dev_err(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n", + status, chanerr); + dev_err(to_dev(ioat_chan), "Errors:\n"); + ioat_print_chanerrs(ioat_chan, chanerr); + + dev_dbg(to_dev(ioat_chan), "Active descriptors: %d\n", + ioat_ring_active(ioat_chan)); spin_lock_bh(&ioat_chan->prep_lock); set_bit(IOAT_CHAN_DOWN, &ioat_chan->state); From 9cabc2691e9d21b840b145a944f09299f895a7e0 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:14:07 -0700 Subject: [PATCH 37/44] ntb: add DMA error handling for TX DMA Adding support on the tx DMA path to allow recovery of errors when DMA responds with error status and abort all the subsequent ops. Signed-off-by: Dave Jiang Acked-by: Allen Hubbe Cc: Jon Mason Cc: linux-ntb@googlegroups.com Signed-off-by: Vinod Koul --- drivers/ntb/ntb_transport.c | 110 +++++++++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 27 deletions(-) diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index d5c5894f252e..e61db11e6ccc 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -102,6 +102,9 @@ struct ntb_queue_entry { void *buf; unsigned int len; unsigned int flags; + int retries; + int errors; + unsigned int tx_index; struct ntb_transport_qp *qp; union { @@ -259,6 +262,9 @@ enum { static void ntb_transport_rxc_db(unsigned long data); static const struct ntb_ctx_ops ntb_transport_ops; static struct ntb_client ntb_transport_client; +static int ntb_async_tx_submit(struct ntb_transport_qp *qp, + struct ntb_queue_entry *entry); +static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset); static int ntb_transport_bus_match(struct device *dev, struct device_driver *drv) @@ -1467,12 +1473,39 @@ static void ntb_transport_rxc_db(unsigned long data) } } -static void ntb_tx_copy_callback(void *data) +static void ntb_tx_copy_callback(void *data, + const struct dmaengine_result *res) { struct ntb_queue_entry *entry = data; struct ntb_transport_qp *qp = entry->qp; struct ntb_payload_header __iomem *hdr = entry->tx_hdr; + /* we need to check DMA results if we are using DMA */ + if (res) { + enum dmaengine_tx_result dma_err = res->result; + + switch (dma_err) { + case DMA_TRANS_READ_FAILED: + case DMA_TRANS_WRITE_FAILED: + entry->errors++; + case DMA_TRANS_ABORTED: + { + void __iomem *offset = + qp->tx_mw + qp->tx_max_frame * + entry->tx_index; + + /* resubmit via CPU */ + ntb_memcpy_tx(entry, offset); + qp->tx_memcpy++; + return; + } + + case DMA_TRANS_NOERROR: + default: + break; + } + } + iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); @@ -1507,40 +1540,25 @@ static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) /* Ensure that the data is fully copied out before setting the flags */ wmb(); - ntb_tx_copy_callback(entry); + ntb_tx_copy_callback(entry, NULL); } -static void ntb_async_tx(struct ntb_transport_qp *qp, - struct ntb_queue_entry *entry) +static int ntb_async_tx_submit(struct ntb_transport_qp *qp, + struct ntb_queue_entry *entry) { - struct ntb_payload_header __iomem *hdr; struct dma_async_tx_descriptor *txd; struct dma_chan *chan = qp->tx_dma_chan; struct dma_device *device; + size_t len = entry->len; + void *buf = entry->buf; size_t dest_off, buff_off; struct dmaengine_unmap_data *unmap; dma_addr_t dest; dma_cookie_t cookie; - void __iomem *offset; - size_t len = entry->len; - void *buf = entry->buf; int retries = 0; - offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; - hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); - entry->tx_hdr = hdr; - - iowrite32(entry->len, &hdr->len); - iowrite32((u32)qp->tx_pkts, &hdr->ver); - - if (!chan) - goto err; - - if (len < copy_bytes) - goto err; - device = chan->device; - dest = qp->tx_mw_phys + qp->tx_max_frame * qp->tx_index; + dest = qp->tx_mw_phys + qp->tx_max_frame * entry->tx_index; buff_off = (size_t)buf & ~PAGE_MASK; dest_off = (size_t)dest & ~PAGE_MASK; @@ -1560,8 +1578,9 @@ static void ntb_async_tx(struct ntb_transport_qp *qp, unmap->to_cnt = 1; for (retries = 0; retries < DMA_RETRIES; retries++) { - txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], - len, DMA_PREP_INTERRUPT); + txd = device->device_prep_dma_memcpy(chan, dest, + unmap->addr[0], len, + DMA_PREP_INTERRUPT); if (txd) break; @@ -1574,7 +1593,7 @@ static void ntb_async_tx(struct ntb_transport_qp *qp, goto err_get_unmap; } - txd->callback = ntb_tx_copy_callback; + txd->callback_result = ntb_tx_copy_callback; txd->callback_param = entry; dma_set_unmap(txd, unmap); @@ -1585,13 +1604,47 @@ static void ntb_async_tx(struct ntb_transport_qp *qp, dmaengine_unmap_put(unmap); dma_async_issue_pending(chan); - qp->tx_async++; - return; + return 0; err_set_unmap: dmaengine_unmap_put(unmap); err_get_unmap: dmaengine_unmap_put(unmap); +err: + return -ENXIO; +} + +static void ntb_async_tx(struct ntb_transport_qp *qp, + struct ntb_queue_entry *entry) +{ + struct ntb_payload_header __iomem *hdr; + struct dma_chan *chan = qp->tx_dma_chan; + void __iomem *offset; + int res; + + entry->tx_index = qp->tx_index; + offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index; + hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); + entry->tx_hdr = hdr; + + iowrite32(entry->len, &hdr->len); + iowrite32((u32)qp->tx_pkts, &hdr->ver); + + if (!chan) + goto err; + + if (entry->len < copy_bytes) + goto err; + + res = ntb_async_tx_submit(qp, entry); + if (res < 0) + goto err; + + if (!entry->retries) + qp->tx_async++; + + return; + err: ntb_memcpy_tx(entry, offset); qp->tx_memcpy++; @@ -1970,6 +2023,9 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, entry->buf = data; entry->len = len; entry->flags = 0; + entry->errors = 0; + entry->retries = 0; + entry->tx_index = 0; rc = ntb_process_tx(qp, entry); if (rc) From 72203572afd7aef243c182f19925e5a77a1dc6a1 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:14:13 -0700 Subject: [PATCH 38/44] ntb: add DMA error handling for RX DMA Adding support on the rx DMA path to allow recovery of errors when DMA responds with error status and abort all the subsequent ops. Signed-off-by: Dave Jiang Acked-by: Allen Hubbe Cc: Jon Mason Cc: linux-ntb@googlegroups.com Signed-off-by: Vinod Koul --- drivers/ntb/ntb_transport.c | 83 ++++++++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 16 deletions(-) diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index e61db11e6ccc..8601c10acf74 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -105,13 +105,13 @@ struct ntb_queue_entry { int retries; int errors; unsigned int tx_index; + unsigned int rx_index; struct ntb_transport_qp *qp; union { struct ntb_payload_header __iomem *tx_hdr; struct ntb_payload_header *rx_hdr; }; - unsigned int index; }; struct ntb_rx_info { @@ -265,6 +265,9 @@ static struct ntb_client ntb_transport_client; static int ntb_async_tx_submit(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry); static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset); +static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset); +static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset); + static int ntb_transport_bus_match(struct device *dev, struct device_driver *drv) @@ -1235,7 +1238,7 @@ static void ntb_complete_rxc(struct ntb_transport_qp *qp) break; entry->rx_hdr->flags = 0; - iowrite32(entry->index, &qp->rx_info->entry); + iowrite32(entry->rx_index, &qp->rx_info->entry); cb_data = entry->cb_data; len = entry->len; @@ -1253,10 +1256,36 @@ static void ntb_complete_rxc(struct ntb_transport_qp *qp) spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); } -static void ntb_rx_copy_callback(void *data) +static void ntb_rx_copy_callback(void *data, + const struct dmaengine_result *res) { struct ntb_queue_entry *entry = data; + /* we need to check DMA results if we are using DMA */ + if (res) { + enum dmaengine_tx_result dma_err = res->result; + + switch (dma_err) { + case DMA_TRANS_READ_FAILED: + case DMA_TRANS_WRITE_FAILED: + entry->errors++; + case DMA_TRANS_ABORTED: + { + struct ntb_transport_qp *qp = entry->qp; + void *offset = qp->rx_buff + qp->rx_max_frame * + qp->rx_index; + + ntb_memcpy_rx(entry, offset); + qp->rx_memcpy++; + return; + } + + case DMA_TRANS_NOERROR: + default: + break; + } + } + entry->flags |= DESC_DONE_FLAG; ntb_complete_rxc(entry->qp); @@ -1272,10 +1301,10 @@ static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) /* Ensure that the data is fully copied out before clearing the flag */ wmb(); - ntb_rx_copy_callback(entry); + ntb_rx_copy_callback(entry, NULL); } -static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) +static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset) { struct dma_async_tx_descriptor *txd; struct ntb_transport_qp *qp = entry->qp; @@ -1288,13 +1317,6 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) int retries = 0; len = entry->len; - - if (!chan) - goto err; - - if (len < copy_bytes) - goto err; - device = chan->device; pay_off = (size_t)offset & ~PAGE_MASK; buff_off = (size_t)buf & ~PAGE_MASK; @@ -1322,7 +1344,8 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) unmap->from_cnt = 1; for (retries = 0; retries < DMA_RETRIES; retries++) { - txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], + txd = device->device_prep_dma_memcpy(chan, + unmap->addr[1], unmap->addr[0], len, DMA_PREP_INTERRUPT); if (txd) @@ -1337,7 +1360,7 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) goto err_get_unmap; } - txd->callback = ntb_rx_copy_callback; + txd->callback_result = ntb_rx_copy_callback; txd->callback_param = entry; dma_set_unmap(txd, unmap); @@ -1351,12 +1374,37 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) qp->rx_async++; - return; + return 0; err_set_unmap: dmaengine_unmap_put(unmap); err_get_unmap: dmaengine_unmap_put(unmap); +err: + return -ENXIO; +} + +static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) +{ + struct ntb_transport_qp *qp = entry->qp; + struct dma_chan *chan = qp->rx_dma_chan; + int res; + + if (!chan) + goto err; + + if (entry->len < copy_bytes) + goto err; + + res = ntb_async_rx_submit(entry, offset); + if (res < 0) + goto err; + + if (!entry->retries) + qp->rx_async++; + + return; + err: ntb_memcpy_rx(entry, offset); qp->rx_memcpy++; @@ -1403,7 +1451,7 @@ static int ntb_process_rxc(struct ntb_transport_qp *qp) } entry->rx_hdr = hdr; - entry->index = qp->rx_index; + entry->rx_index = qp->rx_index; if (hdr->len > entry->len) { dev_dbg(&qp->ndev->pdev->dev, @@ -1981,6 +2029,9 @@ int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, entry->buf = data; entry->len = len; entry->flags = 0; + entry->retries = 0; + entry->errors = 0; + entry->rx_index = 0; ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q); From 9686218982709122373b755c9d7457bed0872999 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 13:14:19 -0700 Subject: [PATCH 39/44] dmaengine: documentation to the new callback mechanism Adding documentation in dmaengine/provider.txt on the usage of callback with result, callback_result, function pointer to retrieve transanction result from a DMA submission. Signed-off-by: Dave Jiang Reviewed-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- Documentation/dmaengine/provider.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Documentation/dmaengine/provider.txt b/Documentation/dmaengine/provider.txt index 91ce82d5f0c4..c4fd47540b31 100644 --- a/Documentation/dmaengine/provider.txt +++ b/Documentation/dmaengine/provider.txt @@ -282,6 +282,17 @@ supported. that is supposed to push the current transaction descriptor to a pending queue, waiting for issue_pending to be called. + - In this structure the function pointer callback_result can be + initialized in order for the submitter to be notified that a + transaction has completed. In the earlier code the function pointer + callback has been used. However it does not provide any status to the + transaction and will be deprecated. The result structure defined as + dmaengine_result that is passed in to callback_result has two fields: + + result: This provides the transfer result defined by + dmaengine_tx_result. Either success or some error + condition. + + residue: Provides the residue bytes of the transfer for those that + support residue. * device_issue_pending - Takes the first transaction descriptor in the pending queue, From 369dbadac151232960fceb83cb14473e8a4e1a36 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Thu, 4 Aug 2016 18:06:13 +0530 Subject: [PATCH 40/44] dmengine: xilinx_dma: convert callback to helper function Move the xilinx driver to new dmaengine callback Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_dma.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 4e223d094433..8288fe4d17c3 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -755,8 +755,7 @@ static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan) spin_lock_irqsave(&chan->lock, flags); list_for_each_entry_safe(desc, next, &chan->done_list, node) { - dma_async_tx_callback callback; - void *callback_param; + struct dmaengine_desc_callback cb; if (desc->cyclic) { xilinx_dma_chan_handle_cyclic(chan, desc, &flags); @@ -767,11 +766,10 @@ static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan) list_del(&desc->node); /* Run the link descriptor callback function */ - callback = desc->async_tx.callback; - callback_param = desc->async_tx.callback_param; - if (callback) { + dmaengine_desc_get_callback(&desc->async_tx, &cb); + if (dmaengine_desc_callback_valid(&cb)) { spin_unlock_irqrestore(&chan->lock, flags); - callback(callback_param); + dmaengine_desc_callback_invoke(&cb, NULL); spin_lock_irqsave(&chan->lock, flags); } From 73fc45e3ce7838e6f47228dd51144c492931e8ad Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Jul 2016 14:13:09 -0700 Subject: [PATCH 41/44] dmaengine: sh_shdma-base: convert callback to helper function This is in preperation of moving to a callback that provides results to the callback for the transaction. The conversion will maintain current behavior and the driver must convert to new callback mechanism at a later time in order to receive results. Signed-off-by: Dave Jiang Signed-off-by: Vinod Koul --- drivers/dma/sh/shdma-base.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c index 10fcabad80f3..12fa48e380cf 100644 --- a/drivers/dma/sh/shdma-base.c +++ b/drivers/dma/sh/shdma-base.c @@ -330,10 +330,11 @@ static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all) bool head_acked = false; dma_cookie_t cookie = 0; dma_async_tx_callback callback = NULL; - void *param = NULL; + struct dmaengine_desc_callback cb; unsigned long flags; LIST_HEAD(cyclic_list); + memset(&cb, 0, sizeof(cb)); spin_lock_irqsave(&schan->chan_lock, flags); list_for_each_entry_safe(desc, _desc, &schan->ld_queue, node) { struct dma_async_tx_descriptor *tx = &desc->async_tx; @@ -367,8 +368,8 @@ static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all) /* Call callback on the last chunk */ if (desc->mark == DESC_COMPLETED && tx->callback) { desc->mark = DESC_WAITING; + dmaengine_desc_get_callback(tx, &cb); callback = tx->callback; - param = tx->callback_param; dev_dbg(schan->dev, "descriptor #%d@%p on %d callback\n", tx->cookie, tx, schan->id); BUG_ON(desc->chunks != 1); @@ -430,8 +431,7 @@ static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all) spin_unlock_irqrestore(&schan->chan_lock, flags); - if (callback) - callback(param); + dmaengine_desc_callback_invoke(&cb, NULL); return callback; } @@ -885,9 +885,9 @@ bool shdma_reset(struct shdma_dev *sdev) /* Complete all */ list_for_each_entry(sdesc, &dl, node) { struct dma_async_tx_descriptor *tx = &sdesc->async_tx; + sdesc->mark = DESC_IDLE; - if (tx->callback) - tx->callback(tx->callback_param); + dmaengine_desc_get_callback_invoke(tx, NULL); } spin_lock(&schan->chan_lock); From 8a31f8b5db65b860fd0d358dc27f6daf26074406 Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Wed, 31 Aug 2016 11:10:27 -0400 Subject: [PATCH 42/44] dmaengine: qcom_hidma: release the descriptor before the callback There is a race condition between data transfer callback and descriptor free code. The callback routine may decide to clear the resources even though the descriptor has not yet been freed. Instead of calling the callback first and then releasing the memory, this code is changing the order to return the descriptor back to the free pool and then call the user provided callback. Signed-off-by: Sinan Kaya Signed-off-by: Vinod Koul --- drivers/dma/qcom/hidma.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index 1197fbf8f30e..b8493bafdb3f 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -111,6 +111,7 @@ static void hidma_process_completed(struct hidma_chan *mchan) struct dma_async_tx_descriptor *desc; dma_cookie_t last_cookie; struct hidma_desc *mdesc; + struct hidma_desc *next; unsigned long irqflags; struct list_head list; @@ -122,8 +123,9 @@ static void hidma_process_completed(struct hidma_chan *mchan) spin_unlock_irqrestore(&mchan->lock, irqflags); /* Execute callbacks and run dependencies */ - list_for_each_entry(mdesc, &list, node) { + list_for_each_entry_safe(mdesc, next, &list, node) { enum dma_status llstat; + struct dmaengine_desc_callback cb; desc = &mdesc->desc; @@ -132,18 +134,18 @@ static void hidma_process_completed(struct hidma_chan *mchan) spin_unlock_irqrestore(&mchan->lock, irqflags); llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); - if (llstat == DMA_COMPLETE) - dmaengine_desc_get_callback_invoke(desc, NULL); + dmaengine_desc_get_callback(desc, &cb); last_cookie = desc->cookie; dma_run_dependencies(desc); + + spin_lock_irqsave(&mchan->lock, irqflags); + list_move(&mdesc->node, &mchan->free); + spin_unlock_irqrestore(&mchan->lock, irqflags); + + if (llstat == DMA_COMPLETE) + dmaengine_desc_callback_invoke(&cb, NULL); } - - /* Free descriptors */ - spin_lock_irqsave(&mchan->lock, irqflags); - list_splice_tail_init(&list, &mchan->free); - spin_unlock_irqrestore(&mchan->lock, irqflags); - } /* From 55c370e5198e8cf28b1529299e9c1bfe237c9c1e Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Wed, 31 Aug 2016 11:10:28 -0400 Subject: [PATCH 43/44] dmaengine: qcom_hidma: report transfer errors with new interface Pass the DMA errors to the client by passing a result argument. The HW only supports a generic error when something goes wrong. That's why, using DMA_TRANS_ABORTED all the time. Signed-off-by: Sinan Kaya Signed-off-by: Vinod Koul --- drivers/dma/qcom/hidma.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index b8493bafdb3f..ea24863794b9 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -126,6 +126,7 @@ static void hidma_process_completed(struct hidma_chan *mchan) list_for_each_entry_safe(mdesc, next, &list, node) { enum dma_status llstat; struct dmaengine_desc_callback cb; + struct dmaengine_result result; desc = &mdesc->desc; @@ -141,10 +142,15 @@ static void hidma_process_completed(struct hidma_chan *mchan) spin_lock_irqsave(&mchan->lock, irqflags); list_move(&mdesc->node, &mchan->free); - spin_unlock_irqrestore(&mchan->lock, irqflags); if (llstat == DMA_COMPLETE) - dmaengine_desc_callback_invoke(&cb, NULL); + result.result = DMA_TRANS_NOERROR; + else + result.result = DMA_TRANS_ABORTED; + + spin_unlock_irqrestore(&mchan->lock, irqflags); + + dmaengine_desc_callback_invoke(&cb, &result); } } From 793ae66c7dcc7e6655029f6613221a111b15b58e Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Wed, 31 Aug 2016 11:10:29 -0400 Subject: [PATCH 44/44] dmaengine: qcom_hidma: add error reporting for tx_status The HIDMA driver is capable of error detection. However, the error was not being passed back to the client when tx_status API is called. Changing the error handling behavior to follow this oder. 1. dmaengine asserts error interrupt 2. Driver receives and mark's the txn as error 3. Driver completes the txn and intimates the client. No further submissions. Drop the locks before calling callback, as subsequent processing by client maybe in callback thread. 4. Client invokes status and you can return error 5. On error, client calls terminate_all. You can reset channel, free all descriptors in the active, pending and completed lists 6. Client prepares new txn and so on. As part of this work, got rid of the reset in the interrupt handler when an error happens and the HW is put into disabled state. The only way to recover is for the client to terminate the channel. Signed-off-by: Sinan Kaya Signed-off-by: Vinod Koul --- drivers/dma/qcom/hidma.c | 30 +++++++++++++++++++++++++----- drivers/dma/qcom/hidma.h | 2 +- drivers/dma/qcom/hidma_ll.c | 32 +++++++------------------------- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index ea24863794b9..e244e10a94b5 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -129,6 +129,7 @@ static void hidma_process_completed(struct hidma_chan *mchan) struct dmaengine_result result; desc = &mdesc->desc; + last_cookie = desc->cookie; spin_lock_irqsave(&mchan->lock, irqflags); dma_cookie_complete(desc); @@ -137,15 +138,15 @@ static void hidma_process_completed(struct hidma_chan *mchan) llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); dmaengine_desc_get_callback(desc, &cb); - last_cookie = desc->cookie; dma_run_dependencies(desc); spin_lock_irqsave(&mchan->lock, irqflags); list_move(&mdesc->node, &mchan->free); - if (llstat == DMA_COMPLETE) + if (llstat == DMA_COMPLETE) { + mchan->last_success = last_cookie; result.result = DMA_TRANS_NOERROR; - else + } else result.result = DMA_TRANS_ABORTED; spin_unlock_irqrestore(&mchan->lock, irqflags); @@ -246,6 +247,19 @@ static void hidma_issue_pending(struct dma_chan *dmach) hidma_ll_start(dmadev->lldev); } +static inline bool hidma_txn_is_success(dma_cookie_t cookie, + dma_cookie_t last_success, dma_cookie_t last_used) +{ + if (last_success <= last_used) { + if ((cookie <= last_success) || (cookie > last_used)) + return true; + } else { + if ((cookie <= last_success) && (cookie > last_used)) + return true; + } + return false; +} + static enum dma_status hidma_tx_status(struct dma_chan *dmach, dma_cookie_t cookie, struct dma_tx_state *txstate) @@ -254,8 +268,13 @@ static enum dma_status hidma_tx_status(struct dma_chan *dmach, enum dma_status ret; ret = dma_cookie_status(dmach, cookie, txstate); - if (ret == DMA_COMPLETE) - return ret; + if (ret == DMA_COMPLETE) { + bool is_success; + + is_success = hidma_txn_is_success(cookie, mchan->last_success, + dmach->cookie); + return is_success ? ret : DMA_ERROR; + } if (mchan->paused && (ret == DMA_IN_PROGRESS)) { unsigned long flags; @@ -406,6 +425,7 @@ static int hidma_terminate_channel(struct dma_chan *chan) hidma_process_completed(mchan); spin_lock_irqsave(&mchan->lock, irqflags); + mchan->last_success = 0; list_splice_init(&mchan->active, &list); list_splice_init(&mchan->prepared, &list); list_splice_init(&mchan->completed, &list); diff --git a/drivers/dma/qcom/hidma.h b/drivers/dma/qcom/hidma.h index db413a5efc4e..e52e20716303 100644 --- a/drivers/dma/qcom/hidma.h +++ b/drivers/dma/qcom/hidma.h @@ -72,7 +72,6 @@ struct hidma_lldev { u32 tre_write_offset; /* TRE write location */ struct tasklet_struct task; /* task delivering notifications */ - struct tasklet_struct rst_task; /* task to reset HW */ DECLARE_KFIFO_PTR(handoff_fifo, struct hidma_tre *); /* pending TREs FIFO */ }; @@ -89,6 +88,7 @@ struct hidma_chan { bool allocated; char dbg_name[16]; u32 dma_sig; + dma_cookie_t last_success; /* * active descriptor on this channel diff --git a/drivers/dma/qcom/hidma_ll.c b/drivers/dma/qcom/hidma_ll.c index ad20dfb64c71..3224f24c577b 100644 --- a/drivers/dma/qcom/hidma_ll.c +++ b/drivers/dma/qcom/hidma_ll.c @@ -380,27 +380,6 @@ static int hidma_ll_reset(struct hidma_lldev *lldev) return 0; } -/* - * Abort all transactions and perform a reset. - */ -static void hidma_ll_abort(unsigned long arg) -{ - struct hidma_lldev *lldev = (struct hidma_lldev *)arg; - u8 err_code = HIDMA_EVRE_STATUS_ERROR; - u8 err_info = 0xFF; - int rc; - - hidma_cleanup_pending_tre(lldev, err_info, err_code); - - /* reset the channel for recovery */ - rc = hidma_ll_setup(lldev); - if (rc) { - dev_err(lldev->dev, "channel reinitialize failed after error\n"); - return; - } - writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); -} - /* * The interrupt handler for HIDMA will try to consume as many pending * EVRE from the event queue as possible. Each EVRE has an associated @@ -454,13 +433,18 @@ irqreturn_t hidma_ll_inthandler(int chirq, void *arg) while (cause) { if (cause & HIDMA_ERR_INT_MASK) { - dev_err(lldev->dev, "error 0x%x, resetting...\n", + dev_err(lldev->dev, "error 0x%x, disabling...\n", cause); /* Clear out pending interrupts */ writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); - tasklet_schedule(&lldev->rst_task); + /* No further submissions. */ + hidma_ll_disable(lldev); + + /* Driver completes the txn and intimates the client.*/ + hidma_cleanup_pending_tre(lldev, 0xFF, + HIDMA_EVRE_STATUS_ERROR); goto out; } @@ -808,7 +792,6 @@ struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres, return NULL; spin_lock_init(&lldev->lock); - tasklet_init(&lldev->rst_task, hidma_ll_abort, (unsigned long)lldev); tasklet_init(&lldev->task, hidma_ll_tre_complete, (unsigned long)lldev); lldev->initialized = 1; writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); @@ -831,7 +814,6 @@ int hidma_ll_uninit(struct hidma_lldev *lldev) required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres; tasklet_kill(&lldev->task); - tasklet_kill(&lldev->rst_task); memset(lldev->trepool, 0, required_bytes); lldev->trepool = NULL; lldev->pending_tre_count = 0;