From 016b719166785746cbec0b7435f558ced9eba1a8 Mon Sep 17 00:00:00 2001 From: Han Xu Date: Wed, 7 Aug 2019 16:20:02 -0500 Subject: [PATCH 01/52] MLK-19897: dma: mxs-dma: filter out the unrelated dma channels update mxs-dma filter function to firstly filter the dma channels only for mxs-dma, rather than checking unrelated dma chans in following code. Signed-off-by: Han Xu --- drivers/dma/mxs-dma.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index 3039bba0e4d5..6982046fd30f 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -729,6 +729,12 @@ static bool mxs_dma_filter_fn(struct dma_chan *chan, void *fn_param) struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; int chan_irq; + if (strcmp(chan->device->dev->driver->name, "mxs-dma")) + return false; + + if (!mxs_dma) + return false; + if (chan->chan_id != param->chan_id) return false; From f84f5223ac4850358b635fc6cf2dbc2581d4590c Mon Sep 17 00:00:00 2001 From: Han Xu Date: Wed, 13 Nov 2019 22:03:14 -0600 Subject: [PATCH 02/52] dma: mxs-dma: change the way to register the probe function change the old way to register the probe function in driver Signed-off-by: Han Xu --- drivers/dma/mxs-dma.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index 6982046fd30f..3c8cc5077269 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -766,7 +766,7 @@ static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec, ofdma->of_node); } -static int __init mxs_dma_probe(struct platform_device *pdev) +static int mxs_dma_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; const struct platform_device_id *id_entry; @@ -875,10 +875,6 @@ static struct platform_driver mxs_dma_driver = { .of_match_table = mxs_dma_dt_ids, }, .id_table = mxs_dma_ids, + .probe = mxs_dma_probe, }; - -static int __init mxs_dma_module_init(void) -{ - return platform_driver_probe(&mxs_dma_driver, mxs_dma_probe); -} -subsys_initcall(mxs_dma_module_init); +module_platform_driver(mxs_dma_driver); From 2cb2cdf8ed086b9ba8dcae4df1e10792f26fe84e Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 9 Apr 2019 09:45:39 +0800 Subject: [PATCH 03/52] Revert "dmaengine: imx-sdma: refine to load context only once" This reverts commit ad0d92d7ba6aecbe2705907c38ff8d8be4da1e9c, because in spi-imx case, burst length may be changed dynamically. Signed-off-by: Robin Gong --- drivers/dma/imx-sdma.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index c27e206a764c..99e036c868d7 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -377,7 +377,6 @@ struct sdma_channel { unsigned long watermark_level; u32 shp_addr, per_addr; enum dma_status status; - bool context_loaded; struct imx_dma_data data; struct work_struct terminate_worker; }; @@ -988,9 +987,6 @@ static int sdma_load_context(struct sdma_channel *sdmac) int ret; unsigned long flags; - if (sdmac->context_loaded) - return 0; - if (sdmac->direction == DMA_DEV_TO_MEM) load_address = sdmac->pc_from_device; else if (sdmac->direction == DMA_DEV_TO_DEV) @@ -1033,8 +1029,6 @@ static int sdma_load_context(struct sdma_channel *sdmac) spin_unlock_irqrestore(&sdma->channel_0_lock, flags); - sdmac->context_loaded = true; - return ret; } @@ -1074,7 +1068,6 @@ static void sdma_channel_terminate_work(struct work_struct *work) sdmac->desc = NULL; spin_unlock_irqrestore(&sdmac->vc.lock, flags); vchan_dma_desc_free_list(&sdmac->vc, &head); - sdmac->context_loaded = false; } static int sdma_disable_channel_async(struct dma_chan *chan) From 3ce7fb0ce72fed21dfa250342fcb96ea3b176be1 Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Tue, 23 Jan 2018 13:27:39 +0800 Subject: [PATCH 04/52] MLK-16224-1: dmaengine: add src_fifo_num and dst_fifo_num in dma_slave_config In order to support multi-fifo sdma script, the audio driver need to send the fifo number to dma driver through dma_slave_config, so add src_fifo_num and dst_fifo_num two new variable for struct dma_slave_config. src_fifo_num: bit 0-7 is the fifo number, bit:8-11 is the fifo offset; dst_fifo_num: same as src_fifo_num Signed-off-by: Shengjiu Wang Reviewed-by: Robin Gong --- include/linux/dmaengine.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 8fcdee1c0cf9..f44382cf4f41 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -333,6 +333,8 @@ enum dma_slave_buswidth { * loops in this area in order to transfer the data. * @dst_port_window_size: same as src_port_window_size but for the destination * port. + * @src_fifo_num: bit 0-7 is the fifo number, bit:8-11 is the fifo offset; + * @dst_fifo_num: same as src_fifo_num * @device_fc: Flow Controller Settings. Only valid for slave channels. Fill * with 'true' if peripheral should be flow controller. Direction will be * selected at Runtime. @@ -362,6 +364,8 @@ struct dma_slave_config { u32 dst_maxburst; u32 src_port_window_size; u32 dst_port_window_size; + u32 src_fifo_num; + u32 dst_fifo_num; bool device_fc; unsigned int slave_id; }; From 9b50f7037dd5a971d3e3343fb7a24442ab673f82 Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Tue, 29 Jan 2019 13:08:07 +0800 Subject: [PATCH 05/52] include: dma-imx: support dual fifo for for DEV_TO_DEV Add src_dualfifo and dst_dualfifo in imx_dma_data to support dual fifo for DMA_DEV_TO_DEV Signed-off-by: Shengjiu Wang --- include/linux/platform_data/dma-imx.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/platform_data/dma-imx.h b/include/linux/platform_data/dma-imx.h index 281adbb26e6b..694183a2bc5f 100644 --- a/include/linux/platform_data/dma-imx.h +++ b/include/linux/platform_data/dma-imx.h @@ -52,6 +52,8 @@ struct imx_dma_data { int dma_request2; /* secondary DMA request line */ enum sdma_peripheral_type peripheral_type; int priority; + bool src_dualfifo; + bool dst_dualfifo; }; static inline int imx_dma_is_ipu(struct dma_chan *chan) From cdb5dd1f2b74d1b30c1bc3f7819b2d7955425bd6 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 16 Jan 2018 17:52:49 +0800 Subject: [PATCH 06/52] MLK-17385: dma: imx-sdma: update sdma script for multi fifo on SAI update sdma script for multi fifo SAI on i.mx8MQ. Signed-off-by: Robin Gong [ Aisheng: fix build error ] Signed-off-by: Dong Aisheng --- drivers/dma/imx-sdma.c | 7 ++++--- include/linux/platform_data/dma-imx-sdma.h | 2 ++ include/linux/platform_data/dma-imx.h | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 99e036c868d7..d76c6f572d91 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -8,7 +8,8 @@ // // Based on code from Freescale: // -// Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved. +// Copyright 2004-2016 Freescale Semiconductor, Inc. All Rights Reserved. +// Copyright 2018 NXP. #include #include @@ -1686,8 +1687,8 @@ static void sdma_issue_pending(struct dma_chan *chan) #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 41 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 42 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 43 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 44 static void sdma_add_scripts(struct sdma_engine *sdma, const struct sdma_script_start_addrs *addr) diff --git a/include/linux/platform_data/dma-imx-sdma.h b/include/linux/platform_data/dma-imx-sdma.h index 30e676b36b24..da06645bea80 100644 --- a/include/linux/platform_data/dma-imx-sdma.h +++ b/include/linux/platform_data/dma-imx-sdma.h @@ -52,6 +52,8 @@ struct sdma_script_start_addrs { s32 zcanfd_2_mcu_addr; s32 zqspi_2_mcu_addr; s32 mcu_2_ecspi_addr; + s32 mcu_2_sai_addr; + s32 sai_2_mcu_addr; /* End of v3 array */ s32 mcu_2_zqspi_addr; /* End of v4 array */ diff --git a/include/linux/platform_data/dma-imx.h b/include/linux/platform_data/dma-imx.h index 694183a2bc5f..c94a05dd654c 100644 --- a/include/linux/platform_data/dma-imx.h +++ b/include/linux/platform_data/dma-imx.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved. + * Copyright 2004-2015 Freescale Semiconductor, Inc. All Rights Reserved. + * Copyright 2018 NXP. */ #ifndef __ASM_ARCH_MXC_DMA_H__ @@ -39,6 +40,7 @@ enum sdma_peripheral_type { IMX_DMATYPE_SSI_DUAL, /* SSI Dual FIFO */ IMX_DMATYPE_ASRC_SP, /* Shared ASRC */ IMX_DMATYPE_SAI, /* SAI */ + IMX_DMATYPE_MULTI_SAI, /* MULTI FIFOs For Audio */ }; enum imx_dma_prio { From ea884be2721f74f2f730a7fdbf47f34f04b09228 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 23 Apr 2019 16:29:28 +0800 Subject: [PATCH 07/52] dmaengine: imx-sdma: remove dupilicated sdma_load_context Since sdma_transfer_init() will do sdma_load_context before any sdma transfer, no need once more in sdma_config_channel(). Signed-off-by: Robin Gong --- drivers/dma/imx-sdma.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index d76c6f572d91..99b96c9e21cd 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1135,7 +1135,6 @@ static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) static int sdma_config_channel(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); - int ret; sdma_disable_channel(chan); @@ -1175,9 +1174,7 @@ static int sdma_config_channel(struct dma_chan *chan) sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */ } - ret = sdma_load_context(sdmac); - - return ret; + return 0; } static int sdma_set_channel_priority(struct sdma_channel *sdmac, From 7542e1d08a80922bcc9f1242ceba6111c656acc7 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 24 Apr 2019 17:05:36 +0800 Subject: [PATCH 08/52] dma: engine: imx-sdma: add mcu_2_ecspi script Add mcu_2_ecspi script to fix ecspi errata ERR009165. Signed-off-by: Robin Gong --- drivers/dma/imx-sdma.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 99b96c9e21cd..5bfec34f69a6 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -925,6 +925,9 @@ static void sdma_get_pc(struct sdma_channel *sdmac, emi_2_per = sdma->script_addrs->mcu_2_ata_addr; break; case IMX_DMATYPE_CSPI: + per_2_emi = sdma->script_addrs->app_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_ecspi_addr; + break; case IMX_DMATYPE_EXT: case IMX_DMATYPE_SSI: case IMX_DMATYPE_SAI: From d89ee194f6965dca5c1790b7c29b00035fee468e Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 23 Apr 2019 21:21:46 +0800 Subject: [PATCH 09/52] dmaengine: imx-sdma: remove ERR009165 on i.mx6ul ECSPI issue fixed from i.mx6ul at hardware level, no need ERR009165 anymore on those chips such as i.mx8mq. Add i.mx6sx from where i.mx6ul source. Signed-off-by: Robin Gong --- drivers/dma/imx-sdma.c | 52 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 5bfec34f69a6..4d21a4339c3b 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -420,6 +420,13 @@ struct sdma_driver_data { int num_events; struct sdma_script_start_addrs *script_addrs; bool check_ratio; + /* + * ecspi ERR009165 fixed should be done in sdma script + * and it be fixed in soc from i.mx6ul. + * please get more information from below link: + * https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf + */ + bool ecspi_fixed; }; struct sdma_engine { @@ -540,6 +547,31 @@ static struct sdma_driver_data sdma_imx6q = { .script_addrs = &sdma_script_imx6q, }; +static struct sdma_script_start_addrs sdma_script_imx6sx = { + .ap_2_ap_addr = 642, + .uart_2_mcu_addr = 817, + .mcu_2_app_addr = 747, + .uartsh_2_mcu_addr = 1032, + .mcu_2_shp_addr = 960, + .app_2_mcu_addr = 683, + .shp_2_mcu_addr = 891, + .spdif_2_mcu_addr = 1100, + .mcu_2_spdif_addr = 1134, +}; + +static struct sdma_driver_data sdma_imx6sx = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx6sx, +}; + +static struct sdma_driver_data sdma_imx6ul = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx6sx, + .ecspi_fixed = true, +}; + static struct sdma_script_start_addrs sdma_script_imx7d = { .ap_2_ap_addr = 644, .uart_2_mcu_addr = 819, @@ -563,6 +595,7 @@ static struct sdma_driver_data sdma_imx8mq = { .num_events = 48, .script_addrs = &sdma_script_imx7d, .check_ratio = 1, + .ecspi_fixed = true, }; static const struct platform_device_id sdma_devtypes[] = { @@ -584,9 +617,15 @@ static const struct platform_device_id sdma_devtypes[] = { }, { .name = "imx6q-sdma", .driver_data = (unsigned long)&sdma_imx6q, + }, { + .name = "imx6sx-sdma", + .driver_data = (unsigned long)&sdma_imx6sx, }, { .name = "imx7d-sdma", .driver_data = (unsigned long)&sdma_imx7d, + }, { + .name = "imx6ul-sdma", + .driver_data = (unsigned long)&sdma_imx6ul, }, { .name = "imx8mq-sdma", .driver_data = (unsigned long)&sdma_imx8mq, @@ -603,7 +642,9 @@ static const struct of_device_id sdma_dt_ids[] = { { .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, }, { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, }, { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, }, + { .compatible = "fsl,imx6sx-sdma", .data = &sdma_imx6sx, }, { .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, }, + { .compatible = "fsl,imx6ul-sdma", .data = &sdma_imx6ul, }, { .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, }, { /* sentinel */ } }; @@ -1167,8 +1208,17 @@ static int sdma_config_channel(struct dma_chan *chan) if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP || sdmac->peripheral_type == IMX_DMATYPE_ASRC) sdma_set_watermarklevel_for_p2p(sdmac); - } else + } else { + /* + * ERR009165 fixed from i.mx6ul, no errata need, + * set bit31 to let sdma script skip the errata. + */ + if (sdmac->peripheral_type == IMX_DMATYPE_CSPI && + sdmac->direction == DMA_MEM_TO_DEV && + sdmac->sdma->drvdata->ecspi_fixed) + __set_bit(31, &sdmac->watermark_level); __set_bit(sdmac->event_id0, sdmac->event_mask); + } /* Address */ sdmac->shp_addr = sdmac->per_address; From 76772e7931af8e1231c8bab3740d3ba1cf599b9e Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 23 Apr 2019 21:24:57 +0800 Subject: [PATCH 10/52] dt-bindings: dma: imx-sdma: add i.mx6ul/6sx compatible name Add i.mx6ul and i.mx6sx compatible name. Signed-off-by: Robin Gong --- Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt b/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt index 9d8bbac27d8b..d024a83e83ad 100644 --- a/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt +++ b/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt @@ -9,6 +9,8 @@ Required properties: "fsl,imx53-sdma" "fsl,imx6q-sdma" "fsl,imx7d-sdma" + "fsl,imx6sx-sdma" + "fsl,imx6ul-sdma" "fsl,imx8mq-sdma" The -to variants should be preferred since they allow to determine the correct ROM script addresses needed for the driver to work without additional From e4770db1f1608c6102f95cd987d10ac9a3ce47dd Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 23 Apr 2019 15:13:08 +0800 Subject: [PATCH 11/52] dmaengine: imx-sdma: fix ecspi1 rx dma not work on i.mx8mm Because the number of ecspi1 rx event on i.mx8mm is 0, the condition check ignore such special case without dma channel enabled, which caused ecspi1 rx works failed. Actually, no need to check event_id0, checking event_id1 is enough for DEV_2_DEV case because it's so lucky that event_id1 never be 0. Signed-off-by: Robin Gong --- drivers/dma/imx-sdma.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 4d21a4339c3b..910041310fae 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1372,8 +1372,8 @@ static void sdma_free_chan_resources(struct dma_chan *chan) sdma_channel_synchronize(chan); - if (sdmac->event_id0) - sdma_event_disable(sdmac, sdmac->event_id0); + sdma_event_disable(sdmac, sdmac->event_id0); + if (sdmac->event_id1) sdma_event_disable(sdmac, sdmac->event_id1); @@ -1672,11 +1672,9 @@ static int sdma_config(struct dma_chan *chan, memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg)); /* Set ENBLn earlier to make sure dma request triggered after that */ - if (sdmac->event_id0) { - if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) - return -EINVAL; - sdma_event_enable(sdmac, sdmac->event_id0); - } + if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) + return -EINVAL; + sdma_event_enable(sdmac, sdmac->event_id0); if (sdmac->event_id1) { if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) From 2d1e5b8f23c20ccdd2336685467f72e618715658 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 24 Apr 2019 14:19:58 +0800 Subject: [PATCH 12/52] MLK-21309: dma: engine: sdma: add uart rom script For syncing with unstreaming kernel on UART driver from 4.19 changed to rom script for uart rx path, and the compatiblity of legacy kernel using ram script, add both uart rom and ram script support, so add rom script address. ram script: uart_2_mcu_fix_addr uartsh_2_mcu_fix_addr /* through spba bus */ rom script: uart_2_mcu_addr uartsh_2_mcu_addr /* through spba bus */ Signed-off-by: Robin Gong --- include/linux/platform_data/dma-imx-sdma.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/linux/platform_data/dma-imx-sdma.h b/include/linux/platform_data/dma-imx-sdma.h index da06645bea80..e12d2e8c246b 100644 --- a/include/linux/platform_data/dma-imx-sdma.h +++ b/include/linux/platform_data/dma-imx-sdma.h @@ -20,12 +20,12 @@ struct sdma_script_start_addrs { s32 per_2_firi_addr; s32 mcu_2_firi_addr; s32 uart_2_per_addr; - s32 uart_2_mcu_addr; + s32 uart_2_mcu_ram_addr; s32 per_2_app_addr; s32 mcu_2_app_addr; s32 per_2_per_addr; s32 uartsh_2_per_addr; - s32 uartsh_2_mcu_addr; + s32 uartsh_2_mcu_ram_addr; s32 per_2_shp_addr; s32 mcu_2_shp_addr; s32 ata_2_mcu_addr; @@ -54,6 +54,8 @@ struct sdma_script_start_addrs { s32 mcu_2_ecspi_addr; s32 mcu_2_sai_addr; s32 sai_2_mcu_addr; + s32 uart_2_mcu_addr; + s32 uartsh_2_mcu_addr; /* End of v3 array */ s32 mcu_2_zqspi_addr; /* End of v4 array */ From ff3799c817e1eef261d9261976f2175ddfc11336 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 5 Nov 2013 19:19:07 +0800 Subject: [PATCH 13/52] dmaengine: imx-sdma: support allocate memory from iram Allocate memory from SoC internal SRAM so that we can turn off voltage of external DDR to save power if 'iram' property in dts. Signed-off-by: Robin Gong --- drivers/dma/imx-sdma.c | 60 +++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 910041310fae..cd3b28ac9ab0 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -451,6 +452,7 @@ struct sdma_engine { struct sdma_buffer_descriptor *bd0; /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/ bool clk_ratio; + struct gen_pool *iram_pool; }; static int sdma_config_write(struct dma_chan *chan, @@ -736,10 +738,13 @@ static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, int ret; unsigned long flags; - buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL); - if (!buf_virt) { + if (sdma->iram_pool) + buf_virt = gen_pool_dma_alloc(sdma->iram_pool, size, &buf_phys); + else + buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, + GFP_KERNEL); + if (!buf_virt) return -ENOMEM; - } spin_lock_irqsave(&sdma->channel_0_lock, flags); @@ -755,7 +760,10 @@ static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, spin_unlock_irqrestore(&sdma->channel_0_lock, flags); - dma_free_coherent(sdma->dev, size, buf_virt, buf_phys); + if (sdma->iram_pool) + gen_pool_free(sdma->iram_pool, (unsigned long)buf_virt, size); + else + dma_free_coherent(sdma->dev, size, buf_virt, buf_phys); return ret; } @@ -1250,8 +1258,12 @@ static int sdma_request_channel0(struct sdma_engine *sdma) { int ret = -EBUSY; - sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys, - GFP_NOWAIT); + if (sdma->iram_pool) + sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool, PAGE_SIZE, + &sdma->bd0_phys); + else + sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, + &sdma->bd0_phys, GFP_NOWAIT); if (!sdma->bd0) { ret = -ENOMEM; goto out; @@ -1271,10 +1283,15 @@ out: static int sdma_alloc_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; int ret = 0; - desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size, - &desc->bd_phys, GFP_NOWAIT); + if (sdma->iram_pool) + desc->bd = gen_pool_dma_alloc(sdma->iram_pool, PAGE_SIZE, + &desc->bd_phys); + else + desc->bd = dma_alloc_coherent(sdma->dev, bd_size, + &desc->bd_phys, GFP_NOWAIT); if (!desc->bd) { ret = -ENOMEM; goto out; @@ -1286,9 +1303,14 @@ out: static void sdma_free_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; - dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, - desc->bd_phys); + if (sdma->iram_pool) + gen_pool_free(sdma->iram_pool, (unsigned long)desc->bd, + PAGE_SIZE); + else + dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, + desc->bd_phys); } static void sdma_desc_free(struct virt_dma_desc *vd) @@ -1904,7 +1926,7 @@ static int sdma_get_firmware(struct sdma_engine *sdma, static int sdma_init(struct sdma_engine *sdma) { - int i, ret; + int i, ret, ccbsize; dma_addr_t ccb_phys; ret = clk_enable(sdma->clk_ipg); @@ -1921,11 +1943,15 @@ static int sdma_init(struct sdma_engine *sdma) /* Be sure SDMA has not started yet */ writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); - sdma->channel_control = dma_alloc_coherent(sdma->dev, - MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) + - sizeof(struct sdma_context_data), - &ccb_phys, GFP_KERNEL); + ccbsize = MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control) + + sizeof(struct sdma_context_data); + if (sdma->iram_pool) + sdma->channel_control = gen_pool_dma_alloc(sdma->iram_pool, + ccbsize, &ccb_phys); + else + sdma->channel_control = dma_alloc_coherent(sdma->dev, ccbsize, + &ccb_phys, GFP_KERNEL); if (!sdma->channel_control) { ret = -ENOMEM; goto err_dma_alloc; @@ -2179,6 +2205,10 @@ static int sdma_probe(struct platform_device *pdev) sdma->spba_end_addr = spba_res.end; } of_node_put(spba_bus); + + sdma->iram_pool = of_gen_pool_get(np, "iram", 0); + if (sdma->iram_pool) + dev_info(&pdev->dev, "alloc bd from iram. \n"); } /* From a3e3585408dfb82cd534792722b3f8ba3ba84107 Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Fri, 5 Sep 2014 18:51:36 +0800 Subject: [PATCH 14/52] dmaengine: imx-sdma: Add hdmi audio support in sdma Add hdmi audio support. Signed-off-by: Robin Gong --- .../devicetree/bindings/dma/fsl-imx-sdma.txt | 1 + drivers/dma/imx-sdma.c | 35 +++++++++++++++---- include/linux/platform_data/dma-imx.h | 1 + 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt b/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt index d024a83e83ad..0232a60422a1 100644 --- a/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt +++ b/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt @@ -53,6 +53,7 @@ The full ID of peripheral types can be found below. 22 SSI Dual FIFO (needs firmware ver >= 2) 23 Shared ASRC 24 SAI + 25 HDMI Audio The third cell specifies the transfer priority as below. diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index cd3b28ac9ab0..2afe967c016a 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -915,7 +915,10 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) desc = sdmac->desc; if (desc) { if (sdmac->flags & IMX_DMA_SG_LOOP) { - sdma_update_channel_loop(sdmac); + if (sdmac->peripheral_type != IMX_DMATYPE_HDMI) + sdma_update_channel_loop(sdmac); + else + vchan_cyclic_callback(&desc->vd); } else { mxc_sdma_handle_channel_normal(sdmac); vchan_cookie_complete(&desc->vd); @@ -1020,6 +1023,9 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_IPU_MEMORY: emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr; break; + case IMX_DMATYPE_HDMI: + emi_2_per = sdma->script_addrs->hdmi_dma_addr; + break; default: break; } @@ -1067,11 +1073,16 @@ static int sdma_load_context(struct sdma_channel *sdmac) /* Send by context the event mask,base address for peripheral * and watermark level */ - context->gReg[0] = sdmac->event_mask[1]; - context->gReg[1] = sdmac->event_mask[0]; - context->gReg[2] = sdmac->per_addr; - context->gReg[6] = sdmac->shp_addr; - context->gReg[7] = sdmac->watermark_level; + if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { + context->gReg[4] = sdmac->per_addr; + context->gReg[6] = sdmac->shp_addr; + } else { + context->gReg[0] = sdmac->event_mask[1]; + context->gReg[1] = sdmac->event_mask[0]; + context->gReg[2] = sdmac->per_addr; + context->gReg[6] = sdmac->shp_addr; + context->gReg[7] = sdmac->watermark_level; + } bd0->mode.command = C0_SETDM; bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; @@ -1593,13 +1604,16 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( { struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; - int num_periods = buf_len / period_len; + int num_periods = 0; int channel = sdmac->channel; int i = 0, buf = 0; struct sdma_desc *desc; dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel); + if (sdmac->peripheral_type != IMX_DMATYPE_HDMI) + num_periods = buf_len / period_len; + sdma_config_write(chan, &sdmac->slave_config, direction); desc = sdma_transfer_init(sdmac, direction, num_periods); @@ -1616,6 +1630,9 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( goto err_bd_out; } + if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); + while (buf < buf_len) { struct sdma_buffer_descriptor *bd = &desc->bd[i]; int param; @@ -1676,6 +1693,10 @@ static int sdma_config_write(struct dma_chan *chan, sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) & SDMA_WATERMARK_LEVEL_HWML; sdmac->word_size = dmaengine_cfg->dst_addr_width; + } else if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { + sdmac->per_address = dmaengine_cfg->dst_addr; + sdmac->per_address2 = dmaengine_cfg->src_addr; + sdmac->watermark_level = 0; } else { sdmac->per_address = dmaengine_cfg->dst_addr; sdmac->watermark_level = dmaengine_cfg->dst_maxburst * diff --git a/include/linux/platform_data/dma-imx.h b/include/linux/platform_data/dma-imx.h index c94a05dd654c..d26007b456a5 100644 --- a/include/linux/platform_data/dma-imx.h +++ b/include/linux/platform_data/dma-imx.h @@ -41,6 +41,7 @@ enum sdma_peripheral_type { IMX_DMATYPE_ASRC_SP, /* Shared ASRC */ IMX_DMATYPE_SAI, /* SAI */ IMX_DMATYPE_MULTI_SAI, /* MULTI FIFOs For Audio */ + IMX_DMATYPE_HDMI, /* HDMI Audio */ }; enum imx_dma_prio { From e977370b6d91700e7ac4308cfe57f415ab528261 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Sun, 28 Apr 2019 23:24:17 +0800 Subject: [PATCH 15/52] dmaengine: imx-sdma: add sdma resume back after megafast off On i.mx6sx or i.mx7d chip, megafast could be off in suspend which means sdma controller will be power-ed off, thus sdma driver should resume back including firmware loaded again. Signed-off-by: Robin Gong [ Aisheng: fix rebase conflict ] Signed-off-by: Dong Aisheng --- drivers/dma/imx-sdma.c | 173 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 172 insertions(+), 1 deletion(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 2afe967c016a..3a5536055729 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -381,6 +381,7 @@ struct sdma_channel { enum dma_status status; struct imx_dma_data data; struct work_struct terminate_worker; + bool is_ram_script; }; #define IMX_DMA_SG_LOOP BIT(0) @@ -390,6 +391,15 @@ struct sdma_channel { #define MXC_SDMA_MIN_PRIORITY 1 #define MXC_SDMA_MAX_PRIORITY 7 +/* + * 0x78(SDMA_XTRIG_CONF2+4)~0x100(SDMA_CHNPRI_O) registers are reserved and + * can't be accessed. Skip these register touch in suspend/resume. Also below + * two macros are only used on i.mx6sx. + */ +#define MXC_SDMA_RESERVED_REG (SDMA_CHNPRI_0 - SDMA_XTRIG_CONF2 - 4) +#define MXC_SDMA_SAVED_REG_NUM (((SDMA_CHNENBL0_IMX35 + 4 * 48) - \ + MXC_SDMA_RESERVED_REG) / 4) + #define SDMA_FIRMWARE_MAGIC 0x414d4453 /** @@ -435,6 +445,8 @@ struct sdma_engine { struct device_dma_parameters dma_parms; struct sdma_channel channel[MAX_DMA_CHANNELS]; struct sdma_channel_control *channel_control; + u32 save_regs[MXC_SDMA_SAVED_REG_NUM]; + const char *fw_name; void __iomem *regs; struct sdma_context_data *context; dma_addr_t context_phys; @@ -453,6 +465,8 @@ struct sdma_engine { /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/ bool clk_ratio; struct gen_pool *iram_pool; + bool fw_loaded; + unsigned short ram_code_start; }; static int sdma_config_write(struct dma_chan *chan, @@ -1034,6 +1048,13 @@ static void sdma_get_pc(struct sdma_channel *sdmac, sdmac->pc_to_device = emi_2_per; sdmac->device_to_device = per_2_per; sdmac->pc_to_pc = emi_2_emi; + + if (sdma->ram_code_start && + ((sdmac->pc_from_device >= sdma->ram_code_start) || + (sdmac->pc_to_device >= sdma->ram_code_start) || + (sdmac->device_to_device >= sdma->ram_code_start || + (sdmac->pc_to_pc >= sdma->ram_code_start)))) + sdmac->is_ram_script = true; } static int sdma_load_context(struct sdma_channel *sdmac) @@ -1096,6 +1117,31 @@ static int sdma_load_context(struct sdma_channel *sdmac) return ret; } +static int sdma_save_restore_context(struct sdma_engine *sdma, bool save) +{ + struct sdma_context_data *context = sdma->context; + struct sdma_buffer_descriptor *bd0 = sdma->bd0; + unsigned long flags; + int ret; + + spin_lock_irqsave(&sdma->channel_0_lock, flags); + + if (save) + bd0->mode.command = C0_GETDM; + else + bd0->mode.command = C0_SETDM; + + bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; + bd0->mode.count = MAX_DMA_CHANNELS * sizeof(*context) / 4; + bd0->buffer_addr = sdma->context_phys; + bd0->ext_buffer_addr = 2048; + ret = sdma_run_channel0(sdma); + + spin_unlock_irqrestore(&sdma->channel_0_lock, flags); + + return ret; +} + static struct sdma_channel *to_sdma_chan(struct dma_chan *chan) { return container_of(chan, struct sdma_channel, vc.chan); @@ -1424,6 +1470,11 @@ static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, { struct sdma_desc *desc; + if (!sdmac->sdma->fw_loaded && sdmac->is_ram_script) { + dev_err(sdmac->sdma->dev, "sdma firmware not ready!\n"); + goto err_out; + } + desc = kzalloc((sizeof(*desc)), GFP_NOWAIT); if (!desc) goto err_out; @@ -1818,7 +1869,7 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) return; } - if (fw->size < sizeof(*header)) + if (fw->size < sizeof(*header) || sdma->fw_loaded) goto err_firmware; header = (struct sdma_firmware_header *)fw->data; @@ -1847,6 +1898,7 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) addr = (void *)header + header->script_addrs_start; ram_code = (void *)header + header->ram_code_start; + sdma->ram_code_start = header->ram_code_start; clk_enable(sdma->clk_ipg); clk_enable(sdma->clk_ahb); @@ -1859,6 +1911,8 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) sdma_add_scripts(sdma, addr); + sdma->fw_loaded = true; + dev_info(sdma->dev, "loaded firmware %d.%d\n", header->version_major, header->version_minor); @@ -2259,6 +2313,8 @@ static int sdma_probe(struct platform_device *pdev) } } + sdma->fw_name = fw_name; + return 0; err_register: @@ -2294,6 +2350,121 @@ static int sdma_remove(struct platform_device *pdev) return 0; } +#ifdef CONFIG_PM_SLEEP +static int sdma_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sdma_engine *sdma = platform_get_drvdata(pdev); + int i, ret = 0; + + /* Do nothing if not i.MX6SX or i.MX7D*/ + if (sdma->drvdata != &sdma_imx6sx && sdma->drvdata != &sdma_imx7d + && sdma->drvdata != &sdma_imx6ul) + return 0; + + clk_enable(sdma->clk_ipg); + clk_enable(sdma->clk_ahb); + + ret = sdma_save_restore_context(sdma, true); + if (ret) { + dev_err(sdma->dev, "save context error!\n"); + return ret; + } + /* save regs */ + for (i = 0; i < MXC_SDMA_SAVED_REG_NUM; i++) { + /* + * 0x78(SDMA_XTRIG_CONF2+4)~0x100(SDMA_CHNPRI_O) registers are + * reserved and can't be touched. Skip these regs. + */ + if (i > SDMA_XTRIG_CONF2 / 4) + sdma->save_regs[i] = readl_relaxed(sdma->regs + + MXC_SDMA_RESERVED_REG + + 4 * i); + else + sdma->save_regs[i] = readl_relaxed(sdma->regs + 4 * i); + } + + clk_disable(sdma->clk_ipg); + clk_disable(sdma->clk_ahb); + + return 0; +} + +static int sdma_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sdma_engine *sdma = platform_get_drvdata(pdev); + unsigned long timeout = jiffies + msecs_to_jiffies(2); + int i, ret; + + /* Do nothing if not i.MX6SX or i.MX7D*/ + if (sdma->drvdata != &sdma_imx6sx && sdma->drvdata != &sdma_imx7d + && sdma->drvdata != &sdma_imx6ul) + return 0; + + clk_enable(sdma->clk_ipg); + clk_enable(sdma->clk_ahb); + /* Do nothing if mega/fast mix not turned off */ + if (readl_relaxed(sdma->regs + SDMA_H_C0PTR)) { + clk_disable(sdma->clk_ipg); + clk_disable(sdma->clk_ahb); + return 0; + } + + /* Firmware was lost, mark as "not ready" */ + sdma->fw_loaded = false; + + /* restore regs and load firmware */ + for (i = 0; i < MXC_SDMA_SAVED_REG_NUM; i++) { + /* + * 0x78(SDMA_XTRIG_CONF2+4)~0x100(SDMA_CHNPRI_O) registers are + * reserved and can't be touched. Skip these regs. + */ + if (i > SDMA_XTRIG_CONF2 / 4) + writel_relaxed(sdma->save_regs[i], sdma->regs + + MXC_SDMA_RESERVED_REG + 4 * i); + /* set static context switch mode before channel0 running */ + else if (i == SDMA_H_CONFIG / 4) + writel_relaxed(sdma->save_regs[i] & ~SDMA_H_CONFIG_CSM, + sdma->regs + SDMA_H_CONFIG); + else + writel_relaxed(sdma->save_regs[i], sdma->regs + 4 * i); + } + + /* prepare priority for channel0 to start */ + sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY); + + ret = sdma_get_firmware(sdma, sdma->fw_name); + if (ret) { + dev_warn(&pdev->dev, "failed to get firmware\n"); + goto out; + } + /* wait firmware loaded */ + do { + if (time_after(jiffies, timeout)) { + dev_warn(&pdev->dev, "failed to load firmware\n"); + break; + } + usleep_range(50, 500); + } while (!sdma->fw_loaded); + + ret = sdma_save_restore_context(sdma, false); + if (ret) { + dev_err(sdma->dev, "restore context error!\n"); + goto out; + } +out: + clk_disable(sdma->clk_ipg); + clk_disable(sdma->clk_ahb); + + return ret; +} +#endif + +static const struct dev_pm_ops sdma_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(sdma_suspend, sdma_resume) +}; + static struct platform_driver sdma_driver = { .driver = { .name = "imx-sdma", From a89c75d447599ba8e8d2fdb81993622a6665daee Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Fri, 17 Jul 2015 13:52:37 +0800 Subject: [PATCH 16/52] MLK-11259-1: dmaengine: imx-sdma: support dual fifo for DEV_TO_DEV As SSI has dual fifo, add src_dualfifo and dst_dualfifo in imx_dma_data to support dual fifo in DMA_DEV_TO_DEV. Signed-off-by: Shengjiu Wang (cherry picked from commit cfde1308f170166a0099ca39ee8733895f9626f0) (Vipul: Fixed merge conflicts) Signed-off-by: Vipul Kumar Signed-off-by: Srikanth Krishnakar (cherry picked from commit 0ea90e5f83725fe5e4b8933cc546c1ff22156717) Signed-off-by: Robin Gong --- drivers/dma/imx-sdma.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 3a5536055729..c226db318092 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -170,6 +170,8 @@ #define SDMA_WATERMARK_LEVEL_SPDIF BIT(10) #define SDMA_WATERMARK_LEVEL_SP BIT(11) #define SDMA_WATERMARK_LEVEL_DP BIT(12) +#define SDMA_WATERMARK_LEVEL_SD BIT(13) +#define SDMA_WATERMARK_LEVEL_DD BIT(14) #define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16) #define SDMA_WATERMARK_LEVEL_LWE BIT(28) #define SDMA_WATERMARK_LEVEL_HWE BIT(29) @@ -382,6 +384,8 @@ struct sdma_channel { struct imx_dma_data data; struct work_struct terminate_worker; bool is_ram_script; + bool src_dualfifo; + bool dst_dualfifo; }; #define IMX_DMA_SG_LOOP BIT(0) @@ -1239,6 +1243,11 @@ static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP; sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT; + + if (sdmac->src_dualfifo) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SD; + if (sdmac->dst_dualfifo) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD; } static int sdma_config_channel(struct dma_chan *chan) @@ -1421,6 +1430,8 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) sdmac->peripheral_type = data->peripheral_type; sdmac->event_id0 = data->dma_request; sdmac->event_id1 = data->dma_request2; + sdmac->src_dualfifo = data->src_dualfifo; + sdmac->dst_dualfifo = data->dst_dualfifo; ret = clk_enable(sdmac->sdma->clk_ipg); if (ret) @@ -2102,6 +2113,8 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, if (dma_spec->args_count != 3) return NULL; + memset(&data, 0, sizeof(data)); + data.dma_request = dma_spec->args[0]; data.peripheral_type = dma_spec->args[1]; data.priority = dma_spec->args[2]; From 40373109c47f7b278f36207c4ba41bfac4a0d948 Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Mon, 20 Nov 2017 16:21:24 +0800 Subject: [PATCH 17/52] MLK-16885-1: DMA: imx-sdma: update the buswidth that is supported update buswidth that is supported by sdma. Signed-off-by: Shengjiu Wang Signed-off-by: Vipul Kumar Signed-off-by: Srikanth Krishnakar (cherry picked from commit c6e6f9cf2661dc1a5ba6fbcfdd15f096191ed47d) --- drivers/dma/imx-sdma.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index c226db318092..fa233885c240 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -179,6 +179,7 @@ #define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) #define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \ From b9416dfe215dfeb752225f950282f01f0c9024ea Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 16 Jan 2018 17:52:49 +0800 Subject: [PATCH 18/52] dmaengine: imx-sdma: update sdma script for multi fifo on SAI update sdma script for multi fifo SAI on i.mx8MQ. Besides,Add new cell for sw_done/sw_done_selector, because PDM need enable software done feature in sdma script(same multi fifo script). The new fourth cell defined as below: Bit31: sw_done Bit15~bit0: selector For example: 0x80000000 means sw_done enabled for done0 sector which is for PDM on i.mx8mm. Signed-off-by: Robin Gong --- .../devicetree/bindings/dma/fsl-imx-sdma.txt | 10 ++- drivers/dma/imx-sdma.c | 66 ++++++++++++++++++- include/linux/platform_data/dma-imx.h | 1 + 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt b/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt index 0232a60422a1..ebbf7cfb1241 100644 --- a/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt +++ b/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt @@ -55,7 +55,12 @@ The full ID of peripheral types can be found below. 24 SAI 25 HDMI Audio -The third cell specifies the transfer priority as below. +The third cell specifies the transfer priority and software done +as below. + + Bit31: sw_done + Bit15~Bit8: selector + Bit7~Bit0: priority level ID transfer priority ------------------------- @@ -63,6 +68,9 @@ The third cell specifies the transfer priority as below. 1 Medium 2 Low +For example: 0x80000000 means sw_done enabled for done0 sector and + High priority for PDM on i.mx8mm. + Optional properties: - gpr : The phandle to the General Purpose Register (GPR) node. diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index fa233885c240..77e2ce12e9b4 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -76,6 +76,9 @@ #define SDMA_CHNENBL0_IMX35 0x200 #define SDMA_CHNENBL0_IMX31 0x080 #define SDMA_CHNPRI_0 0x100 +#define SDMA_DONE0_CONFIG 0x1000 +#define SDMA_DONE0_CONFIG_DONE_SEL 0x7 +#define SDMA_DONE0_CONFIG_DONE_DIS 0x6 /* * Buffer descriptor status values. @@ -186,6 +189,10 @@ BIT(DMA_MEM_TO_DEV) | \ BIT(DMA_DEV_TO_DEV)) +#define SDMA_WATERMARK_LEVEL_FIFOS_OFF 12 +#define SDMA_WATERMARK_LEVEL_SW_DONE BIT(23) +#define SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF 24 + /* * Mode/Count of data node descriptors - IPCv2 */ @@ -387,6 +394,9 @@ struct sdma_channel { bool is_ram_script; bool src_dualfifo; bool dst_dualfifo; + unsigned int fifo_num; + bool sw_done; + u32 sw_done_sel; }; #define IMX_DMA_SG_LOOP BIT(0) @@ -797,6 +807,21 @@ static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event) val = readl_relaxed(sdma->regs + chnenbl); __set_bit(channel, &val); writel_relaxed(val, sdma->regs + chnenbl); + + /* Set SDMA_DONEx_CONFIG is sw_done enabled */ + if (sdmac->sw_done) { + u32 offset = SDMA_DONE0_CONFIG + sdmac->sw_done_sel / 4; + u32 done_sel = SDMA_DONE0_CONFIG_DONE_SEL + + ((sdmac->sw_done_sel % 4) << 3); + u32 sw_done_dis = SDMA_DONE0_CONFIG_DONE_DIS + + ((sdmac->sw_done_sel % 4) << 3); + + val = readl_relaxed(sdma->regs + offset); + __set_bit(done_sel, &val); + __clear_bit(sw_done_dis, &val); + writel_relaxed(val, sdma->regs + offset); + } + } static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event) @@ -1045,6 +1070,9 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_HDMI: emi_2_per = sdma->script_addrs->hdmi_dma_addr; break; + case IMX_DMATYPE_MULTI_SAI: + per_2_emi = sdma->script_addrs->sai_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_sai_addr; default: break; } @@ -1251,6 +1279,26 @@ static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD; } +static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac) +{ + sdmac->watermark_level &= ~(0xFF << SDMA_WATERMARK_LEVEL_FIFOS_OFF | + SDMA_WATERMARK_LEVEL_SW_DONE | + 0xf << SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF); + + if (sdmac->sw_done) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SW_DONE | + sdmac->sw_done_sel << + SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF; + + /* For fifo_num + * bit 12-15 is the fifo number; + * bit 16-19 is the fifo offset, + * so here only need to shift left fifo_num 12 bit for watermake_level + */ + sdmac->watermark_level |= sdmac->fifo_num<< + SDMA_WATERMARK_LEVEL_FIFOS_OFF; +} + static int sdma_config_channel(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); @@ -1292,6 +1340,10 @@ static int sdma_config_channel(struct dma_chan *chan) sdmac->direction == DMA_MEM_TO_DEV && sdmac->sdma->drvdata->ecspi_fixed) __set_bit(31, &sdmac->watermark_level); + else if (sdmac->peripheral_type == + IMX_DMATYPE_MULTI_SAI) + sdma_set_watermarklevel_for_sais(sdmac); + __set_bit(sdmac->event_id0, sdmac->event_mask); } @@ -1433,6 +1485,11 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) sdmac->event_id1 = data->dma_request2; sdmac->src_dualfifo = data->src_dualfifo; sdmac->dst_dualfifo = data->dst_dualfifo; + /* Get software done selector if sw_done enabled */ + if (data->done_sel & BIT(31)) { + sdmac->sw_done = true; + sdmac->sw_done_sel = (data->done_sel >> 8) & 0xff; + } ret = clk_enable(sdmac->sdma->clk_ipg); if (ret) @@ -1743,11 +1800,14 @@ static int sdma_config_write(struct dma_chan *chan, { struct sdma_channel *sdmac = to_sdma_chan(chan); + sdmac->watermark_level = 0; + if (direction == DMA_DEV_TO_MEM) { sdmac->per_address = dmaengine_cfg->src_addr; sdmac->watermark_level = dmaengine_cfg->src_maxburst * dmaengine_cfg->src_addr_width; sdmac->word_size = dmaengine_cfg->src_addr_width; + sdmac->fifo_num = dmaengine_cfg->src_fifo_num; } else if (direction == DMA_DEV_TO_DEV) { sdmac->per_address2 = dmaengine_cfg->src_addr; sdmac->per_address = dmaengine_cfg->dst_addr; @@ -1765,6 +1825,7 @@ static int sdma_config_write(struct dma_chan *chan, sdmac->watermark_level = dmaengine_cfg->dst_maxburst * dmaengine_cfg->dst_addr_width; sdmac->word_size = dmaengine_cfg->dst_addr_width; + sdmac->fifo_num = dmaengine_cfg->dst_fifo_num; } sdmac->direction = direction; return sdma_config_channel(chan); @@ -2118,7 +2179,10 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, data.dma_request = dma_spec->args[0]; data.peripheral_type = dma_spec->args[1]; - data.priority = dma_spec->args[2]; + /* Get sw_done setting if sw_done enabled */ + if (dma_spec->args[2] & BIT(31)) + data.done_sel = dma_spec->args[2]; + data.priority = dma_spec->args[2] & 0xff; /* * init dma_request2 to zero, which is not used by the dts. * For P2P, dma_request2 is init from dma_request_channel(), diff --git a/include/linux/platform_data/dma-imx.h b/include/linux/platform_data/dma-imx.h index d26007b456a5..8ed073ab5b75 100644 --- a/include/linux/platform_data/dma-imx.h +++ b/include/linux/platform_data/dma-imx.h @@ -57,6 +57,7 @@ struct imx_dma_data { int priority; bool src_dualfifo; bool dst_dualfifo; + int done_sel; }; static inline int imx_dma_is_ipu(struct dma_chan *chan) From 95fdc2b77fafe255c2a2c99abd06109566a8306a Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Thu, 11 Jul 2019 11:05:14 +0800 Subject: [PATCH 19/52] MLK-22239-1: dmaengine: imx-sdma: Support 24bit/3bytes for sg mode Support 24bit/3bytes for sg mode. Signed-off-by: Shengjiu Wang Acked-by: Robin Gong --- drivers/dma/imx-sdma.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 77e2ce12e9b4..ca6c5e8f048a 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1680,6 +1680,9 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( if (count & 3 || sg->dma_address & 3) goto err_bd_out; break; + case DMA_SLAVE_BUSWIDTH_3_BYTES: + bd->mode.command = 3; + break; case DMA_SLAVE_BUSWIDTH_2_BYTES: bd->mode.command = 2; if (count & 1 || sg->dma_address & 1) From d75a298a952b6ddcc6a8cffffb0ed7b3872ccf7f Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Mon, 23 Sep 2019 17:42:16 +0800 Subject: [PATCH 20/52] dma: imx-sdma: Fix issue with IMX_DMATYPE_HDMI There is no bds for HDMI, so add one more condition for sdma_alloc_bd Signed-off-by: Shengjiu Wang --- drivers/dma/imx-sdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index ca6c5e8f048a..8274a8946103 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1559,7 +1559,7 @@ static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, desc->sdmac = sdmac; desc->num_bd = bds; - if (sdma_alloc_bd(desc)) + if (bds && sdma_alloc_bd(desc)) goto err_desc_out; /* No slave_config called in MEMCPY case, so do here */ From 64acbb6d0a0c7a1a6d06bfb5cbd5bbe39fd73bbe Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Wed, 9 Oct 2019 11:34:49 +0800 Subject: [PATCH 21/52] dma: imx-sdma: Add pm_ops to support suspend & resume Add pm_ops to support suspend & resume Signed-off-by: Shengjiu Wang --- drivers/dma/imx-sdma.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 8274a8946103..bed7c2884188 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -2550,6 +2550,7 @@ static struct platform_driver sdma_driver = { .driver = { .name = "imx-sdma", .of_match_table = sdma_dt_ids, + .pm = &sdma_pm_ops, }, .id_table = sdma_devtypes, .remove = sdma_remove, From bb526c17a4ba8509431c66d7f74602744d9ae24a Mon Sep 17 00:00:00 2001 From: Fugang Duan Date: Wed, 13 Nov 2019 20:16:56 +0800 Subject: [PATCH 22/52] MLK-22972 dmaengine: imx-sdma: correct the script number for v3 Correct the ram script number for v3. Signed-off-by: Fugang Duan --- drivers/dma/imx-sdma.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index bed7c2884188..1d6ef1260623 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1905,8 +1905,8 @@ static void sdma_issue_pending(struct dma_chan *chan) #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 43 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 44 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 45 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 46 static void sdma_add_scripts(struct sdma_engine *sdma, const struct sdma_script_start_addrs *addr) From 58d26348b193e7987e0f61afa3fd0a5bbd5cc858 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 20 Nov 2019 17:52:22 +0800 Subject: [PATCH 23/52] MLK-23005: dmaengine: imx-sdma: correct data size of channel context Correct data size of channel context which would be save and restore back during suspend/resume. Otherwise, potential memory break may come as USB met. Signed-off-by: Robin Gong Acked-by: Fugang Duan Reported-by: Jun Li Tested-by: Jun Li --- drivers/dma/imx-sdma.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 1d6ef1260623..c2d6d49af4de 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -2094,8 +2094,8 @@ static int sdma_init(struct sdma_engine *sdma) /* Be sure SDMA has not started yet */ writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); - ccbsize = MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control) - + sizeof(struct sdma_context_data); + ccbsize = MAX_DMA_CHANNELS * (sizeof(struct sdma_channel_control) + + sizeof(struct sdma_context_data)); if (sdma->iram_pool) sdma->channel_control = gen_pool_dma_alloc(sdma->iram_pool, From 2f9e74892d5c799ef491117aed766dd5aeac857c Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Fri, 31 Mar 2017 15:53:39 +0800 Subject: [PATCH 24/52] MLK-14610 DMA: fsl-edma-v3: add fsl-edma-v3 support Add edma-v3 driver on i.mx8qm. Signed-off-by: Robin Gong (cherry picked from commit d0ac0971c2e637ebddc853f12f71d130f5df4f91) --- .../devicetree/bindings/dma/fsl-edma-v3.txt | 64 ++ drivers/dma/Kconfig | 11 + drivers/dma/Makefile | 1 + drivers/dma/fsl-edma-v3.c | 890 ++++++++++++++++++ 4 files changed, 966 insertions(+) create mode 100644 Documentation/devicetree/bindings/dma/fsl-edma-v3.txt create mode 100644 drivers/dma/fsl-edma-v3.c diff --git a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt new file mode 100644 index 000000000000..4c574937b729 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt @@ -0,0 +1,64 @@ +* Freescale enhanced Direct Memory Access(eDMA-v3) Controller + + The eDMA-v3 controller is inherited from FSL eDMA, and firstly is intergrated + on Freescale i.MX8QM SOC chip. The eDMA channels have multiplex capability by + programmble memory-mapped registers. Specific DMA request source has fixed channel. + +* eDMA Controller +Required properties: +- compatible : + - "fsl,imx8qm-edma" for eDMA used similar to that on i.MX8QM SoC + - "fsl,imx8qm-adma" for audio eDMA used on i.MX8QM +- reg : Specifies base physical address(s) and size of the eDMA channel registers. + Each eDMA channel has separated register's address and size. +- interrupts : A list of interrupt-specifiers, each channel has one interrupt. +- interrupt-names : Should contain: + "edma-chan12-tx" - the channel12 transmission interrupt +- #dma-cells : Must be <3>. + The 1st cell specifies the channel ID. + The 2nd cell specifies the channel priority. + The 3rd cell specifies the channel type like for transmit or receive: + 0: transmit, 1: receive. + See the SoC's reference manual for all the supported request sources. +- dma-channels : Number of channels supported by the controller + +Examples: +edma0: dma-controller@40018000 { + compatible = "fsl,imx8qm-edma"; + reg = <0x0 0x5a2c0000 0x0 0x10000>, /* channel12 UART0 rx */ + <0x0 0x5a2d0000 0x0 0x10000>, /* channel13 UART0 tx */ + <0x0 0x5a2e0000 0x0 0x10000>, /* channel14 UART1 rx */ + <0x0 0x5a2f0000 0x0 0x10000>; /* channel15 UART1 tx */ + #dma-cells = <3>; + dma-channels = <4>; + interrupts = , + , + , + ; + interrupt-names = "edma-chan12-tx", "edma-chan13-tx", + "edma-chan14-tx", "edma-chan15-tx"; + status = "okay"; +}; + +* DMA clients +DMA client drivers that uses the DMA function must use the format described +in the dma.txt file, using a three-cell specifier for each channel: the 1st +specifies the channel number, the 2nd specifies the priority, and the 3rd +specifies the channel type is for transmit or receive: 0: transmit, 1: receive. + +Examples: +lpuart1: serial@5a070000 { + compatible = "fsl,imx8qm-lpuart"; + reg = <0x0 0x5a070000 0x0 0x1000>; + interrupts = ; + interrupt-parent = <&gic>; + clocks = <&clk IMX8QM_UART1_CLK>; + clock-names = "ipg"; + assigned-clock-names = <&clk IMX8QM_UART1_CLK>; + assigned-clock-rates = <80000000>; + power-domains = <&pd_dma_lpuart1>; + dma-names = "tx","rx"; + dmas = <&edma0 15 0 0>, + <&edma0 14 0 1>; + status = "disabled"; +}; diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 7af874b69ffb..25607103f040 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -227,6 +227,17 @@ config FSL_QDMA or dequeuing DMA jobs from, different work queues. This module can be found on NXP Layerscape SoCs. The qdma driver only work on SoCs with a DPAA hardware block. +config FSL_EDMA_V3 + tristate "Freescale eDMA v3 engine support" + depends on OF + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support the Freescale eDMA v3 engine with programmable channel. + This driver is based on FSL_EDMA but big changes come such as + different interrupt for different channel, different register + scope for different channel. + This module can be found on Freescale i.MX8QM. config FSL_RAID tristate "Freescale RAID engine Support" diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index f5ce8665e944..9752eea7358d 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_DW_EDMA) += dw-edma/ obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o obj-$(CONFIG_FSL_DMA) += fsldma.o obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o +obj-$(CONFIG_FSL_EDMA_V3) += fsl-edma-v3.o obj-$(CONFIG_MCF_EDMA) += mcf-edma.o fsl-edma-common.o obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o obj-$(CONFIG_FSL_RAID) += fsl_raid.o diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c new file mode 100644 index 000000000000..042154f320f9 --- /dev/null +++ b/drivers/dma/fsl-edma-v3.c @@ -0,0 +1,890 @@ +/* + * drivers/dma/fsl-edma3-v3.c + * + * Copyright 2017 NXP . + * + * Driver for the Freescale eDMA engine v3. This driver based on fsl-edma3.c + * but changed to meet the IP change on i.MX8QM: every dma channel is specific + * to hardware. For example, channel 14 for LPUART1 receive request and channel + * 13 for transmit requesst. The eDMA block can be found on i.MX8QM + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "virt-dma.h" + +#define EDMA_CH_CSR 0x00 +#define EDMA_CH_ES 0x04 +#define EDMA_CH_INT 0x08 +#define EDMA_CH_SBR 0x0C +#define EDMA_CH_PRI 0x10 +#define EDMA_TCD_SADDR 0x20 +#define EDMA_TCD_SOFF 0x24 +#define EDMA_TCD_ATTR 0x26 +#define EDMA_TCD_NBYTES 0x28 +#define EDMA_TCD_SLAST 0x2C +#define EDMA_TCD_DADDR 0x30 +#define EDMA_TCD_DOFF 0x34 +#define EDMA_TCD_CITER_ELINK 0x36 +#define EDMA_TCD_CITER 0x36 +#define EDMA_TCD_DLAST_SGA 0x38 +#define EDMA_TCD_CSR 0x3C +#define EDMA_TCD_BITER_ELINK 0x3E +#define EDMA_TCD_BITER 0x3E + +#define EDMA_CH_SBR_RD BIT(22) +#define EDMA_CH_SBR_WR BIT(21) +#define EDMA_CH_CSR_ERQ BIT(0) +#define EDMA_CH_CSR_EARQ BIT(1) +#define EDMA_CH_CSR_EEI BIT(2) +#define EDMA_CH_CSR_DONE BIT(30) +#define EDMA_CH_CSR_ACTIVE BIT(31) + +#define EDMA_TCD_ATTR_DSIZE(x) (((x) & 0x0007)) +#define EDMA_TCD_ATTR_DMOD(x) (((x) & 0x001F) << 3) +#define EDMA_TCD_ATTR_SSIZE(x) (((x) & 0x0007) << 8) +#define EDMA_TCD_ATTR_SMOD(x) (((x) & 0x001F) << 11) +#define EDMA_TCD_ATTR_SSIZE_8BIT (0x0000) +#define EDMA_TCD_ATTR_SSIZE_16BIT (0x0100) +#define EDMA_TCD_ATTR_SSIZE_32BIT (0x0200) +#define EDMA_TCD_ATTR_SSIZE_64BIT (0x0300) +#define EDMA_TCD_ATTR_SSIZE_16BYTE (0x0400) +#define EDMA_TCD_ATTR_SSIZE_32BYTE (0x0500) +#define EDMA_TCD_ATTR_SSIZE_64BYTE (0x0600) +#define EDMA_TCD_ATTR_DSIZE_8BIT (0x0000) +#define EDMA_TCD_ATTR_DSIZE_16BIT (0x0001) +#define EDMA_TCD_ATTR_DSIZE_32BIT (0x0002) +#define EDMA_TCD_ATTR_DSIZE_64BIT (0x0003) +#define EDMA_TCD_ATTR_DSIZE_16BYTE (0x0004) +#define EDMA_TCD_ATTR_DSIZE_32BYTE (0x0005) +#define EDMA_TCD_ATTR_DSIZE_64BYTE (0x0006) + +#define EDMA_TCD_SOFF_SOFF(x) (x) +#define EDMA_TCD_NBYTES_NBYTES(x) (x) +#define EDMA_TCD_SLAST_SLAST(x) (x) +#define EDMA_TCD_DADDR_DADDR(x) (x) +#define EDMA_TCD_CITER_CITER(x) ((x) & 0x7FFF) +#define EDMA_TCD_DOFF_DOFF(x) (x) +#define EDMA_TCD_DLAST_SGA_DLAST_SGA(x) (x) +#define EDMA_TCD_BITER_BITER(x) ((x) & 0x7FFF) + +#define EDMA_TCD_CSR_START BIT(0) +#define EDMA_TCD_CSR_INT_MAJOR BIT(1) +#define EDMA_TCD_CSR_INT_HALF BIT(2) +#define EDMA_TCD_CSR_D_REQ BIT(3) +#define EDMA_TCD_CSR_E_SG BIT(4) +#define EDMA_TCD_CSR_E_LINK BIT(5) +#define EDMA_TCD_CSR_ACTIVE BIT(6) +#define EDMA_TCD_CSR_DONE BIT(7) + +#define FSL_EDMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_16_BYTES)) + +struct fsl_edma3_hw_tcd { + __le32 saddr; + __le16 soff; + __le16 attr; + __le32 nbytes; + __le32 slast; + __le32 daddr; + __le16 doff; + __le16 citer; + __le32 dlast_sga; + __le16 csr; + __le16 biter; +}; + +struct fsl_edma3_sw_tcd { + dma_addr_t ptcd; + struct fsl_edma3_hw_tcd *vtcd; +}; + +struct fsl_edma3_slave_config { + enum dma_transfer_direction dir; + enum dma_slave_buswidth addr_width; + u32 dev_addr; + u32 dev2_addr; /* source addr for dev2dev */ + u32 burst; + u32 attr; +}; + +struct fsl_edma3_chan { + struct virt_dma_chan vchan; + enum dma_status status; + struct fsl_edma3_engine *edma3; + struct fsl_edma3_desc *edesc; + struct fsl_edma3_slave_config fsc; + void __iomem *membase; + int txirq; + int hw_chanid; + int priority; + int is_rxchan; + struct dma_pool *tcd_pool; + u32 chn_real_count; + char txirq_name[32]; +}; + +struct fsl_edma3_desc { + struct virt_dma_desc vdesc; + struct fsl_edma3_chan *echan; + bool iscyclic; + unsigned int n_tcds; + struct fsl_edma3_sw_tcd tcd[]; +}; + +struct fsl_edma3_engine { + struct dma_device dma_dev; + struct mutex fsl_edma3_mutex; + u32 n_chans; + int errirq; + bool swap; /* remote/local swapped on Audio edma */ + struct fsl_edma3_chan chans[]; +}; + +static struct fsl_edma3_chan *to_fsl_edma3_chan(struct dma_chan *chan) +{ + return container_of(chan, struct fsl_edma3_chan, vchan.chan); +} + +static struct fsl_edma3_desc *to_fsl_edma3_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct fsl_edma3_desc, vdesc); +} + +static void fsl_edma3_enable_request(struct fsl_edma3_chan *fsl_chan) +{ + void __iomem *addr = fsl_chan->membase; + u32 val; + + val = readl(addr + EDMA_CH_SBR); + /* Remote/local swapped wrongly on iMX8 QM Audio edma */ + if (fsl_chan->edma3->swap) { + if (!fsl_chan->is_rxchan) + val |= EDMA_CH_SBR_RD; + else + val |= EDMA_CH_SBR_WR; + } else { + if (fsl_chan->is_rxchan) + val |= EDMA_CH_SBR_RD; + else + val |= EDMA_CH_SBR_WR; + } + writel(val, addr + EDMA_CH_SBR); + + val = readl(addr + EDMA_CH_CSR); + + val |= EDMA_CH_CSR_ERQ; + writel(val, addr + EDMA_CH_CSR); +} + +static void fsl_edma3_disable_request(struct fsl_edma3_chan *fsl_chan) +{ + void __iomem *addr = fsl_chan->membase; + u32 val = readl(addr + EDMA_CH_CSR); + + val &= ~EDMA_CH_CSR_ERQ; + writel(val, addr + EDMA_CH_CSR); +} + +static unsigned int fsl_edma3_get_tcd_attr(enum dma_slave_buswidth addr_width) +{ + switch (addr_width) { + case 1: + return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; + case 2: + return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; + case 4: + return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; + case 8: + return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; + case 16: + return EDMA_TCD_ATTR_SSIZE_16BYTE | EDMA_TCD_ATTR_DSIZE_16BYTE; + case 32: + return EDMA_TCD_ATTR_SSIZE_32BYTE | EDMA_TCD_ATTR_DSIZE_32BYTE; + case 64: + return EDMA_TCD_ATTR_SSIZE_64BYTE | EDMA_TCD_ATTR_DSIZE_64BYTE; + default: + return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; + } +} + +static void fsl_edma3_free_desc(struct virt_dma_desc *vdesc) +{ + struct fsl_edma3_desc *fsl_desc; + int i; + + fsl_desc = to_fsl_edma3_desc(vdesc); + for (i = 0; i < fsl_desc->n_tcds; i++) + dma_pool_free(fsl_desc->echan->tcd_pool, fsl_desc->tcd[i].vtcd, + fsl_desc->tcd[i].ptcd); + kfree(fsl_desc); +} + +static int fsl_edma3_terminate_all(struct dma_chan *chan) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + fsl_edma3_disable_request(fsl_chan); + fsl_chan->edesc = NULL; + vchan_get_all_descriptors(&fsl_chan->vchan, &head); + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + return 0; +} + +static int fsl_edma3_pause(struct dma_chan *chan) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + if (fsl_chan->edesc) { + fsl_edma3_disable_request(fsl_chan); + fsl_chan->status = DMA_PAUSED; + } + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + return 0; +} + +static int fsl_edma3_resume(struct dma_chan *chan) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + if (fsl_chan->edesc) { + fsl_edma3_enable_request(fsl_chan); + fsl_chan->status = DMA_IN_PROGRESS; + } + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + return 0; +} + +static int fsl_edma3_slave_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + + fsl_chan->fsc.dir = cfg->direction; + if (cfg->direction == DMA_DEV_TO_MEM) { + fsl_chan->fsc.dev_addr = cfg->src_addr; + fsl_chan->fsc.addr_width = cfg->src_addr_width; + fsl_chan->fsc.burst = cfg->src_maxburst; + fsl_chan->fsc.attr = fsl_edma3_get_tcd_attr + (cfg->src_addr_width); + } else if (cfg->direction == DMA_MEM_TO_DEV) { + fsl_chan->fsc.dev_addr = cfg->dst_addr; + fsl_chan->fsc.addr_width = cfg->dst_addr_width; + fsl_chan->fsc.burst = cfg->dst_maxburst; + fsl_chan->fsc.attr = fsl_edma3_get_tcd_attr + (cfg->dst_addr_width); + } else if (cfg->direction == DMA_DEV_TO_DEV) { + fsl_chan->fsc.dev2_addr = cfg->src_addr; + fsl_chan->fsc.dev_addr = cfg->dst_addr; + fsl_chan->fsc.addr_width = cfg->dst_addr_width; + fsl_chan->fsc.burst = cfg->dst_maxburst; + fsl_chan->fsc.attr = fsl_edma3_get_tcd_attr + (cfg->dst_addr_width); + } else { + return -EINVAL; + } + return 0; +} + +static size_t fsl_edma3_desc_residue(struct fsl_edma3_chan *fsl_chan, + struct virt_dma_desc *vdesc, bool in_progress) +{ + struct fsl_edma3_desc *edesc = fsl_chan->edesc; + void __iomem *addr = fsl_chan->membase; + enum dma_transfer_direction dir = fsl_chan->fsc.dir; + dma_addr_t cur_addr, dma_addr; + size_t len, size; + int i; + + /* calculate the total size in this desc */ + for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) + len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) + * le16_to_cpu(edesc->tcd[i].vtcd->biter); + + if (!in_progress) + return len; + + if (dir == DMA_MEM_TO_DEV) + cur_addr = readl(addr + EDMA_TCD_SADDR); + else + cur_addr = readl(addr + EDMA_TCD_DADDR); + + /* figure out the finished and calculate the residue */ + for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { + size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) + * le16_to_cpu(edesc->tcd[i].vtcd->biter); + if (dir == DMA_MEM_TO_DEV) + dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); + else + dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); + + len -= size; + if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { + len += dma_addr + size - cur_addr; + break; + } + } + + return len; +} + +static enum dma_status fsl_edma3_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + struct virt_dma_desc *vdesc; + enum dma_status status; + unsigned long flags; + + status = dma_cookie_status(chan, cookie, txstate); + if (status == DMA_COMPLETE) { + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + txstate->residue = fsl_chan->chn_real_count; + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + return status; + } + + if (!txstate) + return fsl_chan->status; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + vdesc = vchan_find_desc(&fsl_chan->vchan, cookie); + if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie) + txstate->residue = fsl_edma3_desc_residue(fsl_chan, vdesc, + true); + else if (vdesc) + txstate->residue = fsl_edma3_desc_residue(fsl_chan, vdesc, + false); + else + txstate->residue = 0; + + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + + return fsl_chan->status; +} + +static void fsl_edma3_set_tcd_regs(struct fsl_edma3_chan *fsl_chan, + struct fsl_edma3_hw_tcd *tcd) +{ + void __iomem *addr = fsl_chan->membase; + /* + * TCD parameters are stored in struct fsl_edma3_hw_tcd in little + * endian format. However, we need to load the TCD registers in + * big- or little-endian obeying the eDMA engine model endian. + */ + writew(0, addr + EDMA_TCD_CSR); + writel(le32_to_cpu(tcd->saddr), addr + EDMA_TCD_SADDR); + writel(le32_to_cpu(tcd->daddr), addr + EDMA_TCD_DADDR); + + writew(le16_to_cpu(tcd->attr), addr + EDMA_TCD_ATTR); + writew(le16_to_cpu(tcd->soff), addr + EDMA_TCD_SOFF); + + writel(le32_to_cpu(tcd->nbytes), addr + EDMA_TCD_NBYTES); + writel(le32_to_cpu(tcd->slast), addr + EDMA_TCD_SLAST); + + writew(le16_to_cpu(tcd->citer), addr + EDMA_TCD_CITER); + writew(le16_to_cpu(tcd->biter), addr + EDMA_TCD_BITER); + writew(le16_to_cpu(tcd->doff), addr + EDMA_TCD_DOFF); + + writel(le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA); + + writew(le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR); +} + +static inline +void fsl_edma3_fill_tcd(struct fsl_edma3_chan *fsl_chan, + struct fsl_edma3_hw_tcd *tcd, u32 src, u32 dst, + u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, + u16 biter, u16 doff, u32 dlast_sga, bool major_int, + bool disable_req, bool enable_sg) +{ + u16 csr = 0; + + /* + * eDMA hardware SGs require the TCDs to be stored in little + * endian format irrespective of the register endian model. + * So we put the value in little endian in memory, waiting + * for fsl_edma3_set_tcd_regs doing the swap. + */ + tcd->saddr = cpu_to_le32(src); + tcd->daddr = cpu_to_le32(dst); + + tcd->attr = cpu_to_le16(attr); + + tcd->soff = cpu_to_le16(EDMA_TCD_SOFF_SOFF(soff)); + + tcd->nbytes = cpu_to_le32(EDMA_TCD_NBYTES_NBYTES(nbytes)); + tcd->slast = cpu_to_le32(EDMA_TCD_SLAST_SLAST(slast)); + + tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); + tcd->doff = cpu_to_le16(EDMA_TCD_DOFF_DOFF(doff)); + + tcd->dlast_sga = cpu_to_le32(EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga)); + + tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); + if (major_int) + csr |= EDMA_TCD_CSR_INT_MAJOR; + + if (disable_req) + csr |= EDMA_TCD_CSR_D_REQ; + + if (enable_sg) + csr |= EDMA_TCD_CSR_E_SG; + + if (fsl_chan->is_rxchan) + csr |= EDMA_TCD_CSR_ACTIVE; + + tcd->csr = cpu_to_le16(csr); +} + +static struct fsl_edma3_desc *fsl_edma3_alloc_desc(struct fsl_edma3_chan + *fsl_chan, int sg_len) +{ + struct fsl_edma3_desc *fsl_desc; + int i; + + fsl_desc = kzalloc(sizeof(*fsl_desc) + sizeof(struct fsl_edma3_sw_tcd) + * sg_len, GFP_ATOMIC); + if (!fsl_desc) + return NULL; + + fsl_desc->echan = fsl_chan; + fsl_desc->n_tcds = sg_len; + for (i = 0; i < sg_len; i++) { + fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool, + GFP_ATOMIC, &fsl_desc->tcd[i].ptcd); + if (!fsl_desc->tcd[i].vtcd) + goto err; + } + return fsl_desc; + +err: + while (--i >= 0) + dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd, + fsl_desc->tcd[i].ptcd); + kfree(fsl_desc); + return NULL; +} + +static struct dma_async_tx_descriptor *fsl_edma3_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + struct fsl_edma3_desc *fsl_desc; + dma_addr_t dma_buf_next; + int sg_len, i; + u32 src_addr, dst_addr, last_sg, nbytes; + u16 soff, doff, iter; + + sg_len = buf_len / period_len; + fsl_desc = fsl_edma3_alloc_desc(fsl_chan, sg_len); + if (!fsl_desc) + return NULL; + fsl_desc->iscyclic = true; + + dma_buf_next = dma_addr; + nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; + iter = period_len / nbytes; + + for (i = 0; i < sg_len; i++) { + if (dma_buf_next >= dma_addr + buf_len) + dma_buf_next = dma_addr; + + /* get next sg's physical address */ + last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; + + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { + src_addr = dma_buf_next; + dst_addr = fsl_chan->fsc.dev_addr; + soff = fsl_chan->fsc.addr_width; + doff = 0; + } else if (fsl_chan->fsc.dir == DMA_DEV_TO_MEM) { + src_addr = fsl_chan->fsc.dev_addr; + dst_addr = dma_buf_next; + soff = 0; + doff = fsl_chan->fsc.addr_width; + } else { + /* DMA_DEV_TO_DEV */ + src_addr = fsl_chan->fsc.dev2_addr; + dst_addr = fsl_chan->fsc.dev_addr; + soff = 0; + doff = 0; + } + + fsl_edma3_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->fsc.attr, soff, nbytes, 0, + iter, iter, doff, last_sg, true, false, true); + dma_buf_next += period_len; + } + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); +} + +static struct dma_async_tx_descriptor *fsl_edma3_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + struct fsl_edma3_desc *fsl_desc; + struct scatterlist *sg; + u32 src_addr, dst_addr, last_sg, nbytes; + u16 soff, doff, iter; + int i; + + if (!is_slave_direction(fsl_chan->fsc.dir)) + return NULL; + + fsl_desc = fsl_edma3_alloc_desc(fsl_chan, sg_len); + if (!fsl_desc) + return NULL; + fsl_desc->iscyclic = false; + + nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; + for_each_sg(sgl, sg, sg_len, i) { + /* get next sg's physical address */ + last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; + + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { + src_addr = sg_dma_address(sg); + dst_addr = fsl_chan->fsc.dev_addr; + soff = fsl_chan->fsc.addr_width; + doff = 0; + } else if (fsl_chan->fsc.dir == DMA_DEV_TO_MEM) { + src_addr = fsl_chan->fsc.dev_addr; + dst_addr = sg_dma_address(sg); + soff = 0; + doff = fsl_chan->fsc.addr_width; + } else { + /* DMA_DEV_TO_DEV */ + src_addr = fsl_chan->fsc.dev2_addr; + dst_addr = fsl_chan->fsc.dev_addr; + soff = 0; + doff = 0; + } + + iter = sg_dma_len(sg) / nbytes; + if (i < sg_len - 1) { + last_sg = fsl_desc->tcd[(i + 1)].ptcd; + fsl_edma3_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, + src_addr, dst_addr, fsl_chan->fsc.attr, + soff, nbytes, 0, iter, iter, doff, + last_sg, false, false, true); + } else { + last_sg = 0; + fsl_edma3_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, + src_addr, dst_addr, fsl_chan->fsc.attr, + soff, nbytes, 0, iter, iter, doff, + last_sg, true, true, false); + } + } + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); +} + +static void fsl_edma3_xfer_desc(struct fsl_edma3_chan *fsl_chan) +{ + struct virt_dma_desc *vdesc; + + vdesc = vchan_next_desc(&fsl_chan->vchan); + if (!vdesc) + return; + fsl_chan->edesc = to_fsl_edma3_desc(vdesc); + fsl_edma3_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); + fsl_edma3_enable_request(fsl_chan); + fsl_chan->status = DMA_IN_PROGRESS; +} + +static size_t fsl_edma3_desc_residue(struct fsl_edma3_chan *fsl_chan, + struct virt_dma_desc *vdesc, bool in_progress); + +static void fsl_edma3_get_realcnt(struct fsl_edma3_chan *fsl_chan) +{ + fsl_chan->chn_real_count = fsl_edma3_desc_residue(fsl_chan, NULL, true); +} + +static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) +{ + struct fsl_edma3_chan *fsl_chan = dev_id; + unsigned int intr; + void __iomem *base_addr; + + base_addr = fsl_chan->membase; + + intr = readl(base_addr + EDMA_CH_INT); + if (!intr) + return IRQ_NONE; + + writel(1, base_addr + EDMA_CH_INT); + + spin_lock(&fsl_chan->vchan.lock); + if (!fsl_chan->edesc->iscyclic) { + fsl_edma3_get_realcnt(fsl_chan); + list_del(&fsl_chan->edesc->vdesc.node); + vchan_cookie_complete(&fsl_chan->edesc->vdesc); + fsl_chan->edesc = NULL; + fsl_chan->status = DMA_COMPLETE; + } else { + vchan_cyclic_callback(&fsl_chan->edesc->vdesc); + } + + if (!fsl_chan->edesc) + fsl_edma3_xfer_desc(fsl_chan); + + spin_unlock(&fsl_chan->vchan.lock); + + return IRQ_HANDLED; +} + +static void fsl_edma3_issue_pending(struct dma_chan *chan) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + + if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) + fsl_edma3_xfer_desc(fsl_chan); + + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); +} + +static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct fsl_edma3_engine *fsl_edma3 = ofdma->of_dma_data; + struct dma_chan *chan, *_chan; + struct fsl_edma3_chan *fsl_chan; + + if (dma_spec->args_count != 3) + return NULL; + + mutex_lock(&fsl_edma3->fsl_edma3_mutex); + list_for_each_entry_safe(chan, _chan, &fsl_edma3->dma_dev.channels, + device_node) { + if (chan->client_count) + continue; + + fsl_chan = to_fsl_edma3_chan(chan); + if (fsl_chan->hw_chanid == dma_spec->args[0]) { + chan = dma_get_slave_channel(chan); + chan->device->privatecnt++; + fsl_chan->priority = dma_spec->args[1]; + fsl_chan->is_rxchan = dma_spec->args[2]; + mutex_unlock(&fsl_edma3->fsl_edma3_mutex); + return chan; + } + } + mutex_unlock(&fsl_edma3->fsl_edma3_mutex); + return NULL; +} + +static int fsl_edma3_alloc_chan_resources(struct dma_chan *chan) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + + fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, + sizeof(struct fsl_edma3_hw_tcd), + 32, 0); + return 0; +} + +static void fsl_edma3_free_chan_resources(struct dma_chan *chan) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + fsl_edma3_disable_request(fsl_chan); + fsl_chan->edesc = NULL; + vchan_get_all_descriptors(&fsl_chan->vchan, &head); + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + dma_pool_destroy(fsl_chan->tcd_pool); + fsl_chan->tcd_pool = NULL; +} + +static int fsl_edma3_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct fsl_edma3_engine *fsl_edma3; + struct fsl_edma3_chan *fsl_chan; + struct resource *res; + int len, chans; + int ret, i; + unsigned long irqflag = 0; + + ret = of_property_read_u32(np, "dma-channels", &chans); + if (ret) { + dev_err(&pdev->dev, "Can't get dma-channels.\n"); + return ret; + } + + len = sizeof(*fsl_edma3) + sizeof(*fsl_chan) * chans; + fsl_edma3 = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + if (!fsl_edma3) + return -ENOMEM; + + /* Audio edma rx/tx channel shared interrupt */ + if (of_property_read_bool(np, "shared-interrupt")) + irqflag = IRQF_SHARED; + + fsl_edma3->swap = of_device_is_compatible(np, "fsl,imx8qm-adma"); + fsl_edma3->n_chans = chans; + + INIT_LIST_HEAD(&fsl_edma3->dma_dev.channels); + for (i = 0; i < fsl_edma3->n_chans; i++) { + struct fsl_edma3_chan *fsl_chan = &fsl_edma3->chans[i]; + char *txirq_name = fsl_chan->txirq_name; + + fsl_chan->edma3 = fsl_edma3; + /* Get per channel membase */ + res = platform_get_resource(pdev, IORESOURCE_MEM, i); + fsl_chan->membase = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(fsl_chan->membase)) + return PTR_ERR(fsl_chan->membase); + + /* Get the hardware chanel id by the channel membase + * channel0:0x10000, channel1:0x20000... total 32 channels + */ + fsl_chan->hw_chanid = (res->start >> 16) & 0x1f; + sprintf(txirq_name, "edma-chan%d-tx", fsl_chan->hw_chanid); + + /* request channel irq */ + fsl_chan->txirq = platform_get_irq_byname(pdev, txirq_name); + if (fsl_chan->txirq < 0) { + dev_err(&pdev->dev, "Can't get %s irq.\n", txirq_name); + return fsl_chan->txirq; + } + + ret = devm_request_irq(&pdev->dev, fsl_chan->txirq, + fsl_edma3_tx_handler, irqflag, txirq_name, + fsl_chan); + if (ret) { + dev_err(&pdev->dev, "Can't register %s IRQ.\n", + txirq_name); + return ret; + } + + fsl_chan->vchan.desc_free = fsl_edma3_free_desc; + vchan_init(&fsl_chan->vchan, &fsl_edma3->dma_dev); + } + + mutex_init(&fsl_edma3->fsl_edma3_mutex); + + dma_cap_set(DMA_PRIVATE, fsl_edma3->dma_dev.cap_mask); + dma_cap_set(DMA_SLAVE, fsl_edma3->dma_dev.cap_mask); + dma_cap_set(DMA_CYCLIC, fsl_edma3->dma_dev.cap_mask); + + fsl_edma3->dma_dev.dev = &pdev->dev; + fsl_edma3->dma_dev.device_alloc_chan_resources + = fsl_edma3_alloc_chan_resources; + fsl_edma3->dma_dev.device_free_chan_resources + = fsl_edma3_free_chan_resources; + fsl_edma3->dma_dev.device_tx_status = fsl_edma3_tx_status; + fsl_edma3->dma_dev.device_prep_slave_sg = fsl_edma3_prep_slave_sg; + fsl_edma3->dma_dev.device_prep_dma_cyclic = fsl_edma3_prep_dma_cyclic; + fsl_edma3->dma_dev.device_config = fsl_edma3_slave_config; + fsl_edma3->dma_dev.device_pause = fsl_edma3_pause; + fsl_edma3->dma_dev.device_resume = fsl_edma3_resume; + fsl_edma3->dma_dev.device_terminate_all = fsl_edma3_terminate_all; + fsl_edma3->dma_dev.device_issue_pending = fsl_edma3_issue_pending; + + fsl_edma3->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS; + fsl_edma3->dma_dev.dst_addr_widths = FSL_EDMA_BUSWIDTHS; + fsl_edma3->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | + BIT(DMA_MEM_TO_DEV) | + BIT(DMA_DEV_TO_DEV); + + platform_set_drvdata(pdev, fsl_edma3); + + ret = dma_async_device_register(&fsl_edma3->dma_dev); + if (ret) { + dev_err(&pdev->dev, "Can't register Freescale eDMA engine.\n"); + return ret; + } + + ret = of_dma_controller_register(np, fsl_edma3_xlate, fsl_edma3); + if (ret) { + dev_err(&pdev->dev, "Can't register Freescale eDMA of_dma.\n"); + dma_async_device_unregister(&fsl_edma3->dma_dev); + return ret; + } + + return 0; +} + +static int fsl_edma3_remove(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct fsl_edma3_engine *fsl_edma3 = platform_get_drvdata(pdev); + + of_dma_controller_free(np); + dma_async_device_unregister(&fsl_edma3->dma_dev); + + return 0; +} + +static const struct of_device_id fsl_edma3_dt_ids[] = { + { .compatible = "fsl,imx8qm-edma", }, + { .compatible = "fsl,imx8qm-adma", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, fsl_edma3_dt_ids); + +static struct platform_driver fsl_edma3_driver = { + .driver = { + .name = "fsl-edma-v3", + .of_match_table = fsl_edma3_dt_ids, + }, + .probe = fsl_edma3_probe, + .remove = fsl_edma3_remove, +}; + +static int __init fsl_edma3_init(void) +{ + return platform_driver_register(&fsl_edma3_driver); +} +subsys_initcall(fsl_edma3_init); + +static void __exit fsl_edma3_exit(void) +{ + platform_driver_unregister(&fsl_edma3_driver); +} +module_exit(fsl_edma3_exit); + +MODULE_ALIAS("platform:fsl-edma3"); +MODULE_DESCRIPTION("Freescale eDMA-V3 engine driver"); +MODULE_LICENSE("GPL v2"); From f61b296ab2ebf011cd5732e588144a61c3f959fb Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Mon, 5 Jun 2017 11:05:52 +0800 Subject: [PATCH 25/52] MLK-15003-1: DMA: fsl-edma-v3: add one more parameter for xlate The parameter is "is_remote", which is to use remote access for edma, the default access is local access. Signed-off-by: Shengjiu Wang (cherry picked from commit eee976b30b0523680f30e762742984f5b5a01b97) --- drivers/dma/fsl-edma-v3.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index 042154f320f9..c20306cad822 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -139,6 +139,7 @@ struct fsl_edma3_chan { int hw_chanid; int priority; int is_rxchan; + int is_remote; struct dma_pool *tcd_pool; u32 chn_real_count; char txirq_name[32]; @@ -189,6 +190,10 @@ static void fsl_edma3_enable_request(struct fsl_edma3_chan *fsl_chan) else val |= EDMA_CH_SBR_WR; } + + if (fsl_chan->is_remote) + val &= ~(EDMA_CH_SBR_RD | EDMA_CH_SBR_WR); + writel(val, addr + EDMA_CH_SBR); val = readl(addr + EDMA_CH_CSR); @@ -686,7 +691,7 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, struct dma_chan *chan, *_chan; struct fsl_edma3_chan *fsl_chan; - if (dma_spec->args_count != 3) + if (dma_spec->args_count != 4) return NULL; mutex_lock(&fsl_edma3->fsl_edma3_mutex); @@ -701,6 +706,7 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, chan->device->privatecnt++; fsl_chan->priority = dma_spec->args[1]; fsl_chan->is_rxchan = dma_spec->args[2]; + fsl_chan->is_remote = dma_spec->args[3]; mutex_unlock(&fsl_edma3->fsl_edma3_mutex); return chan; } From 5dd7504280297a19c2812194961feddfff5f52fa Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Mon, 5 Jun 2017 11:37:57 +0800 Subject: [PATCH 26/52] MLK-15003-2: Document: fsl_edma_v3: update document update fsl_edma_v3 document for #dma-cell is changed one more cell is added, which is for local/remote access. Signed-off-by: Shengjiu Wang (cherry picked from commit 65543fb7fefbdb7df4cb60931a88f61507c5073f) --- Documentation/devicetree/bindings/dma/fsl-edma-v3.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt index 4c574937b729..322b2c64f396 100644 --- a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt +++ b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt @@ -14,11 +14,13 @@ Required properties: - interrupts : A list of interrupt-specifiers, each channel has one interrupt. - interrupt-names : Should contain: "edma-chan12-tx" - the channel12 transmission interrupt -- #dma-cells : Must be <3>. +- #dma-cells : Must be <4>. The 1st cell specifies the channel ID. The 2nd cell specifies the channel priority. The 3rd cell specifies the channel type like for transmit or receive: 0: transmit, 1: receive. + The 4th cell specifies the local access or remote access: + 0: local, 1: remote. See the SoC's reference manual for all the supported request sources. - dma-channels : Number of channels supported by the controller @@ -29,7 +31,7 @@ edma0: dma-controller@40018000 { <0x0 0x5a2d0000 0x0 0x10000>, /* channel13 UART0 tx */ <0x0 0x5a2e0000 0x0 0x10000>, /* channel14 UART1 rx */ <0x0 0x5a2f0000 0x0 0x10000>; /* channel15 UART1 tx */ - #dma-cells = <3>; + #dma-cells = <4>; dma-channels = <4>; interrupts = , , @@ -58,7 +60,7 @@ lpuart1: serial@5a070000 { assigned-clock-rates = <80000000>; power-domains = <&pd_dma_lpuart1>; dma-names = "tx","rx"; - dmas = <&edma0 15 0 0>, - <&edma0 14 0 1>; + dmas = <&edma0 15 0 0 0>, + <&edma0 14 0 1 0>; status = "disabled"; }; From b1b8c0ae9463fb493f87467f52a6865775f6c4b2 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 6 Jun 2017 16:56:49 +0800 Subject: [PATCH 27/52] MLK-15014 dma: fsl-edma-v3: clear DONE before E_SG enabled Below described in RM, otherwise, channel error status(CHa_ES) may be triggered: The user must clear the CHa_CSR[DONE] bit before writing the TCDa_CSR[MAJORELINK] or TCDa_CSR[ESG] bits. Signed-off-by: Robin Gong (cherry picked from commit c4164d0a15306174056c6ff423ba2408dd901fcf) --- drivers/dma/fsl-edma-v3.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index c20306cad822..f78284958efd 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -421,6 +421,11 @@ static void fsl_edma3_set_tcd_regs(struct fsl_edma3_chan *fsl_chan, writel(le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA); + /* Must clear CHa_CSR[DONE] bit before enable TCDa_CSR[ESG] */ + if ((EDMA_TCD_CSR_E_SG | le16_to_cpu(tcd->csr)) && + EDMA_CH_CSR_DONE | readl(addr + EDMA_CH_CSR)) + writel(EDMA_CH_CSR_DONE, addr + EDMA_CH_CSR); + writew(le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR); } From 4688ef9be7e93f4eb301f509cbcc22512e1bb862 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 4 Jul 2017 14:45:25 +0800 Subject: [PATCH 28/52] MLK-15330-1: dma: fsl-edma-v3: combine two cells into one For dual fifo case, fsl-edma-v3 need add another cell. It's not friendly for user and it's possible other cells maybe added to other use cases, so combine two cells into one now, and for some special use cases such as dual fifo property can directly be passed by one bit of cell3 rather than another cell. Signed-off-by: Robin Gong (cherry picked from commit 3ecd1b3382e2c746728842fb2c084fbb030eb5de) --- .../devicetree/bindings/dma/fsl-edma-v3.txt | 13 +++++++------ drivers/dma/fsl-edma-v3.c | 9 ++++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt index 322b2c64f396..9c84de7e402c 100644 --- a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt +++ b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt @@ -14,12 +14,13 @@ Required properties: - interrupts : A list of interrupt-specifiers, each channel has one interrupt. - interrupt-names : Should contain: "edma-chan12-tx" - the channel12 transmission interrupt -- #dma-cells : Must be <4>. +- #dma-cells : Must be <3>. The 1st cell specifies the channel ID. The 2nd cell specifies the channel priority. - The 3rd cell specifies the channel type like for transmit or receive: + The 3rd cell specifies the channel attributes which include below: + BIT(0): transmit or receive: 0: transmit, 1: receive. - The 4th cell specifies the local access or remote access: + BIT(1): local or remote access: 0: local, 1: remote. See the SoC's reference manual for all the supported request sources. - dma-channels : Number of channels supported by the controller @@ -31,7 +32,7 @@ edma0: dma-controller@40018000 { <0x0 0x5a2d0000 0x0 0x10000>, /* channel13 UART0 tx */ <0x0 0x5a2e0000 0x0 0x10000>, /* channel14 UART1 rx */ <0x0 0x5a2f0000 0x0 0x10000>; /* channel15 UART1 tx */ - #dma-cells = <4>; + #dma-cells = <3>; dma-channels = <4>; interrupts = , , @@ -60,7 +61,7 @@ lpuart1: serial@5a070000 { assigned-clock-rates = <80000000>; power-domains = <&pd_dma_lpuart1>; dma-names = "tx","rx"; - dmas = <&edma0 15 0 0 0>, - <&edma0 14 0 1 0>; + dmas = <&edma0 15 0 0>, + <&edma0 14 0 1>; status = "disabled"; }; diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index f78284958efd..20126a867dec 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -100,6 +100,9 @@ BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_16_BYTES)) +#define ARGS_RX BIT(0) +#define ARGS_REMOTE BIT(1) + struct fsl_edma3_hw_tcd { __le32 saddr; __le16 soff; @@ -696,7 +699,7 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, struct dma_chan *chan, *_chan; struct fsl_edma3_chan *fsl_chan; - if (dma_spec->args_count != 4) + if (dma_spec->args_count != 3) return NULL; mutex_lock(&fsl_edma3->fsl_edma3_mutex); @@ -710,8 +713,8 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, chan = dma_get_slave_channel(chan); chan->device->privatecnt++; fsl_chan->priority = dma_spec->args[1]; - fsl_chan->is_rxchan = dma_spec->args[2]; - fsl_chan->is_remote = dma_spec->args[3]; + fsl_chan->is_rxchan = dma_spec->args[2] & ARGS_RX; + fsl_chan->is_remote = dma_spec->args[2] & ARGS_REMOTE; mutex_unlock(&fsl_edma3->fsl_edma3_mutex); return chan; } From ec32139328591aae9c656a9ea6c470d139bca99c Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 4 Jul 2017 16:04:36 +0800 Subject: [PATCH 29/52] MLK-15330-3 dma: fsl-edma-v3: add dual fifo support There is Audio dual fifo cause that fill fifo one by one and loop back after every minor loop: -- fill the first 32bit width fifo -- fill the next 32bit width fifo -- +MLOFF signed offset after the above two FIFOs filled -- loop back to the first step to handle the next minor loop. Signed-off-by: Robin Gong (cherry picked from commit 5aa5e9663bb3a834444b75ea086bef8c37ecb636) --- .../devicetree/bindings/dma/fsl-edma-v3.txt | 2 ++ drivers/dma/fsl-edma-v3.c | 29 +++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt index 9c84de7e402c..222ccf8605b0 100644 --- a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt +++ b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt @@ -22,6 +22,8 @@ Required properties: 0: transmit, 1: receive. BIT(1): local or remote access: 0: local, 1: remote. + BIT(2): dualfifo case or not(only in Audio cyclic now): + 0: not dual fifo case, 1: dualfifo case. See the SoC's reference manual for all the supported request sources. - dma-channels : Number of channels supported by the controller diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index 20126a867dec..ef8280e0f2e1 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -78,6 +78,9 @@ #define EDMA_TCD_SOFF_SOFF(x) (x) #define EDMA_TCD_NBYTES_NBYTES(x) (x) +#define EDMA_TCD_NBYTES_MLOFF(x) (x << 10) +#define EDMA_TCD_NBYTES_DMLOE (1 << 30) +#define EDMA_TCD_NBYTES_SMLOE (1 << 31) #define EDMA_TCD_SLAST_SLAST(x) (x) #define EDMA_TCD_DADDR_DADDR(x) (x) #define EDMA_TCD_CITER_CITER(x) ((x) & 0x7FFF) @@ -102,6 +105,7 @@ #define ARGS_RX BIT(0) #define ARGS_REMOTE BIT(1) +#define ARGS_DFIFO BIT(2) struct fsl_edma3_hw_tcd { __le32 saddr; @@ -143,6 +147,7 @@ struct fsl_edma3_chan { int priority; int is_rxchan; int is_remote; + int is_dfifo; struct dma_pool *tcd_pool; u32 chn_real_count; char txirq_name[32]; @@ -454,6 +459,19 @@ void fsl_edma3_fill_tcd(struct fsl_edma3_chan *fsl_chan, tcd->soff = cpu_to_le16(EDMA_TCD_SOFF_SOFF(soff)); + if (fsl_chan->is_dfifo) { + /* set mloff as -8 */ + nbytes |= EDMA_TCD_NBYTES_MLOFF(-8); + /* enable DMLOE/SMLOE */ + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { + nbytes |= EDMA_TCD_NBYTES_DMLOE; + nbytes &= ~EDMA_TCD_NBYTES_SMLOE; + } else { + nbytes |= EDMA_TCD_NBYTES_SMLOE; + nbytes &= ~EDMA_TCD_NBYTES_DMLOE; + } + } + tcd->nbytes = cpu_to_le32(EDMA_TCD_NBYTES_NBYTES(nbytes)); tcd->slast = cpu_to_le32(EDMA_TCD_SLAST_SLAST(slast)); @@ -540,11 +558,17 @@ static struct dma_async_tx_descriptor *fsl_edma3_prep_dma_cyclic( src_addr = dma_buf_next; dst_addr = fsl_chan->fsc.dev_addr; soff = fsl_chan->fsc.addr_width; - doff = 0; + if (fsl_chan->is_dfifo) + doff = 4; + else + doff = 0; } else if (fsl_chan->fsc.dir == DMA_DEV_TO_MEM) { src_addr = fsl_chan->fsc.dev_addr; dst_addr = dma_buf_next; - soff = 0; + if (fsl_chan->is_dfifo) + soff = 4; + else + soff = 0; doff = fsl_chan->fsc.addr_width; } else { /* DMA_DEV_TO_DEV */ @@ -715,6 +739,7 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, fsl_chan->priority = dma_spec->args[1]; fsl_chan->is_rxchan = dma_spec->args[2] & ARGS_RX; fsl_chan->is_remote = dma_spec->args[2] & ARGS_REMOTE; + fsl_chan->is_dfifo = dma_spec->args[2] & ARGS_DFIFO; mutex_unlock(&fsl_edma3->fsl_edma3_mutex); return chan; } From 9d36acb8605f81c3c4d3b479c0c1403fa07e8f7a Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 30 Aug 2017 18:51:16 +0800 Subject: [PATCH 30/52] MLK-16327-1: dma: fsl-edma-v3: make exclusive channel name for all edma channels Since there are multi edmav3 instances on i.mx8, every edma channel name is better unique.But so far, all edma channel name is 'edma-channel(id)- tx',thus some edma channels which share the same channel id but different edma instance will show the same channel name in kernel and this is not friendly to debug in kernel. Now the edma channel name(interrupt-names property) is define in dts as below: "edmaX-chanX-Xx" | | |---> receive/transmit, r or t | |---> channel id, the max number is 32 |---> edma controller instance, 0, 1, 2,..etc and get below correct name with 'cat /proc/interrupts': 43: 0 0 0 0 GICv3 466 Level edma0-chan8-rx 44: 0 0 0 0 GICv3 467 Level edma0-chan9-tx 45: 79 0 0 0 GICv3 468 Level edma0-chan10-rx 46: 311 0 0 0 GICv3 469 Level edma0-chan11-tx 47: 0 0 0 0 GICv3 470 Level edma0-chan12-rx 48: 0 0 0 0 GICv3 471 Level edma0-chan13-tx 49: 0 0 0 0 GICv3 472 Level edma0-chan14-rx 50: 0 0 0 0 GICv3 473 Level edma0-chan15-tx 51: 0 0 0 0 GICv3 406 Level edma2-chan0-tx 52: 0 0 0 0 GICv3 407 Level edma2-chan1-tx 53: 0 0 0 0 GICv3 408 Level edma2-chan2-tx 54: 0 0 0 0 GICv3 409 Level edma2-chan3-tx 55: 0 0 0 0 GICv3 410 Level edma2-chan4-tx 56: 0 0 0 0 GICv3 411 Level edma2-chan5-tx 57: 0 0 0 0 GICv3 442 Level edma2-chan6-rx, edma2-chan7-tx Signed-off-by: Robin Gong Reviewed-by: Shengjiu Wang (cherry picked from commit af8e197a92c9c024ec4fbfcf543d744e81748773) --- .../devicetree/bindings/dma/fsl-edma-v3.txt | 12 ++++--- drivers/dma/fsl-edma-v3.c | 35 +++++++++++++++++-- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt index 222ccf8605b0..a8d05427f954 100644 --- a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt +++ b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt @@ -12,8 +12,12 @@ Required properties: - reg : Specifies base physical address(s) and size of the eDMA channel registers. Each eDMA channel has separated register's address and size. - interrupts : A list of interrupt-specifiers, each channel has one interrupt. -- interrupt-names : Should contain: - "edma-chan12-tx" - the channel12 transmission interrupt +- interrupt-names : Should contain below template: + "edmaX-chanX-Xx" + | | |---> receive/transmit, r or t + | |---> channel id, the max number is 32 + |---> edma controller instance, 0, 1, 2,..etc + - #dma-cells : Must be <3>. The 1st cell specifies the channel ID. The 2nd cell specifies the channel priority. @@ -40,8 +44,8 @@ edma0: dma-controller@40018000 { , , ; - interrupt-names = "edma-chan12-tx", "edma-chan13-tx", - "edma-chan14-tx", "edma-chan15-tx"; + interrupt-names = "edma0-chan12-rx", "edma0-chan13-tx", + "edma0-chan14-rx", "edma0-chan15-tx"; status = "okay"; }; diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index ef8280e0f2e1..28b13fb454e1 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -107,6 +107,10 @@ #define ARGS_REMOTE BIT(1) #define ARGS_DFIFO BIT(2) +/* channel name template define in dts */ +#define CHAN_PREFIX "edma0-chan" +#define CHAN_POSFIX "-tx" + struct fsl_edma3_hw_tcd { __le32 saddr; __le16 soff; @@ -806,7 +810,10 @@ static int fsl_edma3_probe(struct platform_device *pdev) INIT_LIST_HEAD(&fsl_edma3->dma_dev.channels); for (i = 0; i < fsl_edma3->n_chans; i++) { struct fsl_edma3_chan *fsl_chan = &fsl_edma3->chans[i]; - char *txirq_name = fsl_chan->txirq_name; + const char *txirq_name = fsl_chan->txirq_name; + char chanid[3], id_len = 0; + char *p = chanid; + unsigned long val; fsl_chan->edma3 = fsl_edma3; /* Get per channel membase */ @@ -819,7 +826,31 @@ static int fsl_edma3_probe(struct platform_device *pdev) * channel0:0x10000, channel1:0x20000... total 32 channels */ fsl_chan->hw_chanid = (res->start >> 16) & 0x1f; - sprintf(txirq_name, "edma-chan%d-tx", fsl_chan->hw_chanid); + + ret = of_property_read_string_index(np, "interrupt-names", i, + &txirq_name); + if (ret) { + dev_err(&pdev->dev, "read interrupt-names fail.\n"); + return ret; + } + /* Get channel id length from dts, one-digit or double-digit */ + id_len = strlen(txirq_name) - strlen(CHAN_PREFIX) - + strlen(CHAN_POSFIX); + if (id_len > 2) { + dev_err(&pdev->dev, "%s is edmaX-chanX-tx in dts?\n", + res->name); + return -EINVAL; + } + /* Grab channel id from txirq_name */ + strncpy(p, txirq_name + strlen(CHAN_PREFIX), id_len); + *(p + id_len) = '\0'; + + /* check if the channel id match well with hw_chanid */ + ret = kstrtoul(chanid, 0, &val); + if (ret || val != fsl_chan->hw_chanid) { + dev_err(&pdev->dev, "%s,wrong id?\n", txirq_name); + return -EINVAL; + } /* request channel irq */ fsl_chan->txirq = platform_get_irq_byname(pdev, txirq_name); From fe8cf28f2b0ba625c25ef1de2ef737b0c96f6ca4 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 13 Sep 2017 16:39:49 +0800 Subject: [PATCH 31/52] MLK-16437: dma: fsl-edma-v3: fix kernel crash while edma interrupt trigger after channel disabled edma interrupt may come after channel terminated, so should ignore interrupts, else kernel crash as below since fsl_chan->edesc set to NULL when terminate. 606.837306] Unable to handle kernel NULL pointer dereference at virtual address 00000060 [ 606.845411] pgd = ffff000009295000 [ 606.848814] [00000060] *pgd=00000008bfffe003[ 606.852906] , *pud=00000008bfffd003 , *pmd=0000000000000000[ 606.858395] [ 606.859885] Internal error: Oops: 96000006 1 PREEMPT SMP [ 606.865460] Modules linked in: [ 606.868522] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.9.11-03371-g9904ea0 #42 [ 606.875832] Hardware name: Freescale i.MX8QXP LPDDR4 ARM2 (DT) [ 606.881662] task: ffff000009120680 task.stack: ffff000009110000 [ 606.887588] PC is at fsl_edma3_tx_handler+0x50/0x150 Signed-off-by: Robin Gong Tested-by: Daniel Baluta (cherry picked from commit 625afad5a0900bc3e3288510f61647b1d891a5a4) --- drivers/dma/fsl-edma-v3.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index 28b13fb454e1..d0aabbcc3486 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -689,6 +689,11 @@ static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) writel(1, base_addr + EDMA_CH_INT); spin_lock(&fsl_chan->vchan.lock); + + /* Ignore this interrupt since channel has been disabled already */ + if (!fsl_chan->edesc) + return IRQ_HANDLED; + if (!fsl_chan->edesc->iscyclic) { fsl_edma3_get_realcnt(fsl_chan); list_del(&fsl_chan->edesc->vdesc.node); From f8e9e04f6dd3d1705fdcaf90f4ae325f778a1ae6 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 19 Sep 2017 11:36:58 +0800 Subject: [PATCH 32/52] MLK-16482: dma: fsl-edma-v3: Fix RCU issue while playing Audio That's caused by commit 593034f1b908 ("MLK-16437: dma: fsl-edma-v3: fix kernel crash while edma interrupt trigger after channel disabled"). Because fsl_chan->vchan.lock will be hold always and trigger RCU report as below: 1571.3 Playing WAVE '/mnt/nfs/vte_mx82/../test_stream/esai_stream/48k16bit-six.wav' : Signed 16 bit Little Endian, Rate 48000 Hz, Channels 6 1571.5 [ 4642.698771] INFO: rcu_preempt detected stalls on CPUs/tasks: 1571.6 [ 4642.704443] 0-...: (1 GPs behind) idle=2c5/140000000000000/0 softirq=155373/155374 fqs=2541 1571.7 [ 4642.712967] (detected by 2, t=5252 jiffies, g=104259, c=104258, q=22) 1571.8 [ 4642.719501] Task dump for CPU 0: 1571.9 [ 4642.722724] aplay R running task 0 15723 15721 0x00000202 1571.10 [ 4642.729786] Call trace: 1571.11 [ 4642.732239] [] __switch_to+0x8c/0xa0 1571.12 [ 4642.737379] [] dma_chan_put+0x70/0xa0 1571.13 [ 4642.742603] [] dma_release_channel+0x34/0xa0 1571.14 [ 4642.748435] [] fsl_asrc_dma_hw_free+0x38/0x50 1571.15 [ 4642.754358] [] soc_pcm_hw_free+0x110/0x1a8 1571.16 [ 4642.760013] [] dpcm_fe_dai_hw_free+0x6c/0xe0 1571.17 [ 4642.765844] [] snd_pcm_common_ioctl1+0xb40/0xce0 1571.18 [ 4642.772028] [] snd_pcm_playback_ioctl1+0x1dc/0x310 1571.19 [ 4642.778378] [] snd_pcm_playback_ioctl+0x28/0x40 1571.20 [ 4642.784470] [] do_vfs_ioctl+0xa4/0x748 1571.21 [ 4642.789784] [] SyS_ioctl+0x8c/0xa0 1571.22 [ 4642.794745] [] __sys_trace_return+0x0/0x4 1571.23 [ 4705.718740] INFO: rcu_preempt detected stalls on CPUs/tasks: 1571.24 [ 4705.724420] 0-...: (1 GPs behind) idle=2c5/140000000000000/0 softirq=155373/155374 fqs=10407 1571.25 [ 4705.733030] (detected by 1, t=21010 jiffies, g=104259, c=104258, q=119) Signed-off-by: Robin Gong Reported-by: Jason Liu Reviewed-by: Daniel Baluta Fixes: 593034f1b908 ("MLK-16437: dma: fsl-edma-v3: fix kernel crash while edma interrupt trigger after channel disabled"). (cherry picked from commit e62e8707154f47e168fcfd148e97be4e2f991898) --- drivers/dma/fsl-edma-v3.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index d0aabbcc3486..a0495782e281 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -692,7 +692,7 @@ static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) /* Ignore this interrupt since channel has been disabled already */ if (!fsl_chan->edesc) - return IRQ_HANDLED; + goto irq_handled; if (!fsl_chan->edesc->iscyclic) { fsl_edma3_get_realcnt(fsl_chan); @@ -706,7 +706,7 @@ static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) if (!fsl_chan->edesc) fsl_edma3_xfer_desc(fsl_chan); - +irq_handled: spin_unlock(&fsl_chan->vchan.lock); return IRQ_HANDLED; From e2c42307075aef321d176bc3414d9351087481a8 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Mon, 4 Dec 2017 15:35:09 +0800 Subject: [PATCH 33/52] MLK-17094 dma: fsl-edma-v3: add suspend/resume to restore back channel registers Add suspend to save channel registers and resume to restore them back since edmav3 may powered off in suspend. Signed-off-by: Robin Gong (cherry picked from commit 7eda1ae538ec7e7c0f993b3ea91805459f3dedd3) --- drivers/dma/fsl-edma-v3.c | 86 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index a0495782e281..6e7c5030abf6 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -111,6 +111,11 @@ #define CHAN_PREFIX "edma0-chan" #define CHAN_POSFIX "-tx" +enum fsl_edma3_pm_state { + RUNNING = 0, + SUSPENDED, +}; + struct fsl_edma3_hw_tcd { __le32 saddr; __le16 soff; @@ -142,6 +147,8 @@ struct fsl_edma3_slave_config { struct fsl_edma3_chan { struct virt_dma_chan vchan; enum dma_status status; + enum fsl_edma3_pm_state pm_state; + bool idle; struct fsl_edma3_engine *edma3; struct fsl_edma3_desc *edesc; struct fsl_edma3_slave_config fsc; @@ -165,11 +172,18 @@ struct fsl_edma3_desc { struct fsl_edma3_sw_tcd tcd[]; }; +struct fsl_edma3_reg_save { + u32 csr; + u32 sbr; +}; + struct fsl_edma3_engine { struct dma_device dma_dev; struct mutex fsl_edma3_mutex; u32 n_chans; int errirq; + #define MAX_CHAN_NUM 32 + struct fsl_edma3_reg_save edma_regs[MAX_CHAN_NUM]; bool swap; /* remote/local swapped on Audio edma */ struct fsl_edma3_chan chans[]; }; @@ -266,6 +280,7 @@ static int fsl_edma3_terminate_all(struct dma_chan *chan) spin_lock_irqsave(&fsl_chan->vchan.lock, flags); fsl_edma3_disable_request(fsl_chan); fsl_chan->edesc = NULL; + fsl_chan->idle = true; vchan_get_all_descriptors(&fsl_chan->vchan, &head); spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); vchan_dma_desc_free_list(&fsl_chan->vchan, &head); @@ -281,6 +296,7 @@ static int fsl_edma3_pause(struct dma_chan *chan) if (fsl_chan->edesc) { fsl_edma3_disable_request(fsl_chan); fsl_chan->status = DMA_PAUSED; + fsl_chan->idle = true; } spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); return 0; @@ -295,6 +311,7 @@ static int fsl_edma3_resume(struct dma_chan *chan) if (fsl_chan->edesc) { fsl_edma3_enable_request(fsl_chan); fsl_chan->status = DMA_IN_PROGRESS; + fsl_chan->idle = false; } spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); return 0; @@ -664,6 +681,7 @@ static void fsl_edma3_xfer_desc(struct fsl_edma3_chan *fsl_chan) fsl_edma3_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); fsl_edma3_enable_request(fsl_chan); fsl_chan->status = DMA_IN_PROGRESS; + fsl_chan->idle = false; } static size_t fsl_edma3_desc_residue(struct fsl_edma3_chan *fsl_chan, @@ -700,6 +718,7 @@ static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) vchan_cookie_complete(&fsl_chan->edesc->vdesc); fsl_chan->edesc = NULL; fsl_chan->status = DMA_COMPLETE; + fsl_chan->idle = true; } else { vchan_cyclic_callback(&fsl_chan->edesc->vdesc); } @@ -719,6 +738,12 @@ static void fsl_edma3_issue_pending(struct dma_chan *chan) spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + if (unlikely(fsl_chan->pm_state != RUNNING)) { + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + /* cannot submit due to suspend */ + return; + } + if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) fsl_edma3_xfer_desc(fsl_chan); @@ -821,6 +846,8 @@ static int fsl_edma3_probe(struct platform_device *pdev) unsigned long val; fsl_chan->edma3 = fsl_edma3; + fsl_chan->pm_state = RUNNING; + fsl_chan->idle = true; /* Get per channel membase */ res = platform_get_resource(pdev, IORESOURCE_MEM, i); fsl_chan->membase = devm_ioremap_resource(&pdev->dev, res); @@ -932,6 +959,64 @@ static int fsl_edma3_remove(struct platform_device *pdev) return 0; } +static int fsl_edma3_suspend_late(struct device *dev) +{ + struct fsl_edma3_engine *fsl_edma = dev_get_drvdata(dev); + struct fsl_edma3_chan *fsl_chan; + unsigned long flags; + void __iomem *addr; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + fsl_chan = &fsl_edma->chans[i]; + addr = fsl_chan->membase; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + fsl_edma->edma_regs[i].csr = readl(addr + EDMA_CH_CSR); + fsl_edma->edma_regs[i].sbr = readl(addr + EDMA_CH_SBR); + /* Make sure chan is idle or will force disable. */ + if (unlikely(!fsl_chan->idle)) { + dev_warn(dev, "WARN: There is non-idle channel."); + fsl_edma3_disable_request(fsl_chan); + } + fsl_chan->pm_state = SUSPENDED; + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + } + + return 0; +} + +static int fsl_edma3_resume_early(struct device *dev) +{ + struct fsl_edma3_engine *fsl_edma = dev_get_drvdata(dev); + struct fsl_edma3_chan *fsl_chan; + void __iomem *addr; + unsigned long flags; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + fsl_chan = &fsl_edma->chans[i]; + addr = fsl_chan->membase; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + writel(fsl_edma->edma_regs[i].csr, addr + EDMA_CH_CSR); + writel(fsl_edma->edma_regs[i].sbr, addr + EDMA_CH_SBR); + /* restore tcd if this channel not terminated before suspend */ + if (fsl_chan->edesc) + fsl_edma3_set_tcd_regs(fsl_chan, + fsl_chan->edesc->tcd[0].vtcd); + fsl_chan->pm_state = RUNNING; + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + } + + return 0; +} + +static const struct dev_pm_ops fsl_edma3_pm_ops = { + .suspend_late = fsl_edma3_suspend_late, + .resume_early = fsl_edma3_resume_early, +}; + static const struct of_device_id fsl_edma3_dt_ids[] = { { .compatible = "fsl,imx8qm-edma", }, { .compatible = "fsl,imx8qm-adma", }, @@ -943,6 +1028,7 @@ static struct platform_driver fsl_edma3_driver = { .driver = { .name = "fsl-edma-v3", .of_match_table = fsl_edma3_dt_ids, + .pm = &fsl_edma3_pm_ops, }, .probe = fsl_edma3_probe, .remove = fsl_edma3_remove, From e3890608e2734bb63ba652be0df436c1d4057eb6 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 13 Mar 2018 02:03:09 +0800 Subject: [PATCH 34/52] MLK-17782 dma: fsl-edma-v3: fix issue reported by Coverity Fix below issue reported by Coverity, actually, don't need this condition check here, remove it. CID undefined (#1 of 1): Wrong operator used (CONSTANT_EXPRESSION_RESULT)operator_confusion: (16UL /* 1UL << 4 */) | (__u16)(__le16)tcd->csr is always 1/true regardless of the values of its operand. This occurs as the logical first operand of "&&". Signed-off-by: Robin Gong Reviewed-by: Shengjiu Wang (cherry picked from commit ab942110975cadcde57ab1110df03f526bd3fec5) --- drivers/dma/fsl-edma-v3.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index 6e7c5030abf6..7f5e37c7190d 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -1,7 +1,7 @@ /* * drivers/dma/fsl-edma3-v3.c * - * Copyright 2017 NXP . + * Copyright 2017-2018 NXP . * * Driver for the Freescale eDMA engine v3. This driver based on fsl-edma3.c * but changed to meet the IP change on i.MX8QM: every dma channel is specific @@ -451,9 +451,7 @@ static void fsl_edma3_set_tcd_regs(struct fsl_edma3_chan *fsl_chan, writel(le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA); /* Must clear CHa_CSR[DONE] bit before enable TCDa_CSR[ESG] */ - if ((EDMA_TCD_CSR_E_SG | le16_to_cpu(tcd->csr)) && - EDMA_CH_CSR_DONE | readl(addr + EDMA_CH_CSR)) - writel(EDMA_CH_CSR_DONE, addr + EDMA_CH_CSR); + writel(readl(addr + EDMA_CH_CSR), addr + EDMA_CH_CSR); writew(le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR); } From 9dc7c227d410b058a6714665acb3f259247175a7 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Thu, 10 May 2018 01:02:01 +0800 Subject: [PATCH 35/52] MLK-18248: dma: fsl-edma-v3: avoid touch unused edma channel Avoid touch unused edma channel register in susped/resume, otherwise, kernel crash if XRDC enabled in scfw. Signed-off-by: Robin Gong Acked-by: Fugang Duan (cherry picked from commit aa221c4aba34c6ce1ce5f561fa073bb8297cc0ff) --- drivers/dma/fsl-edma-v3.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index 7f5e37c7190d..734c469d0cb2 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -149,6 +149,7 @@ struct fsl_edma3_chan { enum dma_status status; enum fsl_edma3_pm_state pm_state; bool idle; + bool used; struct fsl_edma3_engine *edma3; struct fsl_edma3_desc *edesc; struct fsl_edma3_slave_config fsc; @@ -226,6 +227,8 @@ static void fsl_edma3_enable_request(struct fsl_edma3_chan *fsl_chan) val |= EDMA_CH_CSR_ERQ; writel(val, addr + EDMA_CH_CSR); + + fsl_chan->used = true; } static void fsl_edma3_disable_request(struct fsl_edma3_chan *fsl_chan) @@ -281,6 +284,7 @@ static int fsl_edma3_terminate_all(struct dma_chan *chan) fsl_edma3_disable_request(fsl_chan); fsl_chan->edesc = NULL; fsl_chan->idle = true; + fsl_chan->used = false; vchan_get_all_descriptors(&fsl_chan->vchan, &head); spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); vchan_dma_desc_free_list(&fsl_chan->vchan, &head); @@ -805,6 +809,7 @@ static void fsl_edma3_free_chan_resources(struct dma_chan *chan) vchan_dma_desc_free_list(&fsl_chan->vchan, &head); dma_pool_destroy(fsl_chan->tcd_pool); fsl_chan->tcd_pool = NULL; + fsl_chan->used = false; } static int fsl_edma3_probe(struct platform_device *pdev) @@ -900,6 +905,7 @@ static int fsl_edma3_probe(struct platform_device *pdev) fsl_chan->vchan.desc_free = fsl_edma3_free_desc; vchan_init(&fsl_chan->vchan, &fsl_edma3->dma_dev); + fsl_chan->used = false; } mutex_init(&fsl_edma3->fsl_edma3_mutex); @@ -969,6 +975,8 @@ static int fsl_edma3_suspend_late(struct device *dev) fsl_chan = &fsl_edma->chans[i]; addr = fsl_chan->membase; + if (!fsl_chan->used) + continue; spin_lock_irqsave(&fsl_chan->vchan.lock, flags); fsl_edma->edma_regs[i].csr = readl(addr + EDMA_CH_CSR); fsl_edma->edma_regs[i].sbr = readl(addr + EDMA_CH_SBR); @@ -996,6 +1004,9 @@ static int fsl_edma3_resume_early(struct device *dev) fsl_chan = &fsl_edma->chans[i]; addr = fsl_chan->membase; + if (!fsl_chan->used) + continue; + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); writel(fsl_edma->edma_regs[i].csr, addr + EDMA_CH_CSR); writel(fsl_edma->edma_regs[i].sbr, addr + EDMA_CH_SBR); From 0b02c5f7d53c1cddc94d85e68336b3a865151283 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 24 Jul 2018 22:12:24 +0800 Subject: [PATCH 36/52] MLK-19022-2: dmaengine: fsl-edma-v3: add device_synchronize Add device_synchronize for edma driver, since some driver such as Audio need it to make sure dma done callback never come out after resource related with dma channel free-ed by Audio driver. Android team report such issue on MA-12087. Signed-off-by: Robin Gong (cherry picked from commit 483519c063b08fc1ce0dd297b6c186799cf639d6) (cherry picked from commit 29ab274aca01ef8f5fc70e8c0a6d43a5bdb3c689) --- drivers/dma/fsl-edma-v3.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index 734c469d0cb2..a7b57f4ef503 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -812,6 +812,13 @@ static void fsl_edma3_free_chan_resources(struct dma_chan *chan) fsl_chan->used = false; } +static void fsl_edma3_synchronize(struct dma_chan *chan) +{ + struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + + vchan_synchronize(&fsl_chan->vchan); +} + static int fsl_edma3_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -927,6 +934,7 @@ static int fsl_edma3_probe(struct platform_device *pdev) fsl_edma3->dma_dev.device_resume = fsl_edma3_resume; fsl_edma3->dma_dev.device_terminate_all = fsl_edma3_terminate_all; fsl_edma3->dma_dev.device_issue_pending = fsl_edma3_issue_pending; + fsl_edma3->dma_dev.device_synchronize = fsl_edma3_synchronize; fsl_edma3->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS; fsl_edma3->dma_dev.dst_addr_widths = FSL_EDMA_BUSWIDTHS; From 9ff1f271a0e2518ee385638bf1ee03b69d6874be Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 16 Oct 2018 01:06:29 +0800 Subject: [PATCH 37/52] MLK-19931-1: dmaengine: fsl-edma-v3: fix potential kernel crash in cyclic There is one potential race condition in virt-dma framework as below: terminate dma channel after the last dma done interrupt, but before vchan_complete tasklet scheduled, thus the free-ed 'vd' (free in fsl_edma3_terminate_all) maybe still be touched in vchan_complete() which cause NULL pointer crash. Kernel community noticed this issue and fix it at virt-dma level: https://patchwork.kernel.org/patch/10057791/. To avoid backport too much patches, set 'vc->cyclic = NULL' in terminate dma channel interfaces to fix such issue easily. Signed-off-by: Robin Gong Acked-by: Fugang Duan (cherry picked from commit 18c9083826400a2ef731496391a0b5e71d461a5f) --- drivers/dma/fsl-edma-v3.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index a7b57f4ef503..5a5674f4c292 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -285,6 +285,7 @@ static int fsl_edma3_terminate_all(struct dma_chan *chan) fsl_chan->edesc = NULL; fsl_chan->idle = true; fsl_chan->used = false; + fsl_chan->vchan.cyclic = NULL; vchan_get_all_descriptors(&fsl_chan->vchan, &head); spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); vchan_dma_desc_free_list(&fsl_chan->vchan, &head); From 1432768dff266a12b65fe00b1e3d4d12cd1a746a Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 7 Nov 2018 22:19:07 +0800 Subject: [PATCH 38/52] MLK-20205-1: dmaengine: fsl-edma-v3: fix NULL pointer dereference Fix 'null pointer dereferences issue' reported by coverity(CID-1477441). Signed-off-by: Robin Gong Reviewed-by: Anson Huang (cherry picked from commit 5343c0018af0af2eb3bb90c5a75e765b851a2c12) --- drivers/dma/fsl-edma-v3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index 5a5674f4c292..028b25968195 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -419,7 +419,7 @@ static enum dma_status fsl_edma3_tx_status(struct dma_chan *chan, if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie) txstate->residue = fsl_edma3_desc_residue(fsl_chan, vdesc, true); - else if (vdesc) + else if (fsl_chan->edesc && vdesc) txstate->residue = fsl_edma3_desc_residue(fsl_chan, vdesc, false); else From 2432fcf6040eee05fa8eed8d9a31723f297a9624 Mon Sep 17 00:00:00 2001 From: Richard Zhu Date: Fri, 25 Jan 2019 18:02:22 +0800 Subject: [PATCH 39/52] dma: imx: add the 32bit dma limitation Since the imx8qm/qxp hsio only supports up to 32bit dma capability. Add the 32bit dma limitation into dma binding document. Signed-off-by: Richard Zhu --- .../devicetree/bindings/dma/fsl-imx-dma.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Documentation/devicetree/bindings/dma/fsl-imx-dma.txt b/Documentation/devicetree/bindings/dma/fsl-imx-dma.txt index 7bd8847d6394..d10a9c1d2ce6 100644 --- a/Documentation/devicetree/bindings/dma/fsl-imx-dma.txt +++ b/Documentation/devicetree/bindings/dma/fsl-imx-dma.txt @@ -16,6 +16,21 @@ Optional properties: - #dma-channels : Number of DMA channels supported. Should be 16. - #dma-requests : Number of DMA requests supported. +* DMA capability limitation + +Specify the DMA capability limitations. +For example, some SoCs only support up to 32bit DMA capability, although +they are 64bit SoCs. + +- only-dma-mask32: 1 means that the SoCs only suppot up to 32bit DMA + capability. + +Example: + dma_cap: dma_cap { + compatible = "dma-capability"; + only-dma-mask32 = <1>; + }; + Example: dma: dma@10001000 { From 5b2c91a75ee4fc5248dbb4e6df334aeddc2d4716 Mon Sep 17 00:00:00 2001 From: Fugang Duan Date: Fri, 9 Aug 2019 15:14:08 +0800 Subject: [PATCH 40/52] dmaengine: fsl-edma: calculate the real count for slave sg Calculate the rela count for current slave sg after eDMA stop. Signed-off-by: Fugang Duan --- drivers/dma/fsl-edma-common.c | 11 ++++++++++- drivers/dma/fsl-edma-common.h | 2 ++ drivers/dma/fsl-edma.c | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index b1a7ca91701a..1acf8d67f057 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -305,6 +305,11 @@ static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, return len; } +void fsl_edma_get_realcnt(struct fsl_edma_chan *fsl_chan) +{ + fsl_chan->chn_real_count = fsl_edma_desc_residue(fsl_chan, NULL, true); +} + enum dma_status fsl_edma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { @@ -314,8 +319,12 @@ enum dma_status fsl_edma_tx_status(struct dma_chan *chan, unsigned long flags; status = dma_cookie_status(chan, cookie, txstate); - if (status == DMA_COMPLETE) + if (status == DMA_COMPLETE) { + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + txstate->residue = fsl_chan->chn_real_count; + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); return status; + } if (!txstate) return fsl_chan->status; diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h index 5eaa2902ed39..1b4fc023eb42 100644 --- a/drivers/dma/fsl-edma-common.h +++ b/drivers/dma/fsl-edma-common.h @@ -126,6 +126,7 @@ struct fsl_edma_chan { u32 dma_dev_size; enum dma_data_direction dma_dir; char chan_name[16]; + u32 chn_real_count; }; struct fsl_edma_desc { @@ -229,6 +230,7 @@ int fsl_edma_pause(struct dma_chan *chan); int fsl_edma_resume(struct dma_chan *chan); int fsl_edma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg); +void fsl_edma_get_realcnt(struct fsl_edma_chan *fsl_chan); enum dma_status fsl_edma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate); struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c index b626c06ac2e0..e28bd0275f9f 100644 --- a/drivers/dma/fsl-edma.c +++ b/drivers/dma/fsl-edma.c @@ -46,6 +46,7 @@ static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) spin_lock(&fsl_chan->vchan.lock); if (!fsl_chan->edesc->iscyclic) { + fsl_edma_get_realcnt(fsl_chan); list_del(&fsl_chan->edesc->vdesc.node); vchan_cookie_complete(&fsl_chan->edesc->vdesc); fsl_chan->edesc = NULL; From f55fd6f075cd2adb5ba7fcc2ed8e7acb65192c08 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 23 Oct 2019 00:33:42 +0800 Subject: [PATCH 41/52] MLK-22798-1: dmaengine: fsl-edma-v3: do not enable interrupt in dev_2_dev Do not enable interrupt in dev_2_dev with cyclic case, since in such case no any interrupt needed. Otherwise many interrupt will come in every 64 words transfered in ASRC case, which cause heavy system loading. Signed-off-by: Robin Gong Reviewed-by: Shengjiu Wang (cherry picked from commit f0a3172e1ceb04c46377160486ad7dc6ee022850) --- drivers/dma/fsl-edma-v3.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index 028b25968195..539898607626 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -560,6 +560,7 @@ static struct dma_async_tx_descriptor *fsl_edma3_prep_dma_cyclic( int sg_len, i; u32 src_addr, dst_addr, last_sg, nbytes; u16 soff, doff, iter; + bool major_int = true; sg_len = buf_len / period_len; fsl_desc = fsl_edma3_alloc_desc(fsl_chan, sg_len); @@ -600,11 +601,12 @@ static struct dma_async_tx_descriptor *fsl_edma3_prep_dma_cyclic( dst_addr = fsl_chan->fsc.dev_addr; soff = 0; doff = 0; + major_int = false; } fsl_edma3_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, dst_addr, fsl_chan->fsc.attr, soff, nbytes, 0, - iter, iter, doff, last_sg, true, false, true); + iter, iter, doff, last_sg, major_int, false, true); dma_buf_next += period_len; } From e8e9c9bc996b69ba97eda3e5a10207cdefb798b2 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Thu, 11 Apr 2019 14:36:37 +0800 Subject: [PATCH 42/52] MLK-21443: dmaengine: fsl-edma-v3: clear pending irq before request irq edma interrupt maybe happened during reboot or watchdog reset, meanwhile gic never power down on i.mx8QM/QXP, thus the unexpect irq will come in once edma driver request irq at probe phase. Unfortunately, at that time that edma channel's power domain which power-up by customer driver such as audio/uart driver may not be ready, so kernel panic triggered once touch such edma registers which still not power up in interrupt handler. Move request irq from probe to alloc dma channel so that edma channel's power domain has already been powered, besides, clear meaningless interrupt before request irq. Signed-off-by: Robin Gong Acked-by: Fugang Duan (cherry picked from commit 0a0d8f8b944094342fda18f23f3ac13b8a73871d) --- drivers/dma/fsl-edma-v3.c | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index 539898607626..d74e4d46b907 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -162,7 +162,8 @@ struct fsl_edma3_chan { int is_dfifo; struct dma_pool *tcd_pool; u32 chn_real_count; - char txirq_name[32]; + char txirq_name[32]; + struct platform_device *pdev; }; struct fsl_edma3_desc { @@ -180,6 +181,7 @@ struct fsl_edma3_reg_save { struct fsl_edma3_engine { struct dma_device dma_dev; + unsigned long irqflag; struct mutex fsl_edma3_mutex; u32 n_chans; int errirq; @@ -790,10 +792,23 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, static int fsl_edma3_alloc_chan_resources(struct dma_chan *chan) { struct fsl_edma3_chan *fsl_chan = to_fsl_edma3_chan(chan); + struct platform_device *pdev = fsl_chan->pdev; + int ret; fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, sizeof(struct fsl_edma3_hw_tcd), 32, 0); + /* clear meaningless pending irq anyway */ + writel(1, fsl_chan->membase + EDMA_CH_INT); + ret = devm_request_irq(&pdev->dev, fsl_chan->txirq, + fsl_edma3_tx_handler, fsl_chan->edma3->irqflag, + fsl_chan->txirq_name, fsl_chan); + if (ret) { + dev_err(&pdev->dev, "Can't register %s IRQ.\n", + fsl_chan->txirq_name); + return ret; + } + return 0; } @@ -803,6 +818,8 @@ static void fsl_edma3_free_chan_resources(struct dma_chan *chan) unsigned long flags; LIST_HEAD(head); + devm_free_irq(&fsl_chan->pdev->dev, fsl_chan->txirq, fsl_chan); + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); fsl_edma3_disable_request(fsl_chan); fsl_chan->edesc = NULL; @@ -830,7 +847,6 @@ static int fsl_edma3_probe(struct platform_device *pdev) struct resource *res; int len, chans; int ret, i; - unsigned long irqflag = 0; ret = of_property_read_u32(np, "dma-channels", &chans); if (ret) { @@ -845,7 +861,7 @@ static int fsl_edma3_probe(struct platform_device *pdev) /* Audio edma rx/tx channel shared interrupt */ if (of_property_read_bool(np, "shared-interrupt")) - irqflag = IRQF_SHARED; + fsl_edma3->irqflag = IRQF_SHARED; fsl_edma3->swap = of_device_is_compatible(np, "fsl,imx8qm-adma"); fsl_edma3->n_chans = chans; @@ -853,12 +869,13 @@ static int fsl_edma3_probe(struct platform_device *pdev) INIT_LIST_HEAD(&fsl_edma3->dma_dev.channels); for (i = 0; i < fsl_edma3->n_chans; i++) { struct fsl_edma3_chan *fsl_chan = &fsl_edma3->chans[i]; - const char *txirq_name = fsl_chan->txirq_name; + const char *txirq_name; char chanid[3], id_len = 0; char *p = chanid; unsigned long val; fsl_chan->edma3 = fsl_edma3; + fsl_chan->pdev = pdev; fsl_chan->pm_state = RUNNING; fsl_chan->idle = true; /* Get per channel membase */ @@ -904,14 +921,7 @@ static int fsl_edma3_probe(struct platform_device *pdev) return fsl_chan->txirq; } - ret = devm_request_irq(&pdev->dev, fsl_chan->txirq, - fsl_edma3_tx_handler, irqflag, txirq_name, - fsl_chan); - if (ret) { - dev_err(&pdev->dev, "Can't register %s IRQ.\n", - txirq_name); - return ret; - } + memcpy(fsl_chan->txirq_name, txirq_name, strlen(txirq_name)); fsl_chan->vchan.desc_free = fsl_edma3_free_desc; vchan_init(&fsl_chan->vchan, &fsl_edma3->dma_dev); From cfe6412faa5233e3e4d43ba9c8b3c56260bb3d7a Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 17 Apr 2019 17:05:42 +0800 Subject: [PATCH 43/52] MLK-22284-1 dmaengine: fsl-edma-v3: add power domains for each channel Add power domains for each dma channel so that edma channel could know the power state of every dma channel anytime and clear easily unexpected interrupt which triggered before the last partition reset. Signed-off-by: Robin Gong Reviewed-by: S.j. Wang (cherry picked from commit 0b6da46b7bdb2284e24757d48466268b9feb5b7c) --- .../devicetree/bindings/dma/fsl-edma-v3.txt | 11 +++- drivers/dma/fsl-edma-v3.c | 58 ++++++++++++++++++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt index a8d05427f954..8fe82ce63632 100644 --- a/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt +++ b/Documentation/devicetree/bindings/dma/fsl-edma-v3.txt @@ -30,6 +30,8 @@ Required properties: 0: not dual fifo case, 1: dualfifo case. See the SoC's reference manual for all the supported request sources. - dma-channels : Number of channels supported by the controller +- power-domains: Power domains for edma channel used. +- power-domain-names: Power domains name for edma channel used. Examples: edma0: dma-controller@40018000 { @@ -46,6 +48,12 @@ edma0: dma-controller@40018000 { ; interrupt-names = "edma0-chan12-rx", "edma0-chan13-tx", "edma0-chan14-rx", "edma0-chan15-tx"; + power-domains = <&pd IMX_SC_R_DMA_0_CH12>, + <&pd IMX_SC_R_DMA_0_CH13>, + <&pd IMX_SC_R_DMA_0_CH14>, + <&pd IMX_SC_R_DMA_0_CH15>; + power-domain-names = "edma0-chan12", "edma0-chan13", + "edma0-chan14", "edma0-chan15"; status = "okay"; }; @@ -65,7 +73,8 @@ lpuart1: serial@5a070000 { clock-names = "ipg"; assigned-clock-names = <&clk IMX8QM_UART1_CLK>; assigned-clock-rates = <80000000>; - power-domains = <&pd_dma_lpuart1>; + power-domains = <&pd IMX_SC_R_UART_1>, + power-domain-names = "uart"; dma-names = "tx","rx"; dmas = <&edma0 15 0 0>, <&edma0 14 0 1>; diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index d74e4d46b907..bc6192bb234e 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include "virt-dma.h" @@ -164,6 +166,7 @@ struct fsl_edma3_chan { u32 chn_real_count; char txirq_name[32]; struct platform_device *pdev; + struct device *dev; }; struct fsl_edma3_desc { @@ -798,8 +801,10 @@ static int fsl_edma3_alloc_chan_resources(struct dma_chan *chan) fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, sizeof(struct fsl_edma3_hw_tcd), 32, 0); + pm_runtime_get_sync(fsl_chan->dev); /* clear meaningless pending irq anyway */ writel(1, fsl_chan->membase + EDMA_CH_INT); + ret = devm_request_irq(&pdev->dev, fsl_chan->txirq, fsl_edma3_tx_handler, fsl_chan->edma3->irqflag, fsl_chan->txirq_name, fsl_chan); @@ -830,6 +835,7 @@ static void fsl_edma3_free_chan_resources(struct dma_chan *chan) dma_pool_destroy(fsl_chan->tcd_pool); fsl_chan->tcd_pool = NULL; fsl_chan->used = false; + pm_runtime_put_sync(fsl_chan->dev); } static void fsl_edma3_synchronize(struct dma_chan *chan) @@ -839,6 +845,37 @@ static void fsl_edma3_synchronize(struct dma_chan *chan) vchan_synchronize(&fsl_chan->vchan); } +static struct device *fsl_edma3_attach_pd(struct device *dev, + struct device_node *np, int index) +{ + const char *domn = "edma0-chan01"; + struct device *pd_chan; + struct device_link *link; + int ret; + + ret = of_property_read_string_index(np, "power-domain-names", index, + &domn); + if (ret) { + dev_err(dev, "parse power-domain-names error.(%d)\n", ret); + return NULL; + } + + pd_chan = dev_pm_domain_attach_by_name(dev, domn); + if (!pd_chan) + return NULL; + + link = device_link_add(dev, pd_chan, DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (IS_ERR(link)) { + dev_err(dev, "Failed to add device_link to %s: %ld\n", domn, + PTR_ERR(link)); + return NULL; + } + + return pd_chan; +} + static int fsl_edma3_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -962,6 +999,22 @@ static int fsl_edma3_probe(struct platform_device *pdev) dev_err(&pdev->dev, "Can't register Freescale eDMA engine.\n"); return ret; } + /* Attach power domains from dts for each dma chanel device */ + for (i = 0; i < fsl_edma3->n_chans; i++) { + struct fsl_edma3_chan *fsl_chan = &fsl_edma3->chans[i]; + struct device *dev; + + dev = fsl_edma3_attach_pd(&pdev->dev, np, i); + if (!dev) { + dev_err(dev, "edma channel attach failed.\n"); + return -EINVAL; + } + + fsl_chan->dev = dev; + /* clear meaningless pending irq anyway */ + writel(1, fsl_chan->membase + EDMA_CH_INT); + pm_runtime_put_sync(dev); + } ret = of_dma_controller_register(np, fsl_edma3_xlate, fsl_edma3); if (ret) { @@ -970,6 +1023,9 @@ static int fsl_edma3_probe(struct platform_device *pdev) return ret; } + pm_runtime_dont_use_autosuspend(&pdev->dev); + pm_runtime_enable(&pdev->dev); + return 0; } @@ -1068,7 +1124,7 @@ static int __init fsl_edma3_init(void) { return platform_driver_register(&fsl_edma3_driver); } -subsys_initcall(fsl_edma3_init); +fs_initcall(fsl_edma3_init); static void __exit fsl_edma3_exit(void) { From 7d7fb3da1ca4fd89cf594bb0719aaf6eb6f4c98f Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Wed, 8 May 2019 00:13:40 +0800 Subject: [PATCH 44/52] MLK-22284-2 dmaengine: fsl-edma-v3: check dma description before register touch Check dma desscription firstly to ignore any unexpected interrupt after channel terminate, otherwise, still have chance to touch channel register whose power has been already off. Signed-off-by: Robin Gong Reviewed-by: S.j. Wang (cherry picked from commit fd073e017e317006a4c254ca5ae3ea17b6f32502) --- drivers/dma/fsl-edma-v3.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index bc6192bb234e..a84515bef9d6 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -708,20 +708,20 @@ static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) unsigned int intr; void __iomem *base_addr; - base_addr = fsl_chan->membase; - - intr = readl(base_addr + EDMA_CH_INT); - if (!intr) - return IRQ_NONE; - - writel(1, base_addr + EDMA_CH_INT); - spin_lock(&fsl_chan->vchan.lock); /* Ignore this interrupt since channel has been disabled already */ if (!fsl_chan->edesc) goto irq_handled; + base_addr = fsl_chan->membase; + + intr = readl(base_addr + EDMA_CH_INT); + if (!intr) + goto irq_handled; + + writel(1, base_addr + EDMA_CH_INT); + if (!fsl_chan->edesc->iscyclic) { fsl_edma3_get_realcnt(fsl_chan); list_del(&fsl_chan->edesc->vdesc.node); From 3f8b89fadd7751da40a73f49cf39d1c34dd28be3 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 23 Jul 2019 23:42:53 +0800 Subject: [PATCH 45/52] MLK-22302-2: dmaengine: fsl-edma-v3: fix build warning with CONFIG_PM_SLEEP=n Fix build waring with CONFIG_PM_SLEEP=n. Signed-off-by: Robin Gong Reviewed-by: Andy Duan (cherry picked from commit dfe2a755209615f9592ed937957b2efdc3b6d6c0) --- drivers/dma/fsl-edma-v3.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index a84515bef9d6..90b3dc5fc86d 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -1040,6 +1040,7 @@ static int fsl_edma3_remove(struct platform_device *pdev) return 0; } +#ifdef CONFIG_PM_SLEEP static int fsl_edma3_suspend_late(struct device *dev) { struct fsl_edma3_engine *fsl_edma = dev_get_drvdata(dev); @@ -1097,10 +1098,11 @@ static int fsl_edma3_resume_early(struct device *dev) return 0; } +#endif static const struct dev_pm_ops fsl_edma3_pm_ops = { - .suspend_late = fsl_edma3_suspend_late, - .resume_early = fsl_edma3_resume_early, + SET_LATE_SYSTEM_SLEEP_PM_OPS(fsl_edma3_suspend_late, + fsl_edma3_resume_early) }; static const struct of_device_id fsl_edma3_dt_ids[] = { From 5882a6f8ef88ef4d40c67b57846e6af1f8a2d68f Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 5 Nov 2019 21:32:22 +0800 Subject: [PATCH 46/52] MLK-22909 dmaengine: fsl-edma-v3: clear interrupt coming after channel terminated Clear EDMA_CH_INT in case dma done interrupt comes after channel terminated instead of channel free-ed, otherwise, RCU maybe caught because it's ignored without interrupt status cleared as Android team report in Monkey test. Signed-off-by: Robin Gong Acked-by: Fugang Duan (cherry picked from commit ef91ff6ed465cebe2fe6483a480351abba36e237) (cherry picked from commit 56ee55c71c5f3ef254acb4dee581e68f79ef13a5) --- drivers/dma/fsl-edma-v3.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/dma/fsl-edma-v3.c b/drivers/dma/fsl-edma-v3.c index 90b3dc5fc86d..41762dd73999 100644 --- a/drivers/dma/fsl-edma-v3.c +++ b/drivers/dma/fsl-edma-v3.c @@ -710,8 +710,8 @@ static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) spin_lock(&fsl_chan->vchan.lock); - /* Ignore this interrupt since channel has been disabled already */ - if (!fsl_chan->edesc) + /* Ignore this interrupt since channel has been freeed with power off */ + if (!fsl_chan->edesc && !fsl_chan->tcd_pool) goto irq_handled; base_addr = fsl_chan->membase; @@ -722,6 +722,10 @@ static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) writel(1, base_addr + EDMA_CH_INT); + /* Ignore this interrupt since channel has been disabled already */ + if (!fsl_chan->edesc) + goto irq_handled; + if (!fsl_chan->edesc->iscyclic) { fsl_edma3_get_realcnt(fsl_chan); list_del(&fsl_chan->edesc->vdesc.node); From 1e9b1c9bd736a5c3605a82c3a242dbafdfbb4e81 Mon Sep 17 00:00:00 2001 From: Peng Ma Date: Mon, 4 Mar 2019 15:39:14 +0800 Subject: [PATCH 47/52] dmaengine: fsl-dpaa2-qdma: Add the DPDMAI(Data Path DMA Interface) support The MC(Management Complex) exports the DPDMAI(Data Path DMA Interface) object as an interface to operate the DPAA2(Data Path Acceleration Architecture 2) qDMA Engine. The DPDMAI enables sending frame-based requests to qDMA and receiving back confirmation response on transaction completion, utilizing the DPAA2 QBMan(Queue Manager and Buffer Manager hardware) infrastructure. DPDMAI object provides up to two priorities for processing qDMA requests. The following list summarizes the DPDMAI main features and capabilities: 1. Supports up to two scheduling priorities for processing service requests. - Each DPDMAI transmit queue is mapped to one of two service priorities, allowing further prioritization in hardware between requests from different DPDMAI objects. 2. Supports up to two receive queues for incoming transaction completion confirmations. - Each DPDMAI receive queue is mapped to one of two receive priorities, allowing further prioritization between other interfaces when associating the DPDMAI receive queues to DPIO or DPCON(Data Path Concentrator) objects. 3. Supports different scheduling options for processing received packets: - Queues can be configured either in 'parked' mode (default), or attached to a DPIO object, or attached to DPCON object. 4. Allows interaction with one or more DPIO objects for dequeueing/enqueueing frame descriptors(FD) and for acquiring/releasing buffers. 5. Supports enable, disable, and reset operations. Add dpdmai to support some platforms with dpaa2 qdma engine. Signed-off-by: Peng Ma --- drivers/dma/fsl-dpaa2-qdma/dpdmai.c | 366 ++++++++++++++++++++++++++++ drivers/dma/fsl-dpaa2-qdma/dpdmai.h | 177 ++++++++++++++ 2 files changed, 543 insertions(+) create mode 100644 drivers/dma/fsl-dpaa2-qdma/dpdmai.c create mode 100644 drivers/dma/fsl-dpaa2-qdma/dpdmai.h diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c new file mode 100644 index 000000000000..fbc2b2f39bec --- /dev/null +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2019 NXP + +#include +#include +#include +#include "dpdmai.h" + +struct dpdmai_rsp_get_attributes { + __le32 id; + u8 num_of_priorities; + u8 pad0[3]; + __le16 major; + __le16 minor; +}; + +struct dpdmai_cmd_queue { + __le32 dest_id; + u8 priority; + u8 queue; + u8 dest_type; + u8 pad; + __le64 user_ctx; + union { + __le32 options; + __le32 fqid; + }; +}; + +struct dpdmai_rsp_get_tx_queue { + __le64 pad; + __le32 fqid; +}; + +#define MC_CMD_OP(_cmd, _param, _offset, _width, _type, _arg) \ + ((_cmd).params[_param] |= mc_enc((_offset), (_width), _arg)) + +/* cmd, param, offset, width, type, arg_name */ +#define DPDMAI_CMD_CREATE(_cmd, _cfg) \ +do { \ + typeof(_cmd) (cmd) = (_cmd); \ + typeof(_cfg) (cfg) = (_cfg); \ + MC_CMD_OP(cmd, 0, 8, 8, u8, (cfg)->priorities[0]);\ + MC_CMD_OP(cmd, 0, 16, 8, u8, (cfg)->priorities[1]);\ +} while (0) + +static inline u64 mc_enc(int lsoffset, int width, u64 val) +{ + return (val & MAKE_UMASK64(width)) << lsoffset; +} + +/** + * dpdmai_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpdmai_id: DPDMAI unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpdmai_create() function. + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent commands for + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, + int dpdmai_id, u16 *token) +{ + struct fsl_mc_command cmd = { 0 }; + __le64 *cmd_dpdmai_id; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN, + cmd_flags, 0); + + cmd_dpdmai_id = cmd.params; + *cmd_dpdmai_id = cpu_to_le32(dpdmai_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = mc_cmd_hdr_read_token(&cmd); + + return 0; +} + +/** + * dpdmai_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * + * After this function is called, no further operations are + * allowed on the object without opening a new control session. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct fsl_mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLOSE, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_create() - Create the DPDMAI object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @token: Returned token; use in subsequent API calls + * + * Create the DPDMAI object, allocate required resources and + * perform required initialization. + * + * The object can be created either by declaring it in the + * DPL file, or by calling this function. + * + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent calls to + * this specific object. For objects that are created using the + * DPL file, call dpdmai_open() function to get an authentication + * token first. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, + const struct dpdmai_cfg *cfg, u16 *token) +{ + struct fsl_mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE, + cmd_flags, 0); + DPDMAI_CMD_CREATE(cmd, cfg); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = mc_cmd_hdr_read_token(&cmd); + + return 0; +} + +/** + * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct fsl_mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_ENABLE, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_disable() - Disable the DPDMAI, stop sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct fsl_mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DISABLE, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_reset() - Reset the DPDMAI, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct fsl_mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_RESET, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_get_attributes() - Retrieve DPDMAI attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, struct dpdmai_attr *attr) +{ + struct dpdmai_rsp_get_attributes *rsp_params; + struct fsl_mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_ATTR, + cmd_flags, token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpdmai_rsp_get_attributes *)cmd.params; + attr->id = le32_to_cpu(rsp_params->id); + attr->version.major = le16_to_cpu(rsp_params->major); + attr->version.minor = le16_to_cpu(rsp_params->minor); + attr->num_of_priorities = rsp_params->num_of_priorities; + + return 0; +} + +/** + * dpdmai_set_rx_queue() - Set Rx queue configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * @priority: Select the queue relative to number of + * priorities configured at DPDMAI creation + * @cfg: Rx queue configuration + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 priority, const struct dpdmai_rx_queue_cfg *cfg) +{ + struct dpdmai_cmd_queue *cmd_params; + struct fsl_mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_RX_QUEUE, + cmd_flags, token); + + cmd_params = (struct dpdmai_cmd_queue *)cmd.params; + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); + cmd_params->priority = cfg->dest_cfg.priority; + cmd_params->queue = priority; + cmd_params->dest_type = cfg->dest_cfg.dest_type; + cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); + cmd_params->options = cpu_to_le32(cfg->options); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpdmai_get_rx_queue() - Retrieve Rx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * @priority: Select the queue relative to number of + * priorities configured at DPDMAI creation + * @attr: Returned Rx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 priority, struct dpdmai_rx_queue_attr *attr) +{ + struct dpdmai_cmd_queue *cmd_params; + struct fsl_mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_RX_QUEUE, + cmd_flags, token); + + cmd_params = (struct dpdmai_cmd_queue *)cmd.params; + cmd_params->queue = priority; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id); + attr->dest_cfg.priority = cmd_params->priority; + attr->dest_cfg.dest_type = cmd_params->dest_type; + attr->user_ctx = le64_to_cpu(cmd_params->user_ctx); + attr->fqid = le32_to_cpu(cmd_params->fqid); + + return 0; +} + +/** + * dpdmai_get_tx_queue() - Retrieve Tx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * @priority: Select the queue relative to number of + * priorities configured at DPDMAI creation + * @fqid: Returned Tx queue + * + * Return: '0' on Success; Error code otherwise. + */ +int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, u8 priority, u32 *fqid) +{ + struct dpdmai_rsp_get_tx_queue *rsp_params; + struct dpdmai_cmd_queue *cmd_params; + struct fsl_mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_TX_QUEUE, + cmd_flags, token); + + cmd_params = (struct dpdmai_cmd_queue *)cmd.params; + cmd_params->queue = priority; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + + rsp_params = (struct dpdmai_rsp_get_tx_queue *)cmd.params; + *fqid = le32_to_cpu(rsp_params->fqid); + + return 0; +} diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h new file mode 100644 index 000000000000..6d785093da8e --- /dev/null +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h @@ -0,0 +1,177 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright 2019 NXP */ + +#ifndef __FSL_DPDMAI_H +#define __FSL_DPDMAI_H + +/* DPDMAI Version */ +#define DPDMAI_VER_MAJOR 2 +#define DPDMAI_VER_MINOR 2 + +#define DPDMAI_CMD_BASE_VERSION 0 +#define DPDMAI_CMD_ID_OFFSET 4 + +#define DPDMAI_CMDID_FORMAT(x) (((x) << DPDMAI_CMD_ID_OFFSET) | \ + DPDMAI_CMD_BASE_VERSION) + +/* Command IDs */ +#define DPDMAI_CMDID_CLOSE DPDMAI_CMDID_FORMAT(0x800) +#define DPDMAI_CMDID_OPEN DPDMAI_CMDID_FORMAT(0x80E) +#define DPDMAI_CMDID_CREATE DPDMAI_CMDID_FORMAT(0x90E) + +#define DPDMAI_CMDID_ENABLE DPDMAI_CMDID_FORMAT(0x002) +#define DPDMAI_CMDID_DISABLE DPDMAI_CMDID_FORMAT(0x003) +#define DPDMAI_CMDID_GET_ATTR DPDMAI_CMDID_FORMAT(0x004) +#define DPDMAI_CMDID_RESET DPDMAI_CMDID_FORMAT(0x005) +#define DPDMAI_CMDID_IS_ENABLED DPDMAI_CMDID_FORMAT(0x006) + +#define DPDMAI_CMDID_SET_IRQ DPDMAI_CMDID_FORMAT(0x010) +#define DPDMAI_CMDID_GET_IRQ DPDMAI_CMDID_FORMAT(0x011) +#define DPDMAI_CMDID_SET_IRQ_ENABLE DPDMAI_CMDID_FORMAT(0x012) +#define DPDMAI_CMDID_GET_IRQ_ENABLE DPDMAI_CMDID_FORMAT(0x013) +#define DPDMAI_CMDID_SET_IRQ_MASK DPDMAI_CMDID_FORMAT(0x014) +#define DPDMAI_CMDID_GET_IRQ_MASK DPDMAI_CMDID_FORMAT(0x015) +#define DPDMAI_CMDID_GET_IRQ_STATUS DPDMAI_CMDID_FORMAT(0x016) +#define DPDMAI_CMDID_CLEAR_IRQ_STATUS DPDMAI_CMDID_FORMAT(0x017) + +#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMDID_FORMAT(0x1A0) +#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMDID_FORMAT(0x1A1) +#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMDID_FORMAT(0x1A2) + +#define MC_CMD_HDR_TOKEN_O 32 /* Token field offset */ +#define MC_CMD_HDR_TOKEN_S 16 /* Token field size */ + +#define MAKE_UMASK64(_width) \ + ((u64)((_width) < 64 ? ((u64)1 << (_width)) - 1 : (u64)-1)) + +/* Data Path DMA Interface API + * Contains initialization APIs and runtime control APIs for DPDMAI + */ + +/** + * Maximum number of Tx/Rx priorities per DPDMAI object + */ +#define DPDMAI_PRIO_NUM 2 + +/* DPDMAI queue modification options */ + +/** + * Select to modify the user's context associated with the queue + */ +#define DPDMAI_QUEUE_OPT_USER_CTX 0x1 + +/** + * Select to modify the queue's destination + */ +#define DPDMAI_QUEUE_OPT_DEST 0x2 + +/** + * struct dpdmai_cfg - Structure representing DPDMAI configuration + * @priorities: Priorities for the DMA hardware processing; valid priorities are + * configured with values 1-8; the entry following last valid entry + * should be configured with 0 + */ +struct dpdmai_cfg { + u8 priorities[DPDMAI_PRIO_NUM]; +}; + +/** + * struct dpdmai_attr - Structure representing DPDMAI attributes + * @id: DPDMAI object ID + * @version: DPDMAI version + * @num_of_priorities: number of priorities + */ +struct dpdmai_attr { + int id; + /** + * struct version - DPDMAI version + * @major: DPDMAI major version + * @minor: DPDMAI minor version + */ + struct { + u16 major; + u16 minor; + } version; + u8 num_of_priorities; +}; + +/** + * enum dpdmai_dest - DPDMAI destination types + * @DPDMAI_DEST_NONE: Unassigned destination; The queue is set in parked mode + * and does not generate FQDAN notifications; user is expected to dequeue + * from the queue based on polling or other user-defined method + * @DPDMAI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN + * notifications to the specified DPIO; user is expected to dequeue + * from the queue only after notification is received + * @DPDMAI_DEST_DPCON: The queue is set in schedule mode and does not generate + * FQDAN notifications, but is connected to the specified DPCON object; + * user is expected to dequeue from the DPCON channel + */ +enum dpdmai_dest { + DPDMAI_DEST_NONE = 0, + DPDMAI_DEST_DPIO = 1, + DPDMAI_DEST_DPCON = 2 +}; + +/** + * struct dpdmai_dest_cfg - Structure representing DPDMAI destination parameters + * @dest_type: Destination type + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type + * @priority: Priority selection within the DPIO or DPCON channel; valid values + * are 0-1 or 0-7, depending on the number of priorities in that + * channel; not relevant for 'DPDMAI_DEST_NONE' option + */ +struct dpdmai_dest_cfg { + enum dpdmai_dest dest_type; + int dest_id; + u8 priority; +}; + +/** + * struct dpdmai_rx_queue_cfg - DPDMAI RX queue configuration + * @options: Flags representing the suggested modifications to the queue; + * Use any combination of 'DPDMAI_QUEUE_OPT_' flags + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame; + * valid only if 'DPDMAI_QUEUE_OPT_USER_CTX' is contained in 'options' + * @dest_cfg: Queue destination parameters; + * valid only if 'DPDMAI_QUEUE_OPT_DEST' is contained in 'options' + */ +struct dpdmai_rx_queue_cfg { + struct dpdmai_dest_cfg dest_cfg; + u32 options; + u64 user_ctx; + +}; + +/** + * struct dpdmai_rx_queue_attr - Structure representing attributes of Rx queues + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame + * @dest_cfg: Queue destination configuration + * @fqid: Virtual FQID value to be used for dequeue operations + */ +struct dpdmai_rx_queue_attr { + struct dpdmai_dest_cfg dest_cfg; + u64 user_ctx; + u32 fqid; +}; + +int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, + int dpdmai_id, u16 *token); +int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); +int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, + const struct dpdmai_cfg *cfg, u16 *token); +int dpdmai_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); +int dpdmai_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); +int dpdmai_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); +int dpdmai_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, struct dpdmai_attr *attr); +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 priority, const struct dpdmai_rx_queue_cfg *cfg); +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 priority, struct dpdmai_rx_queue_attr *attr); +int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, u8 priority, u32 *fqid); + +#endif /* __FSL_DPDMAI_H */ From 1154aa773a04c28e01b16606264a67eb6ef5511d Mon Sep 17 00:00:00 2001 From: Peng Ma Date: Mon, 4 Mar 2019 15:45:56 +0800 Subject: [PATCH 48/52] dmaengine: fsl-dpaa2-qdma: Add NXP dpaa2 qDMA controller driver for Layerscape SoCs DPPA2(Data Path Acceleration Architecture 2) qDMA supports virtualized channel by allowing DMA jobs to be enqueued into different work queues. Core can initiate a DMA transaction by preparing a frame descriptor(FD) for each DMA job and enqueuing this job through a hardware portal. DPAA2 components can also prepare a FD and enqueue a DMA job through a hardware portal. The qDMA prefetches DMA jobs through DPAA2 hardware portal. It then schedules and dispatches to internal DMA hardware engines, which generate read and write requests. Both qDMA source data and destination data can be either contiguous or non-contiguous using one or more scatter/gather tables. The qDMA supports global bandwidth flow control where all DMA transactions are stalled if the bandwidth threshold has been reached. Also supported are transaction based read throttling. Add NXP dppa2 qDMA to support some of Layerscape SoCs. such as: LS1088A, LS208xA, LX2, etc. Signed-off-by: Peng Ma --- drivers/dma/Kconfig | 2 + drivers/dma/Makefile | 1 + drivers/dma/fsl-dpaa2-qdma/Kconfig | 9 + drivers/dma/fsl-dpaa2-qdma/Makefile | 3 + drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c | 825 ++++++++++++++++++++++++ drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h | 153 +++++ 6 files changed, 993 insertions(+) create mode 100644 drivers/dma/fsl-dpaa2-qdma/Kconfig create mode 100644 drivers/dma/fsl-dpaa2-qdma/Makefile create mode 100644 drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c create mode 100644 drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 7af874b69ffb..f5decdc24181 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -669,6 +669,8 @@ source "drivers/dma/sh/Kconfig" source "drivers/dma/ti/Kconfig" +source "drivers/dma/fsl-dpaa2-qdma/Kconfig" + # clients comment "DMA Clients" depends on DMA_ENGINE diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index f5ce8665e944..b3d48f928328 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -75,6 +75,7 @@ obj-$(CONFIG_UNIPHIER_MDMAC) += uniphier-mdmac.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o obj-$(CONFIG_ZX_DMA) += zx_dma.o obj-$(CONFIG_ST_FDMA) += st_fdma.o +obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/ obj-y += mediatek/ obj-y += qcom/ diff --git a/drivers/dma/fsl-dpaa2-qdma/Kconfig b/drivers/dma/fsl-dpaa2-qdma/Kconfig new file mode 100644 index 000000000000..258ed6be934d --- /dev/null +++ b/drivers/dma/fsl-dpaa2-qdma/Kconfig @@ -0,0 +1,9 @@ +menuconfig FSL_DPAA2_QDMA + tristate "NXP DPAA2 QDMA" + depends on ARM64 + depends on FSL_MC_BUS && FSL_MC_DPIO + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + NXP Data Path Acceleration Architecture 2 QDMA driver, + using the NXP MC bus driver. diff --git a/drivers/dma/fsl-dpaa2-qdma/Makefile b/drivers/dma/fsl-dpaa2-qdma/Makefile new file mode 100644 index 000000000000..c1d0226f2bd7 --- /dev/null +++ b/drivers/dma/fsl-dpaa2-qdma/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for the NXP DPAA2 qDMA controllers +obj-$(CONFIG_FSL_DPAA2_QDMA) += dpaa2-qdma.o dpdmai.o diff --git a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c new file mode 100644 index 000000000000..c70a7965f140 --- /dev/null +++ b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c @@ -0,0 +1,825 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2019 NXP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../virt-dma.h" +#include "dpdmai.h" +#include "dpaa2-qdma.h" + +static bool smmu_disable = true; + +static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct dpaa2_qdma_chan, vchan.chan); +} + +static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd) +{ + return container_of(vd, struct dpaa2_qdma_comp, vdesc); +} + +static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan) +{ + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); + struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma; + struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev; + + dpaa2_chan->fd_pool = dma_pool_create("fd_pool", dev, + sizeof(struct dpaa2_fd), + sizeof(struct dpaa2_fd), 0); + if (!dpaa2_chan->fd_pool) + goto err; + + dpaa2_chan->fl_pool = dma_pool_create("fl_pool", dev, + sizeof(struct dpaa2_fl_entry), + sizeof(struct dpaa2_fl_entry), 0); + if (!dpaa2_chan->fl_pool) + goto err_fd; + + dpaa2_chan->sdd_pool = + dma_pool_create("sdd_pool", dev, + sizeof(struct dpaa2_qdma_sd_d), + sizeof(struct dpaa2_qdma_sd_d), 0); + if (!dpaa2_chan->sdd_pool) + goto err_fl; + + return dpaa2_qdma->desc_allocated++; +err_fl: + dma_pool_destroy(dpaa2_chan->fl_pool); +err_fd: + dma_pool_destroy(dpaa2_chan->fd_pool); +err: + return -ENOMEM; +} + +static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan) +{ + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); + struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma; + unsigned long flags; + + LIST_HEAD(head); + + spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags); + vchan_get_all_descriptors(&dpaa2_chan->vchan, &head); + spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags); + + vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head); + + dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_used); + dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_free); + + dma_pool_destroy(dpaa2_chan->fd_pool); + dma_pool_destroy(dpaa2_chan->fl_pool); + dma_pool_destroy(dpaa2_chan->sdd_pool); + dpaa2_qdma->desc_allocated--; +} + +/* + * Request a command descriptor for enqueue. + */ +static struct dpaa2_qdma_comp * +dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan) +{ + struct dpaa2_qdma_priv *qdma_priv = dpaa2_chan->qdma->priv; + struct device *dev = &qdma_priv->dpdmai_dev->dev; + struct dpaa2_qdma_comp *comp_temp = NULL; + unsigned long flags; + + spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); + if (list_empty(&dpaa2_chan->comp_free)) { + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); + comp_temp = kzalloc(sizeof(*comp_temp), GFP_NOWAIT); + if (!comp_temp) + goto err; + comp_temp->fd_virt_addr = + dma_pool_alloc(dpaa2_chan->fd_pool, GFP_NOWAIT, + &comp_temp->fd_bus_addr); + if (!comp_temp->fd_virt_addr) + goto err_comp; + + comp_temp->fl_virt_addr = + dma_pool_alloc(dpaa2_chan->fl_pool, GFP_NOWAIT, + &comp_temp->fl_bus_addr); + if (!comp_temp->fl_virt_addr) + goto err_fd_virt; + + comp_temp->desc_virt_addr = + dma_pool_alloc(dpaa2_chan->sdd_pool, GFP_NOWAIT, + &comp_temp->desc_bus_addr); + if (!comp_temp->desc_virt_addr) + goto err_fl_virt; + + comp_temp->qchan = dpaa2_chan; + return comp_temp; + } + + comp_temp = list_first_entry(&dpaa2_chan->comp_free, + struct dpaa2_qdma_comp, list); + list_del(&comp_temp->list); + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); + + comp_temp->qchan = dpaa2_chan; + + return comp_temp; + +err_fl_virt: + dma_pool_free(dpaa2_chan->fl_pool, + comp_temp->fl_virt_addr, + comp_temp->fl_bus_addr); +err_fd_virt: + dma_pool_free(dpaa2_chan->fd_pool, + comp_temp->fd_virt_addr, + comp_temp->fd_bus_addr); +err_comp: + kfree(comp_temp); +err: + dev_err(dev, "Failed to request descriptor\n"); + return NULL; +} + +static void +dpaa2_qdma_populate_fd(u32 format, struct dpaa2_qdma_comp *dpaa2_comp) +{ + struct dpaa2_fd *fd; + + fd = dpaa2_comp->fd_virt_addr; + memset(fd, 0, sizeof(struct dpaa2_fd)); + + /* fd populated */ + dpaa2_fd_set_addr(fd, dpaa2_comp->fl_bus_addr); + + /* + * Bypass memory translation, Frame list format, short length disable + * we need to disable BMT if fsl-mc use iova addr + */ + if (smmu_disable) + dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE); + dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE); + + dpaa2_fd_set_frc(fd, format | QDMA_SER_CTX); +} + +/* first frame list for descriptor buffer */ +static void +dpaa2_qdma_populate_first_framel(struct dpaa2_fl_entry *f_list, + struct dpaa2_qdma_comp *dpaa2_comp, + bool wrt_changed) +{ + struct dpaa2_qdma_sd_d *sdd; + + sdd = dpaa2_comp->desc_virt_addr; + memset(sdd, 0, 2 * (sizeof(*sdd))); + + /* source descriptor CMD */ + sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT); + sdd++; + + /* dest descriptor CMD */ + if (wrt_changed) + sdd->cmd = cpu_to_le32(LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT); + else + sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT); + + memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); + + /* first frame list to source descriptor */ + dpaa2_fl_set_addr(f_list, dpaa2_comp->desc_bus_addr); + dpaa2_fl_set_len(f_list, 0x20); + dpaa2_fl_set_format(f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG); + + /* bypass memory translation */ + if (smmu_disable) + f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); +} + +/* source and destination frame list */ +static void +dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list, + dma_addr_t dst, dma_addr_t src, + size_t len, uint8_t fmt) +{ + /* source frame list to source buffer */ + memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); + + dpaa2_fl_set_addr(f_list, src); + dpaa2_fl_set_len(f_list, len); + + /* single buffer frame or scatter gather frame */ + dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG)); + + /* bypass memory translation */ + if (smmu_disable) + f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); + + f_list++; + + /* destination frame list to destination buffer */ + memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); + + dpaa2_fl_set_addr(f_list, dst); + dpaa2_fl_set_len(f_list, len); + dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG)); + /* single buffer frame or scatter gather frame */ + dpaa2_fl_set_final(f_list, QDMA_FL_F); + /* bypass memory translation */ + if (smmu_disable) + f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); +} + +static struct dma_async_tx_descriptor +*dpaa2_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, + dma_addr_t src, size_t len, ulong flags) +{ + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); + struct dpaa2_qdma_engine *dpaa2_qdma; + struct dpaa2_qdma_comp *dpaa2_comp; + struct dpaa2_fl_entry *f_list; + bool wrt_changed; + + dpaa2_qdma = dpaa2_chan->qdma; + dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan); + if (!dpaa2_comp) + return NULL; + + wrt_changed = (bool)dpaa2_qdma->qdma_wrtype_fixup; + + /* populate Frame descriptor */ + dpaa2_qdma_populate_fd(QDMA_FD_LONG_FORMAT, dpaa2_comp); + + f_list = dpaa2_comp->fl_virt_addr; + + /* first frame list for descriptor buffer (logn format) */ + dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp, wrt_changed); + + f_list++; + + dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF); + + return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags); +} + +static void dpaa2_qdma_issue_pending(struct dma_chan *chan) +{ + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); + struct dpaa2_qdma_comp *dpaa2_comp; + struct virt_dma_desc *vdesc; + struct dpaa2_fd *fd; + unsigned long flags; + int err; + + spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); + spin_lock(&dpaa2_chan->vchan.lock); + if (vchan_issue_pending(&dpaa2_chan->vchan)) { + vdesc = vchan_next_desc(&dpaa2_chan->vchan); + if (!vdesc) + goto err_enqueue; + dpaa2_comp = to_fsl_qdma_comp(vdesc); + + fd = dpaa2_comp->fd_virt_addr; + + list_del(&vdesc->node); + list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used); + + err = dpaa2_io_service_enqueue_fq(NULL, dpaa2_chan->fqid, fd); + if (err) { + list_del(&dpaa2_comp->list); + list_add_tail(&dpaa2_comp->list, + &dpaa2_chan->comp_free); + } + } +err_enqueue: + spin_unlock(&dpaa2_chan->vchan.lock); + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); +} + +static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) +{ + struct dpaa2_qdma_priv_per_prio *ppriv; + struct device *dev = &ls_dev->dev; + struct dpaa2_qdma_priv *priv; + u8 prio_def = DPDMAI_PRIO_NUM; + int err = -EINVAL; + int i; + + priv = dev_get_drvdata(dev); + + priv->dev = dev; + priv->dpqdma_id = ls_dev->obj_desc.id; + + /* Get the handle for the DPDMAI this interface is associate with */ + err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle); + if (err) { + dev_err(dev, "dpdmai_open() failed\n"); + return err; + } + + dev_dbg(dev, "Opened dpdmai object successfully\n"); + + err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, + &priv->dpdmai_attr); + if (err) { + dev_err(dev, "dpdmai_get_attributes() failed\n"); + goto exit; + } + + if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) { + dev_err(dev, "DPDMAI major version mismatch\n" + "Found %u.%u, supported version is %u.%u\n", + priv->dpdmai_attr.version.major, + priv->dpdmai_attr.version.minor, + DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR); + goto exit; + } + + if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) { + dev_err(dev, "DPDMAI minor version mismatch\n" + "Found %u.%u, supported version is %u.%u\n", + priv->dpdmai_attr.version.major, + priv->dpdmai_attr.version.minor, + DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR); + goto exit; + } + + priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def); + ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL); + if (!ppriv) { + err = -ENOMEM; + goto exit; + } + priv->ppriv = ppriv; + + for (i = 0; i < priv->num_pairs; i++) { + err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, + i, &priv->rx_queue_attr[i]); + if (err) { + dev_err(dev, "dpdmai_get_rx_queue() failed\n"); + goto exit; + } + ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; + + err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, + i, &priv->tx_fqid[i]); + if (err) { + dev_err(dev, "dpdmai_get_tx_queue() failed\n"); + goto exit; + } + ppriv->req_fqid = priv->tx_fqid[i]; + ppriv->prio = i; + ppriv->priv = priv; + ppriv++; + } + + return 0; +exit: + dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); + return err; +} + +static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx) +{ + struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx, + struct dpaa2_qdma_priv_per_prio, nctx); + struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp; + struct dpaa2_qdma_priv *priv = ppriv->priv; + u32 n_chans = priv->dpaa2_qdma->n_chans; + struct dpaa2_qdma_chan *qchan; + const struct dpaa2_fd *fd_eq; + const struct dpaa2_fd *fd; + struct dpaa2_dq *dq; + int is_last = 0; + int found; + u8 status; + int err; + int i; + + do { + err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid, + ppriv->store); + } while (err); + + while (!is_last) { + do { + dq = dpaa2_io_store_next(ppriv->store, &is_last); + } while (!is_last && !dq); + if (!dq) { + dev_err(priv->dev, "FQID returned no valid frames!\n"); + continue; + } + + /* obtain FD and process the error */ + fd = dpaa2_dq_fd(dq); + + status = dpaa2_fd_get_ctrl(fd) & 0xff; + if (status) + dev_err(priv->dev, "FD error occurred\n"); + found = 0; + for (i = 0; i < n_chans; i++) { + qchan = &priv->dpaa2_qdma->chans[i]; + spin_lock(&qchan->queue_lock); + if (list_empty(&qchan->comp_used)) { + spin_unlock(&qchan->queue_lock); + continue; + } + list_for_each_entry_safe(dpaa2_comp, _comp_tmp, + &qchan->comp_used, list) { + fd_eq = dpaa2_comp->fd_virt_addr; + + if (le64_to_cpu(fd_eq->simple.addr) == + le64_to_cpu(fd->simple.addr)) { + spin_lock(&qchan->vchan.lock); + vchan_cookie_complete(& + dpaa2_comp->vdesc); + spin_unlock(&qchan->vchan.lock); + found = 1; + break; + } + } + spin_unlock(&qchan->queue_lock); + if (found) + break; + } + } + + dpaa2_io_service_rearm(NULL, ctx); +} + +static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv) +{ + struct dpaa2_qdma_priv_per_prio *ppriv; + struct device *dev = priv->dev; + int err = -EINVAL; + int i, num; + + num = priv->num_pairs; + ppriv = priv->ppriv; + for (i = 0; i < num; i++) { + ppriv->nctx.is_cdan = 0; + ppriv->nctx.desired_cpu = DPAA2_IO_ANY_CPU; + ppriv->nctx.id = ppriv->rsp_fqid; + ppriv->nctx.cb = dpaa2_qdma_fqdan_cb; + err = dpaa2_io_service_register(NULL, &ppriv->nctx, dev); + if (err) { + dev_err(dev, "Notification register failed\n"); + goto err_service; + } + + ppriv->store = + dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE, dev); + if (!ppriv->store) { + dev_err(dev, "dpaa2_io_store_create() failed\n"); + goto err_store; + } + + ppriv++; + } + return 0; + +err_store: + dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev); +err_service: + ppriv--; + while (ppriv >= priv->ppriv) { + dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev); + dpaa2_io_store_destroy(ppriv->store); + ppriv--; + } + return err; +} + +static void dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv) +{ + struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; + int i; + + for (i = 0; i < priv->num_pairs; i++) { + dpaa2_io_store_destroy(ppriv->store); + ppriv++; + } +} + +static void dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv) +{ + struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; + struct device *dev = priv->dev; + int i; + + for (i = 0; i < priv->num_pairs; i++) { + dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev); + ppriv++; + } +} + +static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv) +{ + struct dpdmai_rx_queue_cfg rx_queue_cfg; + struct dpaa2_qdma_priv_per_prio *ppriv; + struct device *dev = priv->dev; + struct fsl_mc_device *ls_dev; + int i, num; + int err; + + ls_dev = to_fsl_mc_device(dev); + num = priv->num_pairs; + ppriv = priv->ppriv; + for (i = 0; i < num; i++) { + rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX | + DPDMAI_QUEUE_OPT_DEST; + rx_queue_cfg.user_ctx = ppriv->nctx.qman64; + rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO; + rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; + rx_queue_cfg.dest_cfg.priority = ppriv->prio; + err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, + rx_queue_cfg.dest_cfg.priority, + &rx_queue_cfg); + if (err) { + dev_err(dev, "dpdmai_set_rx_queue() failed\n"); + return err; + } + + ppriv++; + } + + return 0; +} + +static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv) +{ + struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; + struct device *dev = priv->dev; + struct fsl_mc_device *ls_dev; + int err = 0; + int i; + + ls_dev = to_fsl_mc_device(dev); + + for (i = 0; i < priv->num_pairs; i++) { + ppriv->nctx.qman64 = 0; + ppriv->nctx.dpio_id = 0; + ppriv++; + } + + err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle); + if (err) + dev_err(dev, "dpdmai_reset() failed\n"); + + return err; +} + +static void dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan, + struct list_head *head) +{ + struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp; + unsigned long flags; + + list_for_each_entry_safe(comp_tmp, _comp_tmp, + head, list) { + spin_lock_irqsave(&qchan->queue_lock, flags); + list_del(&comp_tmp->list); + spin_unlock_irqrestore(&qchan->queue_lock, flags); + dma_pool_free(qchan->fd_pool, + comp_tmp->fd_virt_addr, + comp_tmp->fd_bus_addr); + dma_pool_free(qchan->fl_pool, + comp_tmp->fl_virt_addr, + comp_tmp->fl_bus_addr); + dma_pool_free(qchan->sdd_pool, + comp_tmp->desc_virt_addr, + comp_tmp->desc_bus_addr); + kfree(comp_tmp); + } +} + +static void dpaa2_dpdmai_free_channels(struct dpaa2_qdma_engine *dpaa2_qdma) +{ + struct dpaa2_qdma_chan *qchan; + int num, i; + + num = dpaa2_qdma->n_chans; + for (i = 0; i < num; i++) { + qchan = &dpaa2_qdma->chans[i]; + dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used); + dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free); + dma_pool_destroy(qchan->fd_pool); + dma_pool_destroy(qchan->fl_pool); + dma_pool_destroy(qchan->sdd_pool); + } +} + +static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc) +{ + struct dpaa2_qdma_comp *dpaa2_comp; + struct dpaa2_qdma_chan *qchan; + unsigned long flags; + + dpaa2_comp = to_fsl_qdma_comp(vdesc); + qchan = dpaa2_comp->qchan; + spin_lock_irqsave(&qchan->queue_lock, flags); + list_del(&dpaa2_comp->list); + list_add_tail(&dpaa2_comp->list, &qchan->comp_free); + spin_unlock_irqrestore(&qchan->queue_lock, flags); +} + +static int dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine *dpaa2_qdma) +{ + struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv; + struct dpaa2_qdma_chan *dpaa2_chan; + int num = priv->num_pairs; + int i; + + INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels); + for (i = 0; i < dpaa2_qdma->n_chans; i++) { + dpaa2_chan = &dpaa2_qdma->chans[i]; + dpaa2_chan->qdma = dpaa2_qdma; + dpaa2_chan->fqid = priv->tx_fqid[i % num]; + dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc; + vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev); + spin_lock_init(&dpaa2_chan->queue_lock); + INIT_LIST_HEAD(&dpaa2_chan->comp_used); + INIT_LIST_HEAD(&dpaa2_chan->comp_free); + } + return 0; +} + +static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev) +{ + struct device *dev = &dpdmai_dev->dev; + struct dpaa2_qdma_engine *dpaa2_qdma; + struct dpaa2_qdma_priv *priv; + int err; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + dev_set_drvdata(dev, priv); + priv->dpdmai_dev = dpdmai_dev; + + priv->iommu_domain = iommu_get_domain_for_dev(dev); + if (priv->iommu_domain) + smmu_disable = false; + + /* obtain a MC portal */ + err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io); + if (err) { + if (err == -ENXIO) + err = -EPROBE_DEFER; + else + dev_err(dev, "MC portal allocation failed\n"); + goto err_mcportal; + } + + /* DPDMAI initialization */ + err = dpaa2_qdma_setup(dpdmai_dev); + if (err) { + dev_err(dev, "dpaa2_dpdmai_setup() failed\n"); + goto err_dpdmai_setup; + } + + /* DPIO */ + err = dpaa2_qdma_dpio_setup(priv); + if (err) { + dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n"); + goto err_dpio_setup; + } + + /* DPDMAI binding to DPIO */ + err = dpaa2_dpdmai_bind(priv); + if (err) { + dev_err(dev, "dpaa2_dpdmai_bind() failed\n"); + goto err_bind; + } + + /* DPDMAI enable */ + err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle); + if (err) { + dev_err(dev, "dpdmai_enable() faile\n"); + goto err_enable; + } + + dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL); + if (!dpaa2_qdma) { + err = -ENOMEM; + goto err_eng; + } + + priv->dpaa2_qdma = dpaa2_qdma; + dpaa2_qdma->priv = priv; + + dpaa2_qdma->desc_allocated = 0; + dpaa2_qdma->n_chans = NUM_CH; + + dpaa2_dpdmai_init_channels(dpaa2_qdma); + + if (soc_device_match(soc_fixup_tuning)) + dpaa2_qdma->qdma_wrtype_fixup = true; + else + dpaa2_qdma->qdma_wrtype_fixup = false; + + dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask); + dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask); + dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask); + + dpaa2_qdma->dma_dev.dev = dev; + dpaa2_qdma->dma_dev.device_alloc_chan_resources = + dpaa2_qdma_alloc_chan_resources; + dpaa2_qdma->dma_dev.device_free_chan_resources = + dpaa2_qdma_free_chan_resources; + dpaa2_qdma->dma_dev.device_tx_status = dma_cookie_status; + dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy; + dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending; + + err = dma_async_device_register(&dpaa2_qdma->dma_dev); + if (err) { + dev_err(dev, "Can't register NXP QDMA engine.\n"); + goto err_dpaa2_qdma; + } + + return 0; + +err_dpaa2_qdma: + kfree(dpaa2_qdma); +err_eng: + dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle); +err_enable: + dpaa2_dpdmai_dpio_unbind(priv); +err_bind: + dpaa2_dpmai_store_free(priv); + dpaa2_dpdmai_dpio_free(priv); +err_dpio_setup: + kfree(priv->ppriv); + dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle); +err_dpdmai_setup: + fsl_mc_portal_free(priv->mc_io); +err_mcportal: + kfree(priv); + dev_set_drvdata(dev, NULL); + return err; +} + +static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev) +{ + struct dpaa2_qdma_engine *dpaa2_qdma; + struct dpaa2_qdma_priv *priv; + struct device *dev; + + dev = &ls_dev->dev; + priv = dev_get_drvdata(dev); + dpaa2_qdma = priv->dpaa2_qdma; + + dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); + dpaa2_dpdmai_dpio_unbind(priv); + dpaa2_dpmai_store_free(priv); + dpaa2_dpdmai_dpio_free(priv); + dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); + fsl_mc_portal_free(priv->mc_io); + dev_set_drvdata(dev, NULL); + dpaa2_dpdmai_free_channels(dpaa2_qdma); + + dma_async_device_unregister(&dpaa2_qdma->dma_dev); + kfree(priv); + kfree(dpaa2_qdma); + + return 0; +} + +static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { + { + .vendor = FSL_MC_VENDOR_FREESCALE, + .obj_type = "dpdmai", + }, + { .vendor = 0x0 } +}; + +static struct fsl_mc_driver dpaa2_qdma_driver = { + .driver = { + .name = "dpaa2-qdma", + .owner = THIS_MODULE, + }, + .probe = dpaa2_qdma_probe, + .remove = dpaa2_qdma_remove, + .match_id_table = dpaa2_qdma_id_table +}; + +static int __init dpaa2_qdma_driver_init(void) +{ + return fsl_mc_driver_register(&(dpaa2_qdma_driver)); +} +late_initcall(dpaa2_qdma_driver_init); + +static void __exit fsl_qdma_exit(void) +{ + fsl_mc_driver_unregister(&(dpaa2_qdma_driver)); +} +module_exit(fsl_qdma_exit); + +MODULE_ALIAS("platform:fsl-dpaa2-qdma"); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("NXP Layerscape DPAA2 qDMA engine driver"); diff --git a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h new file mode 100644 index 000000000000..7d571849c569 --- /dev/null +++ b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright 2019 NXP */ + +#ifndef __DPAA2_QDMA_H +#define __DPAA2_QDMA_H + +#define DPAA2_QDMA_STORE_SIZE 16 +#define NUM_CH 8 + +struct dpaa2_qdma_sd_d { + u32 rsv:32; + union { + struct { + u32 ssd:12; /* souce stride distance */ + u32 sss:12; /* souce stride size */ + u32 rsv1:8; + } sdf; + struct { + u32 dsd:12; /* Destination stride distance */ + u32 dss:12; /* Destination stride size */ + u32 rsv2:8; + } ddf; + } df; + u32 rbpcmd; /* Route-by-port command */ + u32 cmd; +} __attribute__((__packed__)); + +/* Source descriptor command read transaction type for RBP=0: */ +/* coherent copy of cacheable memory */ +#define QDMA_SD_CMD_RDTTYPE_COHERENT (0xb << 28) +/* Destination descriptor command write transaction type for RBP=0: */ +/* coherent copy of cacheable memory */ +#define QDMA_DD_CMD_WRTTYPE_COHERENT (0x6 << 28) +#define LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT (0xb << 28) + +#define QMAN_FD_FMT_ENABLE BIT(0) /* frame list table enable */ +#define QMAN_FD_BMT_ENABLE BIT(15) /* bypass memory translation */ +#define QMAN_FD_BMT_DISABLE (0) /* bypass memory translation */ +#define QMAN_FD_SL_DISABLE (0) /* short lengthe disabled */ +#define QMAN_FD_SL_ENABLE BIT(14) /* short lengthe enabled */ + +#define QDMA_FINAL_BIT_DISABLE (0) /* final bit disable */ +#define QDMA_FINAL_BIT_ENABLE BIT(31) /* final bit enable */ + +#define QDMA_FD_SHORT_FORMAT BIT(11) /* short format */ +#define QDMA_FD_LONG_FORMAT (0) /* long format */ +#define QDMA_SER_DISABLE (8) /* no notification */ +#define QDMA_SER_CTX BIT(8) /* notification by FQD_CTX[fqid] */ +#define QDMA_SER_DEST (2 << 8) /* notification by destination desc */ +#define QDMA_SER_BOTH (3 << 8) /* soruce and dest notification */ +#define QDMA_FD_SPF_ENALBE BIT(30) /* source prefetch enable */ + +#define QMAN_FD_VA_ENABLE BIT(14) /* Address used is virtual address */ +#define QMAN_FD_VA_DISABLE (0)/* Address used is a real address */ +/* Flow Context: 49bit physical address */ +#define QMAN_FD_CBMT_ENABLE BIT(15) +#define QMAN_FD_CBMT_DISABLE (0) /* Flow Context: 64bit virtual address */ +#define QMAN_FD_SC_DISABLE (0) /* stashing control */ + +#define QDMA_FL_FMT_SBF (0x0) /* Single buffer frame */ +#define QDMA_FL_FMT_SGE (0x2) /* Scatter gather frame */ +#define QDMA_FL_BMT_ENABLE BIT(15) /* enable bypass memory translation */ +#define QDMA_FL_BMT_DISABLE (0x0) /* enable bypass memory translation */ +#define QDMA_FL_SL_LONG (0x0)/* long length */ +#define QDMA_FL_SL_SHORT (0x1) /* short length */ +#define QDMA_FL_F (0x1)/* last frame list bit */ + +/*Description of Frame list table structure*/ +struct dpaa2_qdma_chan { + struct dpaa2_qdma_engine *qdma; + struct virt_dma_chan vchan; + struct virt_dma_desc vdesc; + enum dma_status status; + u32 fqid; + + /* spinlock used by dpaa2 qdma driver */ + spinlock_t queue_lock; + struct dma_pool *fd_pool; + struct dma_pool *fl_pool; + struct dma_pool *sdd_pool; + + struct list_head comp_used; + struct list_head comp_free; + +}; + +struct dpaa2_qdma_comp { + dma_addr_t fd_bus_addr; + dma_addr_t fl_bus_addr; + dma_addr_t desc_bus_addr; + struct dpaa2_fd *fd_virt_addr; + struct dpaa2_fl_entry *fl_virt_addr; + struct dpaa2_qdma_sd_d *desc_virt_addr; + struct dpaa2_qdma_chan *qchan; + struct virt_dma_desc vdesc; + struct list_head list; +}; + +struct dpaa2_qdma_engine { + struct dma_device dma_dev; + u32 n_chans; + struct dpaa2_qdma_chan chans[NUM_CH]; + int qdma_wrtype_fixup; + int desc_allocated; + + struct dpaa2_qdma_priv *priv; +}; + +/* + * dpaa2_qdma_priv - driver private data + */ +struct dpaa2_qdma_priv { + int dpqdma_id; + + struct iommu_domain *iommu_domain; + struct dpdmai_attr dpdmai_attr; + struct device *dev; + struct fsl_mc_io *mc_io; + struct fsl_mc_device *dpdmai_dev; + u8 num_pairs; + + struct dpaa2_qdma_engine *dpaa2_qdma; + struct dpaa2_qdma_priv_per_prio *ppriv; + + struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_PRIO_NUM]; + u32 tx_fqid[DPDMAI_PRIO_NUM]; +}; + +struct dpaa2_qdma_priv_per_prio { + int req_fqid; + int rsp_fqid; + int prio; + + struct dpaa2_io_store *store; + struct dpaa2_io_notification_ctx nctx; + + struct dpaa2_qdma_priv *priv; +}; + +static struct soc_device_attribute soc_fixup_tuning[] = { + { .family = "QorIQ LX2160A"}, + { }, +}; + +/* FD pool size: one FD + 3 Frame list + 2 source/destination descriptor */ +#define FD_POOL_SIZE (sizeof(struct dpaa2_fd) + \ + sizeof(struct dpaa2_fl_entry) * 3 + \ + sizeof(struct dpaa2_qdma_sd_d) * 2) + +static void dpaa2_dpdmai_free_channels(struct dpaa2_qdma_engine *dpaa2_qdma); +static void dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan, + struct list_head *head); +#endif /* __DPAA2_QDMA_H */ From 99b765032797063e5fc6da0778668fa5ba3fdef8 Mon Sep 17 00:00:00 2001 From: Peng Ma Date: Thu, 11 Oct 2018 16:49:41 +0800 Subject: [PATCH 49/52] dma: caam: add dma memcpy driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This module introduces a memcpy DMA driver based on the DMA capabilities of the CAAM hardware block. CAAM DMA is a platform driver that is only probed if the device is defined in the device tree. The driver creates a DMA channel for each JR of the CAAM. This introduces a dependency on the JR driver. Therefore a defering mechanism was used to ensure that the CAAM DMA driver is probed only after the JR driver. Signed-off-by: Radu Alexe Signed-off-by: Tudor Ambarus Signed-off-by: Rajiv Vishwakarma Signed-off-by: Horia Geantă --- drivers/dma/Kconfig | 19 +- drivers/dma/Makefile | 1 + drivers/dma/caam_dma.c | 462 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 481 insertions(+), 1 deletion(-) create mode 100644 drivers/dma/caam_dma.c diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 7af874b69ffb..266a384ca21b 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -131,6 +131,24 @@ config COH901318 help Enable support for ST-Ericsson COH 901 318 DMA. +config CRYPTO_DEV_FSL_CAAM_DMA + tristate "CAAM DMA engine support" + depends on CRYPTO_DEV_FSL_CAAM_JR + default n + select DMA_ENGINE + select ASYNC_CORE + select ASYNC_TX_ENABLE_CHANNEL_SWITCH + help + Selecting this will offload the DMA operations for users of + the scatter gather memcopy API to the CAAM via job rings. The + CAAM is a hardware module that provides hardware acceleration to + cryptographic operations. It has a built-in DMA controller that can + be programmed to read/write cryptographic data. This module defines + a DMA driver that uses the DMA capabilities of the CAAM. + + To compile this as a module, choose M here: the module + will be called caam_dma. + config DMA_BCM2835 tristate "BCM2835 DMA engine support" depends on ARCH_BCM2835 @@ -651,7 +669,6 @@ config ZX_DMA help Support the DMA engine for ZTE ZX family platform devices. - # driver files source "drivers/dma/bestcomm/Kconfig" diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index f5ce8665e944..413a7dec7e69 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -75,6 +75,7 @@ obj-$(CONFIG_UNIPHIER_MDMAC) += uniphier-mdmac.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o obj-$(CONFIG_ZX_DMA) += zx_dma.o obj-$(CONFIG_ST_FDMA) += st_fdma.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_DMA) += caam_dma.o obj-y += mediatek/ obj-y += qcom/ diff --git a/drivers/dma/caam_dma.c b/drivers/dma/caam_dma.c new file mode 100644 index 000000000000..fe1e503ff4a9 --- /dev/null +++ b/drivers/dma/caam_dma.c @@ -0,0 +1,462 @@ +/* + * caam support for SG DMA + * + * Copyright 2016 Freescale Semiconductor, Inc + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the names of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include "dmaengine.h" + +#include "../crypto/caam/regs.h" +#include "../crypto/caam/jr.h" +#include "../crypto/caam/error.h" +#include "../crypto/caam/desc_constr.h" + +#define DESC_DMA_MEMCPY_LEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / \ + CAAM_CMD_SZ) + +/* + * This is max chunk size of a DMA transfer. If a buffer is larger than this + * value it is internally broken into chunks of max CAAM_DMA_CHUNK_SIZE bytes + * and for each chunk a DMA transfer request is issued. + * This value is the largest number on 16 bits that is a multiple of 256 bytes + * (the largest configurable CAAM DMA burst size). + */ +#define CAAM_DMA_CHUNK_SIZE 65280 + +struct caam_dma_sh_desc { + u32 desc[DESC_DMA_MEMCPY_LEN] ____cacheline_aligned; + dma_addr_t desc_dma; +}; + +/* caam dma extended descriptor */ +struct caam_dma_edesc { + struct dma_async_tx_descriptor async_tx; + struct list_head node; + struct caam_dma_ctx *ctx; + dma_addr_t src_dma; + dma_addr_t dst_dma; + unsigned int src_len; + unsigned int dst_len; + u32 jd[] ____cacheline_aligned; +}; + +/* + * caam_dma_ctx - per jr/channel context + * @chan: dma channel used by async_tx API + * @node: list_head used to attach to the global dma_ctx_list + * @jrdev: Job Ring device + * @pending_q: queue of pending (submitted, but not enqueued) jobs + * @done_not_acked: jobs that have been completed by jr, but maybe not acked + * @edesc_lock: protects extended descriptor + */ +struct caam_dma_ctx { + struct dma_chan chan; + struct list_head node; + struct device *jrdev; + struct list_head pending_q; + struct list_head done_not_acked; + spinlock_t edesc_lock; +}; + +static struct dma_device *dma_dev; +static struct caam_dma_sh_desc *dma_sh_desc; +static LIST_HEAD(dma_ctx_list); + +static dma_cookie_t caam_dma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct caam_dma_edesc *edesc = NULL; + struct caam_dma_ctx *ctx = NULL; + dma_cookie_t cookie; + + edesc = container_of(tx, struct caam_dma_edesc, async_tx); + ctx = container_of(tx->chan, struct caam_dma_ctx, chan); + + spin_lock_bh(&ctx->edesc_lock); + + cookie = dma_cookie_assign(tx); + list_add_tail(&edesc->node, &ctx->pending_q); + + spin_unlock_bh(&ctx->edesc_lock); + + return cookie; +} + +static void caam_jr_chan_free_edesc(struct caam_dma_edesc *edesc) +{ + struct caam_dma_ctx *ctx = edesc->ctx; + struct caam_dma_edesc *_edesc = NULL; + + spin_lock_bh(&ctx->edesc_lock); + + list_add_tail(&edesc->node, &ctx->done_not_acked); + list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) { + if (async_tx_test_ack(&edesc->async_tx)) { + list_del(&edesc->node); + kfree(edesc); + } + } + + spin_unlock_bh(&ctx->edesc_lock); +} + +static void caam_dma_done(struct device *dev, u32 *hwdesc, u32 err, + void *context) +{ + struct caam_dma_edesc *edesc = context; + struct caam_dma_ctx *ctx = edesc->ctx; + dma_async_tx_callback callback; + void *callback_param; + + if (err) + caam_jr_strstatus(ctx->jrdev, err); + + dma_run_dependencies(&edesc->async_tx); + + spin_lock_bh(&ctx->edesc_lock); + dma_cookie_complete(&edesc->async_tx); + spin_unlock_bh(&ctx->edesc_lock); + + callback = edesc->async_tx.callback; + callback_param = edesc->async_tx.callback_param; + + dma_descriptor_unmap(&edesc->async_tx); + + caam_jr_chan_free_edesc(edesc); + + if (callback) + callback(callback_param); +} + +static void caam_dma_memcpy_init_job_desc(struct caam_dma_edesc *edesc) +{ + u32 *jd = edesc->jd; + u32 *sh_desc = dma_sh_desc->desc; + dma_addr_t desc_dma = dma_sh_desc->desc_dma; + + /* init the job descriptor */ + init_job_desc_shared(jd, desc_dma, desc_len(sh_desc), HDR_REVERSE); + + /* set SEQIN PTR */ + append_seq_in_ptr(jd, edesc->src_dma, edesc->src_len, 0); + + /* set SEQOUT PTR */ + append_seq_out_ptr(jd, edesc->dst_dma, edesc->dst_len, 0); + + print_hex_dump_debug("caam dma desc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, jd, desc_bytes(jd), 1); +} + +static struct dma_async_tx_descriptor * +caam_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct caam_dma_edesc *edesc; + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, + chan); + + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | GFP_NOWAIT); + if (!edesc) + return ERR_PTR(-ENOMEM); + + dma_async_tx_descriptor_init(&edesc->async_tx, chan); + edesc->async_tx.tx_submit = caam_dma_tx_submit; + edesc->async_tx.flags = flags; + edesc->async_tx.cookie = -EBUSY; + + edesc->src_dma = src; + edesc->src_len = len; + edesc->dst_dma = dst; + edesc->dst_len = len; + edesc->ctx = ctx; + + caam_dma_memcpy_init_job_desc(edesc); + + return &edesc->async_tx; +} + +/* This function can be called in an interrupt context */ +static void caam_dma_issue_pending(struct dma_chan *chan) +{ + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, + chan); + struct caam_dma_edesc *edesc, *_edesc; + + spin_lock_bh(&ctx->edesc_lock); + list_for_each_entry_safe(edesc, _edesc, &ctx->pending_q, node) { + if (caam_jr_enqueue(ctx->jrdev, edesc->jd, + caam_dma_done, edesc) < 0) + break; + list_del(&edesc->node); + } + spin_unlock_bh(&ctx->edesc_lock); +} + +static void caam_dma_free_chan_resources(struct dma_chan *chan) +{ + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, + chan); + struct caam_dma_edesc *edesc, *_edesc; + + spin_lock_bh(&ctx->edesc_lock); + list_for_each_entry_safe(edesc, _edesc, &ctx->pending_q, node) { + list_del(&edesc->node); + kfree(edesc); + } + list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) { + list_del(&edesc->node); + kfree(edesc); + } + spin_unlock_bh(&ctx->edesc_lock); +} + +static int caam_dma_jr_chan_bind(void) +{ + struct device *jrdev; + struct caam_dma_ctx *ctx; + int bonds = 0; + int i; + + for (i = 0; i < caam_jr_driver_probed(); i++) { + jrdev = caam_jridx_alloc(i); + if (IS_ERR(jrdev)) { + pr_err("job ring device %d allocation failed\n", i); + continue; + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) { + caam_jr_free(jrdev); + continue; + } + + ctx->chan.device = dma_dev; + ctx->chan.private = ctx; + + ctx->jrdev = jrdev; + + INIT_LIST_HEAD(&ctx->pending_q); + INIT_LIST_HEAD(&ctx->done_not_acked); + INIT_LIST_HEAD(&ctx->node); + spin_lock_init(&ctx->edesc_lock); + + dma_cookie_init(&ctx->chan); + + /* add the context of this channel to the context list */ + list_add_tail(&ctx->node, &dma_ctx_list); + + /* add this channel to the device chan list */ + list_add_tail(&ctx->chan.device_node, &dma_dev->channels); + + bonds++; + } + + return bonds; +} + +static inline void caam_jr_dma_free(struct dma_chan *chan) +{ + struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, + chan); + + list_del(&ctx->node); + list_del(&chan->device_node); + caam_jr_free(ctx->jrdev); + kfree(ctx); +} + +static void set_caam_dma_desc(u32 *desc) +{ + u32 *jmp_cmd; + + /* dma shared descriptor */ + init_sh_desc(desc, HDR_SHARE_NEVER | (1 << HDR_START_IDX_SHIFT)); + + /* REG1 = CAAM_DMA_CHUNK_SIZE */ + append_math_add_imm_u32(desc, REG1, ZERO, IMM, CAAM_DMA_CHUNK_SIZE); + + /* REG0 = SEQINLEN - CAAM_DMA_CHUNK_SIZE */ + append_math_sub_imm_u32(desc, REG0, SEQINLEN, IMM, CAAM_DMA_CHUNK_SIZE); + + /* + * if (REG0 > 0) + * jmp to LABEL1 + */ + jmp_cmd = append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N | + JUMP_COND_MATH_Z); + + /* REG1 = SEQINLEN */ + append_math_sub(desc, REG1, SEQINLEN, ZERO, CAAM_CMD_SZ); + + /* LABEL1 */ + set_jump_tgt_here(desc, jmp_cmd); + + /* VARSEQINLEN = REG1 */ + append_math_add(desc, VARSEQINLEN, REG1, ZERO, CAAM_CMD_SZ); + + /* VARSEQOUTLEN = REG1 */ + append_math_add(desc, VARSEQOUTLEN, REG1, ZERO, CAAM_CMD_SZ); + + /* do FIFO STORE */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_METADATA | LDST_VLF); + + /* do FIFO LOAD */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IFIFO | LDST_VLF); + + /* + * if (REG0 > 0) + * jmp 0xF8 (after shared desc header) + */ + append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N | + JUMP_COND_MATH_Z | 0xF8); + + print_hex_dump_debug("caam dma shdesc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} + +static int caam_dma_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device *ctrldev = dev->parent; + struct dma_chan *chan, *_chan; + u32 *sh_desc; + int err = -ENOMEM; + int bonds; + + if (!caam_jr_driver_probed()) { + dev_info(dev, "Defer probing after JR driver probing\n"); + return -EPROBE_DEFER; + } + + dma_dev = kzalloc(sizeof(*dma_dev), GFP_KERNEL); + if (!dma_dev) + return -ENOMEM; + + dma_sh_desc = kzalloc(sizeof(*dma_sh_desc), GFP_KERNEL | GFP_DMA); + if (!dma_sh_desc) + goto desc_err; + + sh_desc = dma_sh_desc->desc; + set_caam_dma_desc(sh_desc); + dma_sh_desc->desc_dma = dma_map_single(ctrldev, sh_desc, + desc_bytes(sh_desc), + DMA_TO_DEVICE); + if (dma_mapping_error(ctrldev, dma_sh_desc->desc_dma)) { + dev_err(dev, "unable to map dma descriptor\n"); + goto map_err; + } + + INIT_LIST_HEAD(&dma_dev->channels); + + bonds = caam_dma_jr_chan_bind(); + if (!bonds) { + err = -ENODEV; + goto jr_bind_err; + } + + dma_dev->dev = dev; + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); + dma_dev->device_tx_status = dma_cookie_status; + dma_dev->device_issue_pending = caam_dma_issue_pending; + dma_dev->device_prep_dma_memcpy = caam_dma_prep_memcpy; + dma_dev->device_free_chan_resources = caam_dma_free_chan_resources; + + err = dma_async_device_register(dma_dev); + if (err) { + dev_err(dev, "Failed to register CAAM DMA engine\n"); + goto jr_bind_err; + } + + dev_info(dev, "caam dma support with %d job rings\n", bonds); + + return err; + +jr_bind_err: + list_for_each_entry_safe(chan, _chan, &dma_dev->channels, device_node) + caam_jr_dma_free(chan); + + dma_unmap_single(ctrldev, dma_sh_desc->desc_dma, desc_bytes(sh_desc), + DMA_TO_DEVICE); +map_err: + kfree(dma_sh_desc); +desc_err: + kfree(dma_dev); + return err; +} + +static int caam_dma_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device *ctrldev = dev->parent; + struct caam_dma_ctx *ctx, *_ctx; + + dma_async_device_unregister(dma_dev); + + list_for_each_entry_safe(ctx, _ctx, &dma_ctx_list, node) { + list_del(&ctx->node); + caam_jr_free(ctx->jrdev); + kfree(ctx); + } + + dma_unmap_single(ctrldev, dma_sh_desc->desc_dma, + desc_bytes(dma_sh_desc->desc), DMA_TO_DEVICE); + + kfree(dma_sh_desc); + kfree(dma_dev); + + dev_info(dev, "caam dma support disabled\n"); + return 0; +} + +static struct platform_driver caam_dma_driver = { + .driver = { + .name = "caam-dma", + }, + .probe = caam_dma_probe, + .remove = caam_dma_remove, +}; +module_platform_driver(caam_dma_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("NXP CAAM support for DMA engine"); +MODULE_AUTHOR("NXP Semiconductors"); +MODULE_ALIAS("platform:caam-dma"); From 5393bb7ad5b7be2c252312f00e4b463379ae0c4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 4 Oct 2019 14:07:03 +0300 Subject: [PATCH 50/52] dma: caam: fix compilation error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix compilation error, introduced by incorrect rebase of the commit 9c51c141264c ("dma: caam: add dma memcpy driver" on top of upstream commit 1bcdf5a00f41 ("crypto: caam - make CAAM_PTR_SZ dynamic") Fixes: 9c51c141264c ("dma: caam: add dma memcpy driver") Signed-off-by: Horia Geantă --- drivers/dma/caam_dma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/caam_dma.c b/drivers/dma/caam_dma.c index fe1e503ff4a9..c95b9679694e 100644 --- a/drivers/dma/caam_dma.c +++ b/drivers/dma/caam_dma.c @@ -47,7 +47,7 @@ #include "../crypto/caam/error.h" #include "../crypto/caam/desc_constr.h" -#define DESC_DMA_MEMCPY_LEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / \ +#define DESC_DMA_MEMCPY_LEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN) / \ CAAM_CMD_SZ) /* From 122200c5fa6ff3a1228e48b831f681a3ca60e5c8 Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Fri, 29 Nov 2019 04:11:08 +0800 Subject: [PATCH 51/52] LF-246: dmaengine: imx-sdma: correct is_ram_script checking Correct is_ram_script checking in case sdma firmware not loaded as expected, otherwise all scripts are considered as rom script without correct "sdma firmware not ready!" since sdma->ram_code_start is 0. Note: only add the doubtless is_ram_script for sdma-imx6q.bin/sdma-imx7d. bin , and leave the legacy i.mx5x/i.mx3x/i.mx2x firmware alone since not sure which of the remain scripts should be in ram. Signed-off-by: Robin Gong Reviewed-by: Shengjiu Wang --- drivers/dma/imx-sdma.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index c2d6d49af4de..d5abe81a26ee 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1023,6 +1023,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_CSPI: per_2_emi = sdma->script_addrs->app_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_ecspi_addr; + sdmac->is_ram_script = true; break; case IMX_DMATYPE_EXT: case IMX_DMATYPE_SSI: @@ -1033,6 +1034,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_SSI_DUAL: per_2_emi = sdma->script_addrs->ssish_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_ssish_addr; + sdmac->is_ram_script = true; break; case IMX_DMATYPE_SSI_SP: case IMX_DMATYPE_MMC: @@ -1047,11 +1049,13 @@ static void sdma_get_pc(struct sdma_channel *sdmac, per_2_emi = sdma->script_addrs->asrc_2_mcu_addr; emi_2_per = sdma->script_addrs->asrc_2_mcu_addr; per_2_per = sdma->script_addrs->per_2_per_addr; + sdmac->is_ram_script = true; break; case IMX_DMATYPE_ASRC_SP: per_2_emi = sdma->script_addrs->shp_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_shp_addr; per_2_per = sdma->script_addrs->per_2_per_addr; + sdmac->is_ram_script = true; break; case IMX_DMATYPE_MSHC: per_2_emi = sdma->script_addrs->mshc_2_mcu_addr; @@ -1069,10 +1073,12 @@ static void sdma_get_pc(struct sdma_channel *sdmac, break; case IMX_DMATYPE_HDMI: emi_2_per = sdma->script_addrs->hdmi_dma_addr; + sdmac->is_ram_script = true; break; case IMX_DMATYPE_MULTI_SAI: per_2_emi = sdma->script_addrs->sai_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_sai_addr; + sdmac->is_ram_script = true; default: break; } @@ -1081,13 +1087,6 @@ static void sdma_get_pc(struct sdma_channel *sdmac, sdmac->pc_to_device = emi_2_per; sdmac->device_to_device = per_2_per; sdmac->pc_to_pc = emi_2_emi; - - if (sdma->ram_code_start && - ((sdmac->pc_from_device >= sdma->ram_code_start) || - (sdmac->pc_to_device >= sdma->ram_code_start) || - (sdmac->device_to_device >= sdma->ram_code_start || - (sdmac->pc_to_pc >= sdma->ram_code_start)))) - sdmac->is_ram_script = true; } static int sdma_load_context(struct sdma_channel *sdmac) @@ -1804,6 +1803,7 @@ static int sdma_config_write(struct dma_chan *chan, struct sdma_channel *sdmac = to_sdma_chan(chan); sdmac->watermark_level = 0; + sdmac->is_ram_script = false; if (direction == DMA_DEV_TO_MEM) { sdmac->per_address = dmaengine_cfg->src_addr; From 5ebe728cc7de44282212956b9187c8f1155fb99a Mon Sep 17 00:00:00 2001 From: Robin Gong Date: Tue, 3 Dec 2019 01:38:40 +0800 Subject: [PATCH 52/52] LF-301: dmaengine: imx-sdma: Add once more loading firmware In case 60 seconds maybe not enough for Yocto loading sdma firmware on some poor performance chips such as i.mx6sll in nfs case, add another round to load firmware. Signed-off-by: Robin Gong Reviewed-by: Fugang Duan --- drivers/dma/imx-sdma.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index d5abe81a26ee..dd9b3ff9f7bf 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -481,6 +481,7 @@ struct sdma_engine { bool clk_ratio; struct gen_pool *iram_pool; bool fw_loaded; + u32 fw_fail; unsigned short ram_code_start; }; @@ -1940,8 +1941,17 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) unsigned short *ram_code; if (!fw) { - dev_info(sdma->dev, "external firmware not found, using ROM firmware\n"); - /* In this case we just use the ROM firmware. */ + /* Load firmware once more time if timeout */ + if (sdma->fw_fail) + dev_info(sdma->dev, "external firmware not found, using ROM firmware\n"); + else { + request_firmware_nowait(THIS_MODULE, + FW_ACTION_HOTPLUG, sdma->fw_name, + sdma->dev, GFP_KERNEL, sdma, + sdma_load_firmware); + sdma->fw_fail++; + } + return; }