From b1d2dc009dece4cd7e629419b52266ba51960a6b Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Thu, 23 May 2019 21:06:32 -0700 Subject: [PATCH 01/29] dma-contiguous: add dma_{alloc,free}_contiguous() helpers Both dma_alloc_from_contiguous() and dma_release_from_contiguous() are very simply implemented, but requiring callers to pass certain parameters like count and align, and taking a boolean parameter to check __GFP_NOWARN in the allocation flags. So every function call duplicates similar work: unsigned long order = get_order(size); size_t count = size >> PAGE_SHIFT; page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN); [...] dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); Additionally, as CMA can be used only in the context which permits sleeping, most of callers do a gfpflags_allow_blocking() check and a corresponding fallback allocation of normal pages upon any false result: if (gfpflags_allow_blocking(flag)) page = dma_alloc_from_contiguous(); if (!page) page = alloc_pages(); [...] if (!dma_release_from_contiguous(dev, page, count)) __free_pages(page, get_order(size)); So this patch simplifies those function calls by abstracting these operations into the two new functions: dma_{alloc,free}_contiguous. As some callers of dma_{alloc,release}_from_contiguous() might be complicated, this patch just implements these two new functions to kernel/dma/direct.c only as an initial step. Suggested-by: Christoph Hellwig Signed-off-by: Nicolin Chen Tested-by: dann frazier Signed-off-by: Christoph Hellwig --- include/linux/dma-contiguous.h | 13 ++++++++++ kernel/dma/contiguous.c | 47 ++++++++++++++++++++++++++++++++++ kernel/dma/direct.c | 24 +++-------------- 3 files changed, 64 insertions(+), 20 deletions(-) diff --git a/include/linux/dma-contiguous.h b/include/linux/dma-contiguous.h index 6665fa03c0d1..428f3b7b1c42 100644 --- a/include/linux/dma-contiguous.h +++ b/include/linux/dma-contiguous.h @@ -111,6 +111,8 @@ struct page *dma_alloc_from_contiguous(struct device *dev, size_t count, unsigned int order, bool no_warn); bool dma_release_from_contiguous(struct device *dev, struct page *pages, int count); +struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp); +void dma_free_contiguous(struct device *dev, struct page *page, size_t size); #else @@ -153,6 +155,17 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, return false; } +static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size, + gfp_t gfp) +{ + return NULL; +} + +static inline void dma_free_contiguous(struct device *dev, struct page *page, + size_t size) +{ +} + #endif #endif diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index b2a87905846d..637b120d647b 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -214,6 +214,53 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, return cma_release(dev_get_cma_area(dev), pages, count); } +/** + * dma_alloc_contiguous() - allocate contiguous pages + * @dev: Pointer to device for which the allocation is performed. + * @size: Requested allocation size. + * @gfp: Allocation flags. + * + * This function allocates contiguous memory buffer for specified device. It + * first tries to use device specific contiguous memory area if available or + * the default global one, then tries a fallback allocation of normal pages. + */ +struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) +{ + int node = dev ? dev_to_node(dev) : NUMA_NO_NODE; + size_t count = PAGE_ALIGN(size) >> PAGE_SHIFT; + size_t align = get_order(PAGE_ALIGN(size)); + struct cma *cma = dev_get_cma_area(dev); + struct page *page = NULL; + + /* CMA can be used only in the context which permits sleeping */ + if (cma && gfpflags_allow_blocking(gfp)) { + align = min_t(size_t, align, CONFIG_CMA_ALIGNMENT); + page = cma_alloc(cma, count, align, gfp & __GFP_NOWARN); + } + + /* Fallback allocation of normal pages */ + if (!page) + page = alloc_pages_node(node, gfp, align); + return page; +} + +/** + * dma_free_contiguous() - release allocated pages + * @dev: Pointer to device for which the pages were allocated. + * @page: Pointer to the allocated pages. + * @size: Size of allocated pages. + * + * This function releases memory allocated by dma_alloc_contiguous(). As the + * cma_release returns false when provided pages do not belong to contiguous + * area and true otherwise, this function then does a fallback __free_pages() + * upon a false-return. + */ +void dma_free_contiguous(struct device *dev, struct page *page, size_t size) +{ + if (!cma_release(dev_get_cma_area(dev), page, size >> PAGE_SHIFT)) + __free_pages(page, get_order(size)); +} + /* * Support for reserved memory regions defined in device tree */ diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 2c2772e9702a..0816c1e8b05a 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -96,8 +96,6 @@ static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { - unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; - int page_order = get_order(size); struct page *page = NULL; u64 phys_mask; @@ -109,20 +107,9 @@ struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask, &phys_mask); again: - /* CMA can be used only in the context which permits sleeping */ - if (gfpflags_allow_blocking(gfp)) { - page = dma_alloc_from_contiguous(dev, count, page_order, - gfp & __GFP_NOWARN); - if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { - dma_release_from_contiguous(dev, page, count); - page = NULL; - } - } - if (!page) - page = alloc_pages_node(dev_to_node(dev), gfp, page_order); - + page = dma_alloc_contiguous(dev, size, gfp); if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { - __free_pages(page, page_order); + dma_free_contiguous(dev, page, size); page = NULL; if (IS_ENABLED(CONFIG_ZONE_DMA32) && @@ -154,7 +141,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, if (PageHighMem(page)) { /* * Depending on the cma= arguments and per-arch setup - * dma_alloc_from_contiguous could return highmem pages. + * dma_alloc_contiguous could return highmem pages. * Without remapping there is no way to return them here, * so log an error and fail. */ @@ -176,10 +163,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, void __dma_direct_free_pages(struct device *dev, size_t size, struct page *page) { - unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; - - if (!dma_release_from_contiguous(dev, page, count)) - __free_pages(page, get_order(size)); + dma_free_contiguous(dev, page, size); } void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, From bd2e75633c8012fc8a7431c82fda66237133bf7e Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Thu, 23 May 2019 21:06:33 -0700 Subject: [PATCH 02/29] dma-contiguous: use fallback alloc_pages for single pages The addresses within a single page are always contiguous, so it's not so necessary to always allocate one single page from CMA area. Since the CMA area has a limited predefined size of space, it may run out of space in heavy use cases, where there might be quite a lot CMA pages being allocated for single pages. However, there is also a concern that a device might care where a page comes from -- it might expect the page from CMA area and act differently if the page doesn't. This patch tries to use the fallback alloc_pages path, instead of one-page size allocations from the global CMA area in case that a device does not have its own CMA area. This'd save resources from the CMA global area for more CMA allocations, and also reduce CMA fragmentations resulted from trivial allocations. Signed-off-by: Nicolin Chen Tested-by: dann frazier Signed-off-by: Christoph Hellwig --- kernel/dma/contiguous.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 637b120d647b..bfc0c17f2a3d 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -223,14 +223,23 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, * This function allocates contiguous memory buffer for specified device. It * first tries to use device specific contiguous memory area if available or * the default global one, then tries a fallback allocation of normal pages. + * + * Note that it byapss one-page size of allocations from the global area as + * the addresses within one page are always contiguous, so there is no need + * to waste CMA pages for that kind; it also helps reduce fragmentations. */ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) { int node = dev ? dev_to_node(dev) : NUMA_NO_NODE; size_t count = PAGE_ALIGN(size) >> PAGE_SHIFT; size_t align = get_order(PAGE_ALIGN(size)); - struct cma *cma = dev_get_cma_area(dev); struct page *page = NULL; + struct cma *cma = NULL; + + if (dev && dev->cma_area) + cma = dev->cma_area; + else if (count > 1) + cma = dma_contiguous_default_area; /* CMA can be used only in the context which permits sleeping */ if (cma && gfpflags_allow_blocking(gfp)) { From dd3dcede9fa0a0b661ac1f24843f4a1b1317fdb6 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 29 May 2019 17:54:25 -0700 Subject: [PATCH 03/29] dma-contiguous: fix !CONFIG_DMA_CMA version of dma_{alloc, free}_contiguous() Commit fdaeec198ada ("dma-contiguous: add dma_{alloc,free}_contiguous() helpers") adds a pair of new helper functions so as to abstract code in the dma-direct (and other places in the future), however it breaks QEMU boot feature using x86_64 defconfig. That's because x86_64 defconfig has CONFIG_DMA_CMA=n so those two newly introduced helper functions are empty in their !CONFIG_DMA_CMA version, while previously the platform independent dma-direct code had fallback alloc_pages_node() and __free_pages(). So this patch fixes it by adding alloc_pages_node() and __free_pages() in the !CONFIG_DMA_CMA version of the two helper functions. Tested with below QEMU command: qemu-system-x86_64 -m 512m \ -drive file=images/x86_64/rootfs.ext4,format=raw,if=ide \ -append 'console=ttyS0 root=/dev/sda' -nographic \ -kernel arch/x86_64/boot/bzImage with the rootfs from the below link: https://github.com/ClangBuiltLinux/continuous-integration/raw/master/images/x86_64/rootfs.ext4 Fixes: fdaeec198ada ("dma-contiguous: add dma_{alloc,free}_contiguous() helpers") Reported-by: Nathan Chancellor Signed-off-by: Nicolin Chen Signed-off-by: Christoph Hellwig --- include/linux/dma-contiguous.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/include/linux/dma-contiguous.h b/include/linux/dma-contiguous.h index 428f3b7b1c42..c05d4e661489 100644 --- a/include/linux/dma-contiguous.h +++ b/include/linux/dma-contiguous.h @@ -50,6 +50,7 @@ #ifdef __KERNEL__ #include +#include struct cma; struct page; @@ -155,15 +156,20 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, return false; } +/* Use fallback alloc() and free() when CONFIG_DMA_CMA=n */ static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) { - return NULL; + int node = dev ? dev_to_node(dev) : NUMA_NO_NODE; + size_t align = get_order(PAGE_ALIGN(size)); + + return alloc_pages_node(node, gfp, align); } static inline void dma_free_contiguous(struct device *dev, struct page *page, size_t size) { + __free_pages(page, get_order(size)); } #endif From da83a722959a82733c3ca60030cc364ca2318c5a Mon Sep 17 00:00:00 2001 From: Fredrik Noring Date: Wed, 29 May 2019 13:28:39 +0300 Subject: [PATCH 04/29] lib/genalloc: add gen_pool_dma_zalloc() for zeroed DMA allocations gen_pool_dma_zalloc() is a zeroed memory variant of gen_pool_dma_alloc(). Also document the return values of both, and indicate NULL as a "%NULL" constant. Signed-off-by: Fredrik Noring Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- include/linux/genalloc.h | 1 + lib/genalloc.c | 29 ++++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/include/linux/genalloc.h b/include/linux/genalloc.h index dd0a452373e7..6c62eeca754f 100644 --- a/include/linux/genalloc.h +++ b/include/linux/genalloc.h @@ -121,6 +121,7 @@ extern unsigned long gen_pool_alloc_algo(struct gen_pool *, size_t, genpool_algo_t algo, void *data); extern void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); +void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); extern void gen_pool_free(struct gen_pool *, unsigned long, size_t); extern void gen_pool_for_each_chunk(struct gen_pool *, void (*)(struct gen_pool *, struct gen_pool_chunk *, void *), void *); diff --git a/lib/genalloc.c b/lib/genalloc.c index 7e85d1e37a6e..5db43476a19b 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -337,12 +337,14 @@ EXPORT_SYMBOL(gen_pool_alloc_algo); * gen_pool_dma_alloc - allocate special memory from the pool for DMA usage * @pool: pool to allocate from * @size: number of bytes to allocate from the pool - * @dma: dma-view physical address return value. Use NULL if unneeded. + * @dma: dma-view physical address return value. Use %NULL if unneeded. * * Allocate the requested number of bytes from the specified pool. * Uses the pool allocation function (with first-fit algorithm by default). * Can not be used in NMI handler on architectures without * NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated memory, or %NULL on failure */ void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) { @@ -362,6 +364,31 @@ void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) } EXPORT_SYMBOL(gen_pool_dma_alloc); +/** + * gen_pool_dma_zalloc - allocate special zeroed memory from the pool for + * DMA usage + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: dma-view physical address return value. Use %NULL if unneeded. + * + * Allocate the requested number of zeroed bytes from the specified pool. + * Uses the pool allocation function (with first-fit algorithm by default). + * Can not be used in NMI handler on architectures without + * NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated zeroed memory, or %NULL on failure + */ +void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) +{ + void *vaddr = gen_pool_dma_alloc(pool, size, dma); + + if (vaddr) + memset(vaddr, 0, size); + + return vaddr; +} +EXPORT_SYMBOL(gen_pool_dma_zalloc); + /** * gen_pool_free - free allocated special memory back to the pool * @pool: pool to free to From b0310c2f09bbe8aebefb97ed67949a3a7092aca6 Mon Sep 17 00:00:00 2001 From: Laurentiu Tudor Date: Wed, 29 May 2019 13:28:40 +0300 Subject: [PATCH 05/29] USB: use genalloc for USB HCs with local memory For HCs that have local memory, replace the current DMA API usage with a genalloc generic allocator to manage the mappings for these devices. To help users, introduce a new HCD API, usb_hcd_setup_local_mem() that will setup up the genalloc backing up the device local memory. It will be used in subsequent patches. This is in preparation for dropping the existing "coherent" dma mem declaration APIs. The current implementation was relying on a short circuit in the DMA API that in the end, was acting as an allocator for these type of devices. Signed-off-by: Laurentiu Tudor Tested-by: Fredrik Noring Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- drivers/usb/Kconfig | 1 + drivers/usb/core/buffer.c | 9 +++++++++ drivers/usb/core/hcd.c | 36 ++++++++++++++++++++++++++++++++++++ drivers/usb/host/ohci-hcd.c | 23 ++++++++++++++++++----- drivers/usb/host/ohci-mem.c | 35 +++++++++++++++++++++++++++++++---- drivers/usb/host/ohci.h | 2 ++ include/linux/usb/hcd.h | 5 +++++ 7 files changed, 102 insertions(+), 9 deletions(-) diff --git a/drivers/usb/Kconfig b/drivers/usb/Kconfig index e4b27413f528..389c57d8eba7 100644 --- a/drivers/usb/Kconfig +++ b/drivers/usb/Kconfig @@ -45,6 +45,7 @@ config USB_ARCH_HAS_HCD config USB tristate "Support for Host-side USB" depends on USB_ARCH_HAS_HCD + select GENERIC_ALLOCATOR select USB_COMMON select NLS # for UTF-8 strings ---help--- diff --git a/drivers/usb/core/buffer.c b/drivers/usb/core/buffer.c index f641342cdec0..d2064ad7ad14 100644 --- a/drivers/usb/core/buffer.c +++ b/drivers/usb/core/buffer.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -124,6 +125,9 @@ void *hcd_buffer_alloc( if (size == 0) return NULL; + if (hcd->localmem_pool) + return gen_pool_dma_alloc(hcd->localmem_pool, size, dma); + /* some USB hosts just use PIO */ if (!IS_ENABLED(CONFIG_HAS_DMA) || (!is_device_dma_capable(bus->sysdev) && @@ -152,6 +156,11 @@ void hcd_buffer_free( if (!addr) return; + if (hcd->localmem_pool) { + gen_pool_free(hcd->localmem_pool, (unsigned long)addr, size); + return; + } + if (!IS_ENABLED(CONFIG_HAS_DMA) || (!is_device_dma_capable(bus->sysdev) && !(hcd->driver->flags & HCD_LOCAL_MEM))) { diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index 94d22551fc1b..29b96e5e8621 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -29,6 +29,8 @@ #include #include #include +#include +#include #include #include @@ -3039,6 +3041,40 @@ usb_hcd_platform_shutdown(struct platform_device *dev) } EXPORT_SYMBOL_GPL(usb_hcd_platform_shutdown); +int usb_hcd_setup_local_mem(struct usb_hcd *hcd, phys_addr_t phys_addr, + dma_addr_t dma, size_t size) +{ + int err; + void *local_mem; + + hcd->localmem_pool = devm_gen_pool_create(hcd->self.sysdev, PAGE_SHIFT, + dev_to_node(hcd->self.sysdev), + dev_name(hcd->self.sysdev)); + if (IS_ERR(hcd->localmem_pool)) + return PTR_ERR(hcd->localmem_pool); + + local_mem = devm_memremap(hcd->self.sysdev, phys_addr, + size, MEMREMAP_WC); + if (!local_mem) + return -ENOMEM; + + /* + * Here we pass a dma_addr_t but the arg type is a phys_addr_t. + * It's not backed by system memory and thus there's no kernel mapping + * for it. + */ + err = gen_pool_add_virt(hcd->localmem_pool, (unsigned long)local_mem, + dma, size, dev_to_node(hcd->self.sysdev)); + if (err < 0) { + dev_err(hcd->self.sysdev, "gen_pool_add_virt failed with %d\n", + err); + return err; + } + + return 0; +} +EXPORT_SYMBOL_GPL(usb_hcd_setup_local_mem); + /*-------------------------------------------------------------------------*/ #if IS_ENABLED(CONFIG_USB_MON) diff --git a/drivers/usb/host/ohci-hcd.c b/drivers/usb/host/ohci-hcd.c index 210181fd98d2..b200b19b44fa 100644 --- a/drivers/usb/host/ohci-hcd.c +++ b/drivers/usb/host/ohci-hcd.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -505,8 +506,15 @@ static int ohci_init (struct ohci_hcd *ohci) timer_setup(&ohci->io_watchdog, io_watchdog_func, 0); ohci->prev_frame_no = IO_WATCHDOG_OFF; - ohci->hcca = dma_alloc_coherent (hcd->self.controller, - sizeof(*ohci->hcca), &ohci->hcca_dma, GFP_KERNEL); + if (hcd->localmem_pool) + ohci->hcca = gen_pool_dma_alloc(hcd->localmem_pool, + sizeof(*ohci->hcca), + &ohci->hcca_dma); + else + ohci->hcca = dma_alloc_coherent(hcd->self.controller, + sizeof(*ohci->hcca), + &ohci->hcca_dma, + GFP_KERNEL); if (!ohci->hcca) return -ENOMEM; @@ -990,9 +998,14 @@ static void ohci_stop (struct usb_hcd *hcd) remove_debug_files (ohci); ohci_mem_cleanup (ohci); if (ohci->hcca) { - dma_free_coherent (hcd->self.controller, - sizeof *ohci->hcca, - ohci->hcca, ohci->hcca_dma); + if (hcd->localmem_pool) + gen_pool_free(hcd->localmem_pool, + (unsigned long)ohci->hcca, + sizeof(*ohci->hcca)); + else + dma_free_coherent(hcd->self.controller, + sizeof(*ohci->hcca), + ohci->hcca, ohci->hcca_dma); ohci->hcca = NULL; ohci->hcca_dma = 0; } diff --git a/drivers/usb/host/ohci-mem.c b/drivers/usb/host/ohci-mem.c index 3965ac0341eb..4afe27cc7e46 100644 --- a/drivers/usb/host/ohci-mem.c +++ b/drivers/usb/host/ohci-mem.c @@ -36,6 +36,13 @@ static void ohci_hcd_init (struct ohci_hcd *ohci) static int ohci_mem_init (struct ohci_hcd *ohci) { + /* + * HCs with local memory allocate from localmem_pool so there's + * no need to create the below dma pools. + */ + if (ohci_to_hcd(ohci)->localmem_pool) + return 0; + ohci->td_cache = dma_pool_create ("ohci_td", ohci_to_hcd(ohci)->self.controller, sizeof (struct td), @@ -84,8 +91,12 @@ td_alloc (struct ohci_hcd *hc, gfp_t mem_flags) { dma_addr_t dma; struct td *td; + struct usb_hcd *hcd = ohci_to_hcd(hc); - td = dma_pool_zalloc (hc->td_cache, mem_flags, &dma); + if (hcd->localmem_pool) + td = gen_pool_dma_zalloc(hcd->localmem_pool, sizeof(*td), &dma); + else + td = dma_pool_zalloc(hc->td_cache, mem_flags, &dma); if (td) { /* in case hc fetches it, make it look dead */ td->hwNextTD = cpu_to_hc32 (hc, dma); @@ -99,6 +110,7 @@ static void td_free (struct ohci_hcd *hc, struct td *td) { struct td **prev = &hc->td_hash [TD_HASH_FUNC (td->td_dma)]; + struct usb_hcd *hcd = ohci_to_hcd(hc); while (*prev && *prev != td) prev = &(*prev)->td_hash; @@ -106,7 +118,12 @@ td_free (struct ohci_hcd *hc, struct td *td) *prev = td->td_hash; else if ((td->hwINFO & cpu_to_hc32(hc, TD_DONE)) != 0) ohci_dbg (hc, "no hash for td %p\n", td); - dma_pool_free (hc->td_cache, td, td->td_dma); + + if (hcd->localmem_pool) + gen_pool_free(hcd->localmem_pool, (unsigned long)td, + sizeof(*td)); + else + dma_pool_free(hc->td_cache, td, td->td_dma); } /*-------------------------------------------------------------------------*/ @@ -117,8 +134,12 @@ ed_alloc (struct ohci_hcd *hc, gfp_t mem_flags) { dma_addr_t dma; struct ed *ed; + struct usb_hcd *hcd = ohci_to_hcd(hc); - ed = dma_pool_zalloc (hc->ed_cache, mem_flags, &dma); + if (hcd->localmem_pool) + ed = gen_pool_dma_zalloc(hcd->localmem_pool, sizeof(*ed), &dma); + else + ed = dma_pool_zalloc(hc->ed_cache, mem_flags, &dma); if (ed) { INIT_LIST_HEAD (&ed->td_list); ed->dma = dma; @@ -129,6 +150,12 @@ ed_alloc (struct ohci_hcd *hc, gfp_t mem_flags) static void ed_free (struct ohci_hcd *hc, struct ed *ed) { - dma_pool_free (hc->ed_cache, ed, ed->dma); + struct usb_hcd *hcd = ohci_to_hcd(hc); + + if (hcd->localmem_pool) + gen_pool_free(hcd->localmem_pool, (unsigned long)ed, + sizeof(*ed)); + else + dma_pool_free(hc->ed_cache, ed, ed->dma); } diff --git a/drivers/usb/host/ohci.h b/drivers/usb/host/ohci.h index ef4813bfc5bf..b015b00774b2 100644 --- a/drivers/usb/host/ohci.h +++ b/drivers/usb/host/ohci.h @@ -385,6 +385,8 @@ struct ohci_hcd { /* * memory management for queue data structures + * + * @td_cache and @ed_cache are %NULL if &usb_hcd.localmem_pool is used. */ struct dma_pool *td_cache; struct dma_pool *ed_cache; diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index bb57b5af4700..127560a4bfa0 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -216,6 +216,9 @@ struct usb_hcd { #define HC_IS_RUNNING(state) ((state) & __ACTIVE) #define HC_IS_SUSPENDED(state) ((state) & __SUSPEND) + /* memory pool for HCs having local memory, or %NULL */ + struct gen_pool *localmem_pool; + /* more shared queuing code would be good; it should support * smarter scheduling, handle transaction translators, etc; * input size of periodic table to an interrupt scheduler. @@ -461,6 +464,8 @@ extern int usb_add_hcd(struct usb_hcd *hcd, unsigned int irqnum, unsigned long irqflags); extern void usb_remove_hcd(struct usb_hcd *hcd); extern int usb_hcd_find_raw_port_number(struct usb_hcd *hcd, int port1); +int usb_hcd_setup_local_mem(struct usb_hcd *hcd, phys_addr_t phys_addr, + dma_addr_t dma, size_t size); struct platform_device; extern void usb_hcd_platform_shutdown(struct platform_device *dev); From 7d9e6f5aebe8c03f1a5199ca5c30f0c53042af23 Mon Sep 17 00:00:00 2001 From: Laurentiu Tudor Date: Wed, 29 May 2019 13:28:41 +0300 Subject: [PATCH 06/29] usb: host: ohci-sm501: init genalloc for local memory In preparation for dropping the existing "coherent" dma mem declaration APIs, replace the current dma_declare_coherent_memory() based mechanism with the creation of a genalloc pool that will be used in the OHCI subsystem as replacement for the DMA APIs. Signed-off-by: Laurentiu Tudor Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- drivers/usb/host/ohci-sm501.c | 47 +++++++++++++++-------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/drivers/usb/host/ohci-sm501.c b/drivers/usb/host/ohci-sm501.c index c26228c25f99..b710e100aec9 100644 --- a/drivers/usb/host/ohci-sm501.c +++ b/drivers/usb/host/ohci-sm501.c @@ -110,40 +110,18 @@ static int ohci_hcd_sm501_drv_probe(struct platform_device *pdev) goto err0; } - /* The sm501 chip is equipped with local memory that may be used - * by on-chip devices such as the video controller and the usb host. - * This driver uses dma_declare_coherent_memory() to make sure - * usb allocations with dma_alloc_coherent() allocate from - * this local memory. The dma_handle returned by dma_alloc_coherent() - * will be an offset starting from 0 for the first local memory byte. - * - * So as long as data is allocated using dma_alloc_coherent() all is - * fine. This is however not always the case - buffers may be allocated - * using kmalloc() - so the usb core needs to be told that it must copy - * data into our local memory if the buffers happen to be placed in - * regular memory. The HCD_LOCAL_MEM flag does just that. - */ - - retval = dma_declare_coherent_memory(dev, mem->start, - mem->start - mem->parent->start, - resource_size(mem)); - if (retval) { - dev_err(dev, "cannot declare coherent memory\n"); - goto err1; - } - /* allocate, reserve and remap resources for registers */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (res == NULL) { dev_err(dev, "no resource definition for registers\n"); retval = -ENOENT; - goto err2; + goto err1; } hcd = usb_create_hcd(driver, &pdev->dev, dev_name(&pdev->dev)); if (!hcd) { retval = -ENOMEM; - goto err2; + goto err1; } hcd->rsrc_start = res->start; @@ -164,6 +142,24 @@ static int ohci_hcd_sm501_drv_probe(struct platform_device *pdev) ohci_hcd_init(hcd_to_ohci(hcd)); + /* The sm501 chip is equipped with local memory that may be used + * by on-chip devices such as the video controller and the usb host. + * This driver uses genalloc so that usb allocations with + * gen_pool_dma_alloc() allocate from this local memory. The dma_handle + * returned by gen_pool_dma_alloc() will be an offset starting from 0 + * for the first local memory byte. + * + * So as long as data is allocated using gen_pool_dma_alloc() all is + * fine. This is however not always the case - buffers may be allocated + * using kmalloc() - so the usb core needs to be told that it must copy + * data into our local memory if the buffers happen to be placed in + * regular memory. The HCD_LOCAL_MEM flag does just that. + */ + + if (usb_hcd_setup_local_mem(hcd, mem->start, + mem->start - mem->parent->start, + resource_size(mem)) < 0) + goto err5; retval = usb_add_hcd(hcd, irq, IRQF_SHARED); if (retval) goto err5; @@ -181,8 +177,6 @@ err4: release_mem_region(hcd->rsrc_start, hcd->rsrc_len); err3: usb_put_hcd(hcd); -err2: - dma_release_declared_memory(dev); err1: release_mem_region(mem->start, resource_size(mem)); err0: @@ -197,7 +191,6 @@ static int ohci_hcd_sm501_drv_remove(struct platform_device *pdev) usb_remove_hcd(hcd); release_mem_region(hcd->rsrc_start, hcd->rsrc_len); usb_put_hcd(hcd); - dma_release_declared_memory(&pdev->dev); mem = platform_get_resource(pdev, IORESOURCE_MEM, 1); if (mem) release_mem_region(mem->start, resource_size(mem)); From 7d99532955d4649ba1417fb5a80252e0ce40f1e2 Mon Sep 17 00:00:00 2001 From: Laurentiu Tudor Date: Wed, 29 May 2019 13:28:42 +0300 Subject: [PATCH 07/29] usb: host: ohci-tmio: init genalloc for local memory In preparation for dropping the existing "coherent" dma mem declaration APIs, replace the current dma_declare_coherent_memory() based mechanism with the creation of a genalloc pool that will be used in the OHCI subsystem as replacement for the DMA APIs. Signed-off-by: Laurentiu Tudor Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- drivers/usb/host/ohci-tmio.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/usb/host/ohci-tmio.c b/drivers/usb/host/ohci-tmio.c index f88a0370659f..3b84ce0c3f29 100644 --- a/drivers/usb/host/ohci-tmio.c +++ b/drivers/usb/host/ohci-tmio.c @@ -224,11 +224,6 @@ static int ohci_hcd_tmio_drv_probe(struct platform_device *dev) goto err_ioremap_regs; } - ret = dma_declare_coherent_memory(&dev->dev, sram->start, sram->start, - resource_size(sram)); - if (ret) - goto err_dma_declare; - if (cell->enable) { ret = cell->enable(dev); if (ret) @@ -239,6 +234,11 @@ static int ohci_hcd_tmio_drv_probe(struct platform_device *dev) ohci = hcd_to_ohci(hcd); ohci_hcd_init(ohci); + ret = usb_hcd_setup_local_mem(hcd, sram->start, sram->start, + resource_size(sram)); + if (ret < 0) + goto err_enable; + ret = usb_add_hcd(hcd, irq, 0); if (ret) goto err_add_hcd; @@ -254,8 +254,6 @@ err_add_hcd: if (cell->disable) cell->disable(dev); err_enable: - dma_release_declared_memory(&dev->dev); -err_dma_declare: iounmap(hcd->regs); err_ioremap_regs: iounmap(tmio->ccr); @@ -276,7 +274,6 @@ static int ohci_hcd_tmio_drv_remove(struct platform_device *dev) tmio_stop_hc(dev); if (cell->disable) cell->disable(dev); - dma_release_declared_memory(&dev->dev); iounmap(hcd->regs); iounmap(tmio->ccr); usb_put_hcd(hcd); From 2d7a3dc3e24f43504b1f25eae8195e600f4cce8b Mon Sep 17 00:00:00 2001 From: Laurentiu Tudor Date: Wed, 29 May 2019 13:28:43 +0300 Subject: [PATCH 08/29] USB: drop HCD_LOCAL_MEM flag With the addition of the local memory allocator, the HCD_LOCAL_MEM flag can be dropped and the checks against it replaced with a check for the localmem_pool ptr being initialized. Signed-off-by: Laurentiu Tudor Tested-by: Fredrik Noring Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- drivers/usb/core/buffer.c | 8 +++----- drivers/usb/core/hcd.c | 15 ++++++--------- drivers/usb/host/ehci-hcd.c | 2 +- drivers/usb/host/fotg210-hcd.c | 2 +- drivers/usb/host/ohci-hcd.c | 2 +- drivers/usb/host/ohci-sm501.c | 5 +++-- drivers/usb/host/ohci-tmio.c | 2 +- drivers/usb/host/uhci-hcd.c | 2 +- include/linux/usb/hcd.h | 1 - 9 files changed, 17 insertions(+), 22 deletions(-) diff --git a/drivers/usb/core/buffer.c b/drivers/usb/core/buffer.c index d2064ad7ad14..1359b78a624e 100644 --- a/drivers/usb/core/buffer.c +++ b/drivers/usb/core/buffer.c @@ -68,7 +68,7 @@ int hcd_buffer_create(struct usb_hcd *hcd) if (!IS_ENABLED(CONFIG_HAS_DMA) || (!is_device_dma_capable(hcd->self.sysdev) && - !(hcd->driver->flags & HCD_LOCAL_MEM))) + !hcd->localmem_pool)) return 0; for (i = 0; i < HCD_BUFFER_POOLS; i++) { @@ -130,8 +130,7 @@ void *hcd_buffer_alloc( /* some USB hosts just use PIO */ if (!IS_ENABLED(CONFIG_HAS_DMA) || - (!is_device_dma_capable(bus->sysdev) && - !(hcd->driver->flags & HCD_LOCAL_MEM))) { + !is_device_dma_capable(bus->sysdev)) { *dma = ~(dma_addr_t) 0; return kmalloc(size, mem_flags); } @@ -162,8 +161,7 @@ void hcd_buffer_free( } if (!IS_ENABLED(CONFIG_HAS_DMA) || - (!is_device_dma_capable(bus->sysdev) && - !(hcd->driver->flags & HCD_LOCAL_MEM))) { + !is_device_dma_capable(bus->sysdev)) { kfree(addr); return; } diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index 29b96e5e8621..fe631d18c1ed 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -1347,14 +1347,14 @@ EXPORT_SYMBOL_GPL(usb_hcd_unlink_urb_from_ep); * using regular system memory - like pci devices doing bus mastering. * * To support host controllers with limited dma capabilities we provide dma - * bounce buffers. This feature can be enabled using the HCD_LOCAL_MEM flag. + * bounce buffers. This feature can be enabled by initializing + * hcd->localmem_pool using usb_hcd_setup_local_mem(). * For this to work properly the host controller code must first use the * function dma_declare_coherent_memory() to point out which memory area * that should be used for dma allocations. * - * The HCD_LOCAL_MEM flag then tells the usb code to allocate all data for - * dma using dma_alloc_coherent() which in turn allocates from the memory - * area pointed out with dma_declare_coherent_memory(). + * The initialized hcd->localmem_pool then tells the usb code to allocate all + * data for dma using the genalloc API. * * So, to summarize... * @@ -1364,9 +1364,6 @@ EXPORT_SYMBOL_GPL(usb_hcd_unlink_urb_from_ep); * (a) "normal" kernel memory is no good, and * (b) there's not enough to share * - * - The only *portable* hook for such stuff in the - * DMA framework is dma_declare_coherent_memory() - * * - So we use that, even though the primary requirement * is that the memory be "local" (hence addressable * by that device), not "coherent". @@ -1533,7 +1530,7 @@ int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb, urb->setup_dma)) return -EAGAIN; urb->transfer_flags |= URB_SETUP_MAP_SINGLE; - } else if (hcd->driver->flags & HCD_LOCAL_MEM) { + } else if (hcd->localmem_pool) { ret = hcd_alloc_coherent( urb->dev->bus, mem_flags, &urb->setup_dma, @@ -1603,7 +1600,7 @@ int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb, else urb->transfer_flags |= URB_DMA_MAP_SINGLE; } - } else if (hcd->driver->flags & HCD_LOCAL_MEM) { + } else if (hcd->localmem_pool) { ret = hcd_alloc_coherent( urb->dev->bus, mem_flags, &urb->transfer_dma, diff --git a/drivers/usb/host/ehci-hcd.c b/drivers/usb/host/ehci-hcd.c index cdafa97f632d..9da7e22848c9 100644 --- a/drivers/usb/host/ehci-hcd.c +++ b/drivers/usb/host/ehci-hcd.c @@ -559,7 +559,7 @@ static int ehci_init(struct usb_hcd *hcd) ehci->command = temp; /* Accept arbitrarily long scatter-gather lists */ - if (!(hcd->driver->flags & HCD_LOCAL_MEM)) + if (!hcd->localmem_pool) hcd->self.sg_tablesize = ~0; /* Prepare for unlinking active QHs */ diff --git a/drivers/usb/host/fotg210-hcd.c b/drivers/usb/host/fotg210-hcd.c index 0da68df259c8..5d74ff61fa4c 100644 --- a/drivers/usb/host/fotg210-hcd.c +++ b/drivers/usb/host/fotg210-hcd.c @@ -4995,7 +4995,7 @@ static int hcd_fotg210_init(struct usb_hcd *hcd) fotg210->command = temp; /* Accept arbitrarily long scatter-gather lists */ - if (!(hcd->driver->flags & HCD_LOCAL_MEM)) + if (!hcd->localmem_pool) hcd->self.sg_tablesize = ~0; return 0; } diff --git a/drivers/usb/host/ohci-hcd.c b/drivers/usb/host/ohci-hcd.c index b200b19b44fa..5801858d867e 100644 --- a/drivers/usb/host/ohci-hcd.c +++ b/drivers/usb/host/ohci-hcd.c @@ -448,7 +448,7 @@ static int ohci_init (struct ohci_hcd *ohci) struct usb_hcd *hcd = ohci_to_hcd(ohci); /* Accept arbitrarily long scatter-gather lists */ - if (!(hcd->driver->flags & HCD_LOCAL_MEM)) + if (!hcd->localmem_pool) hcd->self.sg_tablesize = ~0; if (distrust_firmware) diff --git a/drivers/usb/host/ohci-sm501.c b/drivers/usb/host/ohci-sm501.c index b710e100aec9..c158cda9e4b9 100644 --- a/drivers/usb/host/ohci-sm501.c +++ b/drivers/usb/host/ohci-sm501.c @@ -49,7 +49,7 @@ static const struct hc_driver ohci_sm501_hc_driver = { * generic hardware linkage */ .irq = ohci_irq, - .flags = HCD_USB11 | HCD_MEMORY | HCD_LOCAL_MEM, + .flags = HCD_USB11 | HCD_MEMORY, /* * basic lifecycle operations @@ -153,7 +153,8 @@ static int ohci_hcd_sm501_drv_probe(struct platform_device *pdev) * fine. This is however not always the case - buffers may be allocated * using kmalloc() - so the usb core needs to be told that it must copy * data into our local memory if the buffers happen to be placed in - * regular memory. The HCD_LOCAL_MEM flag does just that. + * regular memory. A non-null hcd->localmem_pool initialized by the + * the call to usb_hcd_setup_local_mem() below does just that. */ if (usb_hcd_setup_local_mem(hcd, mem->start, diff --git a/drivers/usb/host/ohci-tmio.c b/drivers/usb/host/ohci-tmio.c index 3b84ce0c3f29..d5a293a707b6 100644 --- a/drivers/usb/host/ohci-tmio.c +++ b/drivers/usb/host/ohci-tmio.c @@ -153,7 +153,7 @@ static const struct hc_driver ohci_tmio_hc_driver = { /* generic hardware linkage */ .irq = ohci_irq, - .flags = HCD_USB11 | HCD_MEMORY | HCD_LOCAL_MEM, + .flags = HCD_USB11 | HCD_MEMORY, /* basic lifecycle operations */ .start = ohci_tmio_start, diff --git a/drivers/usb/host/uhci-hcd.c b/drivers/usb/host/uhci-hcd.c index 98deb5f64268..03bc59755123 100644 --- a/drivers/usb/host/uhci-hcd.c +++ b/drivers/usb/host/uhci-hcd.c @@ -581,7 +581,7 @@ static int uhci_start(struct usb_hcd *hcd) hcd->uses_new_polling = 1; /* Accept arbitrarily long scatter-gather lists */ - if (!(hcd->driver->flags & HCD_LOCAL_MEM)) + if (!hcd->localmem_pool) hcd->self.sg_tablesize = ~0; spin_lock_init(&uhci->lock); diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 127560a4bfa0..bab27ccc8ff5 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -256,7 +256,6 @@ struct hc_driver { int flags; #define HCD_MEMORY 0x0001 /* HC regs use memory (else I/O) */ -#define HCD_LOCAL_MEM 0x0002 /* HC needs local memory */ #define HCD_SHARED 0x0004 /* Two (or more) usb_hcds share HW */ #define HCD_USB11 0x0010 /* USB 1.1 */ #define HCD_USB2 0x0020 /* USB 2.0 */ From e58cfbfb32d1b9bcc8dd5e0c8801c8d518bc51f3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 28 Apr 2019 13:34:59 -0500 Subject: [PATCH 09/29] MIPS: remove the _dma_cache_wback_inv export This export is not used in modular code, which is a good thing as everyone should use the proper DMA API instead. Signed-off-by: Christoph Hellwig Acked-by: Paul Burton --- arch/mips/mm/cache.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index 3da216988672..33b409391ddb 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -62,8 +62,6 @@ void (*_dma_cache_wback_inv)(unsigned long start, unsigned long size); void (*_dma_cache_wback)(unsigned long start, unsigned long size); void (*_dma_cache_inv)(unsigned long start, unsigned long size); -EXPORT_SYMBOL(_dma_cache_wback_inv); - #endif /* CONFIG_DMA_NONCOHERENT */ /* From 67f30ad19c4b329dbe47e1563b2017203bd02e34 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 28 Apr 2019 13:47:05 -0500 Subject: [PATCH 10/29] au1100fb: fix DMA API abuse Virtual addresses return from dma(m)_alloc_coherent are opaque in what backs then, and drivers must not poke into them. Switch the driver to use the generic DMA API mmap helper to avoid these games. Signed-off-by: Christoph Hellwig Acked-by: Bartlomiej Zolnierkiewicz --- drivers/video/fbdev/au1100fb.c | 24 ++++-------------------- drivers/video/fbdev/au1100fb.h | 1 + 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/drivers/video/fbdev/au1100fb.c b/drivers/video/fbdev/au1100fb.c index 0adf0683cf08..99941ae1f3a1 100644 --- a/drivers/video/fbdev/au1100fb.c +++ b/drivers/video/fbdev/au1100fb.c @@ -340,14 +340,12 @@ int au1100fb_fb_pan_display(struct fb_var_screeninfo *var, struct fb_info *fbi) */ int au1100fb_fb_mmap(struct fb_info *fbi, struct vm_area_struct *vma) { - struct au1100fb_device *fbdev; + struct au1100fb_device *fbdev = to_au1100fb_device(fbi); - fbdev = to_au1100fb_device(fbi); - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); pgprot_val(vma->vm_page_prot) |= (6 << 9); //CCA=6 - return vm_iomap_memory(vma, fbdev->fb_phys, fbdev->fb_len); + return dma_mmap_coherent(fbdev->dev, vma, fbdev->fb_mem, fbdev->fb_phys, + fbdev->fb_len); } static struct fb_ops au1100fb_ops = @@ -412,7 +410,6 @@ static int au1100fb_drv_probe(struct platform_device *dev) { struct au1100fb_device *fbdev; struct resource *regs_res; - unsigned long page; struct clk *c; /* Allocate new device private */ @@ -424,6 +421,7 @@ static int au1100fb_drv_probe(struct platform_device *dev) goto failed; platform_set_drvdata(dev, (void *)fbdev); + fbdev->dev = &dev->dev; /* Allocate region for our registers and map them */ regs_res = platform_get_resource(dev, IORESOURCE_MEM, 0); @@ -472,20 +470,6 @@ static int au1100fb_drv_probe(struct platform_device *dev) au1100fb_fix.smem_start = fbdev->fb_phys; au1100fb_fix.smem_len = fbdev->fb_len; - /* - * Set page reserved so that mmap will work. This is necessary - * since we'll be remapping normal memory. - */ - for (page = (unsigned long)fbdev->fb_mem; - page < PAGE_ALIGN((unsigned long)fbdev->fb_mem + fbdev->fb_len); - page += PAGE_SIZE) { -#ifdef CONFIG_DMA_NONCOHERENT - SetPageReserved(virt_to_page(CAC_ADDR((void *)page))); -#else - SetPageReserved(virt_to_page(page)); -#endif - } - print_dbg("Framebuffer memory map at %p", fbdev->fb_mem); print_dbg("phys=0x%08x, size=%dK", fbdev->fb_phys, fbdev->fb_len / 1024); diff --git a/drivers/video/fbdev/au1100fb.h b/drivers/video/fbdev/au1100fb.h index 9af19939a9c6..e7239bceefd3 100644 --- a/drivers/video/fbdev/au1100fb.h +++ b/drivers/video/fbdev/au1100fb.h @@ -110,6 +110,7 @@ struct au1100fb_device { dma_addr_t fb_phys; int panel_idx; struct clk *lcdclk; + struct device *dev; }; /********************************************************************/ From c30700db9eaabb35e0b123301df35a6846e6b6b4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 3 Jun 2019 08:43:51 +0200 Subject: [PATCH 11/29] dma-direct: provide generic support for uncached kernel segments A few architectures support uncached kernel segments. In that case we get an uncached mapping for a given physica address by using an offset in the uncached segement. Implement support for this scheme in the generic dma-direct code instead of duplicating it in arch hooks. Signed-off-by: Christoph Hellwig --- arch/Kconfig | 8 ++++++++ include/linux/dma-noncoherent.h | 3 +++ kernel/dma/direct.c | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index c47b328eada0..e8d19c3cb91f 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -260,6 +260,14 @@ config ARCH_HAS_SET_MEMORY config ARCH_HAS_SET_DIRECT_MAP bool +# +# Select if arch has an uncached kernel segment and provides the +# uncached_kernel_address / cached_kernel_address symbols to use it +# +config ARCH_HAS_UNCACHED_SEGMENT + select ARCH_HAS_DMA_PREP_COHERENT + bool + # Select if arch init_task must go in the __init_task_data section config ARCH_TASK_STRUCT_ON_STACK bool diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 9741767e400f..7e0126a04e02 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -80,4 +80,7 @@ static inline void arch_dma_prep_coherent(struct page *page, size_t size) } #endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */ +void *uncached_kernel_address(void *addr); +void *cached_kernel_address(void *addr); + #endif /* _LINUX_DMA_NONCOHERENT_H */ diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 0816c1e8b05a..b67f0aa08aa3 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -158,6 +158,13 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, *dma_handle = phys_to_dma(dev, page_to_phys(page)); } memset(ret, 0, size); + + if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && + !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) { + arch_dma_prep_coherent(page, size); + ret = uncached_kernel_address(ret); + } + return ret; } @@ -173,13 +180,18 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, if (force_dma_unencrypted()) set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order); + + if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && + !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) + cpu_addr = cached_kernel_address(cpu_addr); __dma_direct_free_pages(dev, size, virt_to_page(cpu_addr)); } void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { - if (!dev_is_dma_coherent(dev)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && + !dev_is_dma_coherent(dev)) return arch_dma_alloc(dev, size, dma_handle, gfp, attrs); return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); } @@ -187,7 +199,8 @@ void *dma_direct_alloc(struct device *dev, size_t size, void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { - if (!dev_is_dma_coherent(dev)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && + !dev_is_dma_coherent(dev)) arch_dma_free(dev, size, cpu_addr, dma_addr, attrs); else dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); From 2e96e04d25caaca8039ba9561e7e02ee8a192553 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 28 Apr 2019 13:57:39 -0500 Subject: [PATCH 12/29] MIPS: use the generic uncached segment support in dma-direct Stop providing the arch alloc/free hooks and just expose the segment offset instead. Signed-off-by: Christoph Hellwig Acked-by: Paul Burton --- arch/mips/Kconfig | 1 + arch/mips/include/asm/page.h | 3 --- arch/mips/jazz/jazzdma.c | 6 ------ arch/mips/mm/dma-noncoherent.c | 28 ++++++++++------------------ 4 files changed, 11 insertions(+), 27 deletions(-) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 70d3200476bf..61a390c2f2c1 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -8,6 +8,7 @@ config MIPS select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN_SANITIZE_ALL + select ARCH_HAS_UNCACHED_SEGMENT select ARCH_SUPPORTS_UPROBES select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF if 64BIT diff --git a/arch/mips/include/asm/page.h b/arch/mips/include/asm/page.h index 6b31c93b5eaa..23e0f1386e04 100644 --- a/arch/mips/include/asm/page.h +++ b/arch/mips/include/asm/page.h @@ -258,9 +258,6 @@ extern int __virt_addr_valid(const volatile void *kaddr); ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) | \ VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) -#define UNCAC_ADDR(addr) (UNCAC_BASE + __pa(addr)) -#define CAC_ADDR(addr) ((unsigned long)__va((addr) - UNCAC_BASE)) - #include #include diff --git a/arch/mips/jazz/jazzdma.c b/arch/mips/jazz/jazzdma.c index bedb5047aff3..1804dc9d8136 100644 --- a/arch/mips/jazz/jazzdma.c +++ b/arch/mips/jazz/jazzdma.c @@ -575,10 +575,6 @@ static void *jazz_dma_alloc(struct device *dev, size_t size, return NULL; } - if (!(attrs & DMA_ATTR_NON_CONSISTENT)) { - dma_cache_wback_inv((unsigned long)ret, size); - ret = (void *)UNCAC_ADDR(ret); - } return ret; } @@ -586,8 +582,6 @@ static void jazz_dma_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { vdma_free(dma_handle); - if (!(attrs & DMA_ATTR_NON_CONSISTENT)) - vaddr = (void *)CAC_ADDR((unsigned long)vaddr); dma_direct_free_pages(dev, size, vaddr, dma_handle, attrs); } diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c index f9549d2fbea3..ed56c6fa7be2 100644 --- a/arch/mips/mm/dma-noncoherent.c +++ b/arch/mips/mm/dma-noncoherent.c @@ -44,33 +44,25 @@ static inline bool cpu_needs_post_dma_flush(struct device *dev) } } -void *arch_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +void arch_dma_prep_coherent(struct page *page, size_t size) { - void *ret; - - ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); - if (ret && !(attrs & DMA_ATTR_NON_CONSISTENT)) { - dma_cache_wback_inv((unsigned long) ret, size); - ret = (void *)UNCAC_ADDR(ret); - } - - return ret; + dma_cache_wback_inv((unsigned long)page_address(page), size); } -void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t dma_addr, unsigned long attrs) +void *uncached_kernel_address(void *addr) { - if (!(attrs & DMA_ATTR_NON_CONSISTENT)) - cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr); - dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); + return (void *)(__pa(addr) + UNCAC_BASE); +} + +void *cached_kernel_address(void *addr) +{ + return __va(addr) - UNCAC_BASE; } long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, dma_addr_t dma_addr) { - unsigned long addr = CAC_ADDR((unsigned long)cpu_addr); - return page_to_pfn(virt_to_page((void *)addr)); + return page_to_pfn(virt_to_page(cached_kernel_address(cpu_addr))); } pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot, From 4b4b077cbd0a998aebaa72c199e06b8a4c8dcfee Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Mon, 10 Jun 2019 15:54:37 -0700 Subject: [PATCH 13/29] dma-remap: Avoid de-referencing NULL atomic_pool With architectures allowing the kernel to be placed almost arbitrarily in memory (e.g.: ARM64), it is possible to have the kernel resides at physical addresses above 4GB, resulting in neither the default CMA area, nor the atomic pool from successfully allocating. This does not prevent specific peripherals from working though, one example is XHCI, which still operates correctly. Trouble comes when the XHCI driver gets suspended and resumed, since we can now trigger the following NPD: [ 12.664170] usb usb1: root hub lost power or was reset [ 12.669387] usb usb2: root hub lost power or was reset [ 12.674662] Unable to handle kernel NULL pointer dereference at virtual address 00000008 [ 12.682896] pgd = ffffffc1365a7000 [ 12.686386] [00000008] *pgd=0000000136500003, *pud=0000000136500003, *pmd=0000000000000000 [ 12.694897] Internal error: Oops: 96000006 [#1] SMP [ 12.699843] Modules linked in: [ 12.702980] CPU: 0 PID: 1499 Comm: pml Not tainted 4.9.135-1.13pre #51 [ 12.709577] Hardware name: BCM97268DV (DT) [ 12.713736] task: ffffffc136bb6540 task.stack: ffffffc1366cc000 [ 12.719740] PC is at addr_in_gen_pool+0x4/0x48 [ 12.724253] LR is at __dma_free+0x64/0xbc [ 12.728325] pc : [] lr : [] pstate: 60000145 [ 12.735825] sp : ffffffc1366cf990 [ 12.739196] x29: ffffffc1366cf990 x28: ffffffc1366cc000 [ 12.744608] x27: 0000000000000000 x26: ffffffc13a8568c8 [ 12.750020] x25: 0000000000000000 x24: ffffff80098f9000 [ 12.755433] x23: 000000013a5ff000 x22: ffffff8009c57000 [ 12.760844] x21: ffffffc13a856810 x20: 0000000000000000 [ 12.766255] x19: 0000000000001000 x18: 000000000000000a [ 12.771667] x17: 0000007f917553e0 x16: 0000000000001002 [ 12.777078] x15: 00000000000a36cb x14: ffffff80898feb77 [ 12.782490] x13: ffffffffffffffff x12: 0000000000000030 [ 12.787899] x11: 00000000fffffffe x10: ffffff80098feb7f [ 12.793311] x9 : 0000000005f5e0ff x8 : 65776f702074736f [ 12.798723] x7 : 6c2062756820746f x6 : ffffff80098febb1 [ 12.804134] x5 : ffffff800809797c x4 : 0000000000000000 [ 12.809545] x3 : 000000013a5ff000 x2 : 0000000000000fff [ 12.814955] x1 : ffffff8009c57000 x0 : 0000000000000000 [ 12.820363] [ 12.821907] Process pml (pid: 1499, stack limit = 0xffffffc1366cc020) [ 12.828421] Stack: (0xffffffc1366cf990 to 0xffffffc1366d0000) [ 12.834240] f980: ffffffc1366cf9e0 ffffff80086004d0 [ 12.842186] f9a0: ffffffc13ab08238 0000000000000010 ffffff80097c2218 ffffffc13a856810 [ 12.850131] f9c0: ffffff8009c57000 000000013a5ff000 0000000000000008 000000013a5ff000 [ 12.858076] f9e0: ffffffc1366cfa50 ffffff80085f9250 ffffffc13ab08238 0000000000000004 [ 12.866021] fa00: ffffffc13ab08000 ffffff80097b6000 ffffffc13ab08130 0000000000000001 [ 12.873966] fa20: 0000000000000008 ffffffc13a8568c8 0000000000000000 ffffffc1366cc000 [ 12.881911] fa40: ffffffc13ab08130 0000000000000001 ffffffc1366cfa90 ffffff80085e3de8 [ 12.889856] fa60: ffffffc13ab08238 0000000000000000 ffffffc136b75b00 0000000000000000 [ 12.897801] fa80: 0000000000000010 ffffff80089ccb92 ffffffc1366cfac0 ffffff80084ad040 [ 12.905746] faa0: ffffffc13a856810 0000000000000000 ffffff80084ad004 ffffff80084b91a8 [ 12.913691] fac0: ffffffc1366cfae0 ffffff80084b91b4 ffffffc13a856810 ffffff80080db5cc [ 12.921636] fae0: ffffffc1366cfb20 ffffff80084b96bc ffffffc13a856810 0000000000000010 [ 12.929581] fb00: ffffffc13a856870 0000000000000000 ffffffc13a856810 ffffff800984d2b8 [ 12.937526] fb20: ffffffc1366cfb50 ffffff80084baa70 ffffff8009932ad0 ffffff800984d260 [ 12.945471] fb40: 0000000000000010 00000002eff0a065 ffffffc1366cfbb0 ffffff80084bafbc [ 12.953415] fb60: 0000000000000010 0000000000000003 ffffff80098fe000 0000000000000000 [ 12.961360] fb80: ffffff80097b6000 ffffff80097b6dc8 ffffff80098c12b8 ffffff80098c12f8 [ 12.969306] fba0: ffffff8008842000 ffffff80097b6dc8 ffffffc1366cfbd0 ffffff80080e0d88 [ 12.977251] fbc0: 00000000fffffffb ffffff80080e10bc ffffffc1366cfc60 ffffff80080e16a8 [ 12.985196] fbe0: 0000000000000000 0000000000000003 ffffff80097b6000 ffffff80098fe9f0 [ 12.993140] fc00: ffffff80097d4000 ffffff8008983802 0000000000000123 0000000000000040 [ 13.001085] fc20: ffffff8008842000 ffffffc1366cc000 ffffff80089803c2 00000000ffffffff [ 13.009029] fc40: 0000000000000000 0000000000000000 ffffffc1366cfc60 0000000000040987 [ 13.016974] fc60: ffffffc1366cfcc0 ffffff80080dfd08 0000000000000003 0000000000000004 [ 13.024919] fc80: 0000000000000003 ffffff80098fea08 ffffffc136577ec0 ffffff80089803c2 [ 13.032864] fca0: 0000000000000123 0000000000000001 0000000500000002 0000000000040987 [ 13.040809] fcc0: ffffffc1366cfd00 ffffff80083a89d4 0000000000000004 ffffffc136577ec0 [ 13.048754] fce0: ffffffc136610cc0 ffffffffffffffea ffffffc1366cfeb0 ffffffc136610cd8 [ 13.056700] fd00: ffffffc1366cfd10 ffffff800822a614 ffffffc1366cfd40 ffffff80082295d4 [ 13.064645] fd20: 0000000000000004 ffffffc136577ec0 ffffffc136610cc0 0000000021670570 [ 13.072590] fd40: ffffffc1366cfd80 ffffff80081b5d10 ffffff80097b6000 ffffffc13aae4200 [ 13.080536] fd60: ffffffc1366cfeb0 0000000000000004 0000000021670570 0000000000000004 [ 13.088481] fd80: ffffffc1366cfe30 ffffff80081b6b20 ffffffc13aae4200 0000000000000000 [ 13.096427] fda0: 0000000000000004 0000000021670570 ffffffc1366cfeb0 ffffffc13a838200 [ 13.104371] fdc0: 0000000000000000 000000000000000a ffffff80097b6000 0000000000040987 [ 13.112316] fde0: ffffffc1366cfe20 ffffff80081b3af0 ffffffc13a838200 0000000000000000 [ 13.120261] fe00: ffffffc1366cfe30 ffffff80081b6b0c ffffffc13aae4200 0000000000000000 [ 13.128206] fe20: 0000000000000004 0000000000040987 ffffffc1366cfe70 ffffff80081b7dd8 [ 13.136151] fe40: ffffff80097b6000 ffffffc13aae4200 ffffffc13aae4200 fffffffffffffff7 [ 13.144096] fe60: 0000000021670570 ffffffc13a8c63c0 0000000000000000 ffffff8008083180 [ 13.152042] fe80: ffffffffffffff1d 0000000021670570 ffffffffffffffff 0000007f917ad9b8 [ 13.159986] fea0: 0000000020000000 0000000000000015 0000000000000000 0000000000040987 [ 13.167930] fec0: 0000000000000001 0000000021670570 0000000000000004 0000000000000000 [ 13.175874] fee0: 0000000000000888 0000440110000000 000000000000006d 0000000000000003 [ 13.183819] ff00: 0000000000000040 ffffff80ffffffc8 0000000000000000 0000000000000020 [ 13.191762] ff20: 0000000000000000 0000000000000000 0000000000000001 0000000000000000 [ 13.199707] ff40: 0000000000000000 0000007f917553e0 0000000000000000 0000000000000004 [ 13.207651] ff60: 0000000021670570 0000007f91835480 0000000000000004 0000007f91831638 [ 13.215595] ff80: 0000000000000004 00000000004b0de0 00000000004b0000 0000000000000000 [ 13.223539] ffa0: 0000000000000000 0000007fc92ac8c0 0000007f9175d178 0000007fc92ac8c0 [ 13.231483] ffc0: 0000007f917ad9b8 0000000020000000 0000000000000001 0000000000000040 [ 13.239427] ffe0: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 [ 13.247360] Call trace: [ 13.249866] Exception stack(0xffffffc1366cf7a0 to 0xffffffc1366cf8d0) [ 13.256386] f7a0: 0000000000001000 0000007fffffffff ffffffc1366cf990 ffffff80083c0df8 [ 13.264331] f7c0: 0000000060000145 ffffff80089b5001 ffffffc13ab08130 0000000000000001 [ 13.272275] f7e0: 0000000000000008 ffffffc13a8568c8 0000000000000000 0000000000000000 [ 13.280220] f800: ffffffc1366cf960 ffffffc1366cf960 ffffffc1366cf930 00000000ffffffd8 [ 13.288165] f820: ffffff8009931ac0 4554535953425553 4544006273753d4d 3831633d45434956 [ 13.296110] f840: ffff003832313a39 ffffff800845926c ffffffc1366cf880 0000000000040987 [ 13.304054] f860: 0000000000000000 ffffff8009c57000 0000000000000fff 000000013a5ff000 [ 13.311999] f880: 0000000000000000 ffffff800809797c ffffff80098febb1 6c2062756820746f [ 13.319944] f8a0: 65776f702074736f 0000000005f5e0ff ffffff80098feb7f 00000000fffffffe [ 13.327884] f8c0: 0000000000000030 ffffffffffffffff [ 13.332835] [] addr_in_gen_pool+0x4/0x48 [ 13.338398] [] xhci_mem_cleanup+0xc8/0x51c [ 13.344137] [] xhci_resume+0x308/0x65c [ 13.349524] [] xhci_brcm_resume+0x84/0x8c [ 13.355174] [] platform_pm_resume+0x3c/0x64 [ 13.360997] [] dpm_run_callback+0x5c/0x15c [ 13.366732] [] device_resume+0xc0/0x190 [ 13.372205] [] dpm_resume+0x144/0x2cc [ 13.377504] [] dpm_resume_end+0x20/0x34 [ 13.382980] [] suspend_devices_and_enter+0x104/0x704 [ 13.389585] [] pm_suspend+0x320/0x53c [ 13.394881] [] state_store+0xbc/0xe0 [ 13.400094] [] kobj_attr_store+0x14/0x24 [ 13.405655] [] sysfs_kf_write+0x60/0x70 [ 13.411128] [] kernfs_fop_write+0x130/0x194 [ 13.416954] [] __vfs_write+0x60/0x150 [ 13.422254] [] vfs_write+0xc8/0x164 [ 13.427376] [] SyS_write+0x70/0xc8 [ 13.432412] [] el0_svc_naked+0x34/0x38 [ 13.437800] Code: 92800173 97f6fb9e 17fffff5 d1000442 (f8408c03) [ 13.444033] ---[ end trace 2effe12f909ce205 ]--- The call path leading to this problem is xhci_mem_cleanup() -> dma_free_coherent() -> dma_free_from_pool() -> addr_in_gen_pool. If the atomic_pool is NULL, we can't possibly have the address in the atomic pool anyway, so guard against that. Signed-off-by: Florian Fainelli Signed-off-by: Christoph Hellwig --- kernel/dma/remap.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/dma/remap.c b/kernel/dma/remap.c index 7a723194ecbe..0207e3764d52 100644 --- a/kernel/dma/remap.c +++ b/kernel/dma/remap.c @@ -158,6 +158,9 @@ out: bool dma_in_atomic_pool(void *start, size_t size) { + if (unlikely(!atomic_pool)) + return false; + return addr_in_gen_pool(atomic_pool, (unsigned long)start, size); } From 591fcf3b301b3396d96f082298a9e8403c027d3f Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Mon, 3 Jun 2019 15:52:59 -0700 Subject: [PATCH 14/29] iommu/dma: Apply dma_{alloc,free}_contiguous functions This patch replaces dma_{alloc,release}_from_contiguous() with dma_{alloc,free}_contiguous() to simplify those function calls. Signed-off-by: Nicolin Chen Acked-by: Robin Murphy Signed-off-by: Christoph Hellwig --- drivers/iommu/dma-iommu.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 0dee374fc64a..cc0613c83d71 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -951,8 +951,8 @@ static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr) if (pages) __iommu_dma_free_pages(pages, count); - if (page && !dma_release_from_contiguous(dev, page, count)) - __free_pages(page, get_order(alloc_size)); + if (page) + dma_free_contiguous(dev, page, alloc_size); } static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, @@ -970,12 +970,7 @@ static void *iommu_dma_alloc_pages(struct device *dev, size_t size, struct page *page = NULL; void *cpu_addr; - if (gfpflags_allow_blocking(gfp)) - page = dma_alloc_from_contiguous(dev, alloc_size >> PAGE_SHIFT, - get_order(alloc_size), - gfp & __GFP_NOWARN); - if (!page) - page = alloc_pages(gfp, get_order(alloc_size)); + page = dma_alloc_contiguous(dev, alloc_size, gfp); if (!page) return NULL; @@ -997,8 +992,7 @@ static void *iommu_dma_alloc_pages(struct device *dev, size_t size, memset(cpu_addr, 0, alloc_size); return cpu_addr; out_free_pages: - if (!dma_release_from_contiguous(dev, page, alloc_size >> PAGE_SHIFT)) - __free_pages(page, get_order(alloc_size)); + dma_free_contiguous(dev, page, alloc_size); return NULL; } From 4a54d16f615f41489b2ecbc940f6eb2618ddafd6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Apr 2019 09:16:42 -0500 Subject: [PATCH 15/29] dma-mapping: truncate dma masks to what dma_addr_t can hold The dma masks in struct device are always 64-bits wide. But for builds using a 32-bit dma_addr_t we need to ensure we don't store an unsupportable value. Before Linux 5.0 this was handled at least by the ARM dma mapping code by never allowing to set a larger dma_mask, but these days we allow the driver to just set the largest supported value and never fall back to a smaller one. Ensure this always works by truncating the value. Fixes: 9eb9e96e97b3 ("Documentation/DMA-API-HOWTO: update dma_mask sections") Signed-off-by: Christoph Hellwig --- kernel/dma/mapping.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index f7afdadb6770..1f628e7ac709 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -317,6 +317,12 @@ void arch_dma_set_mask(struct device *dev, u64 mask); int dma_set_mask(struct device *dev, u64 mask) { + /* + * Truncate the mask to the actually supported dma_addr_t width to + * avoid generating unsupportable addresses. + */ + mask = (dma_addr_t)mask; + if (!dev->dma_mask || !dma_supported(dev, mask)) return -EIO; @@ -330,6 +336,12 @@ EXPORT_SYMBOL(dma_set_mask); #ifndef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK int dma_set_coherent_mask(struct device *dev, u64 mask) { + /* + * Truncate the mask to the actually supported dma_addr_t width to + * avoid generating unsupportable addresses. + */ + mask = (dma_addr_t)mask; + if (!dma_supported(dev, mask)) return -EIO; From ab746573c4055ae1fa226715502fb9bb9be29a79 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Apr 2019 09:04:53 -0500 Subject: [PATCH 16/29] ARM: dma-mapping: allow larger DMA mask than supported Since the Linux 5.1 merge window we allow drivers to just set the largest DMA mask they support instead of falling back to smaller ones. But I forgot to remove a check that prohibits this behavior in the arm DMA code, as it is rather hidden. There is not reason for this check as the code will do the right thing for a "too large" DMA mask, so just remove it. Fixes: 9eb9e96e97b3 ("Documentation/DMA-API-HOWTO: update dma_mask sections") Signed-off-by: Christoph Hellwig --- arch/arm/mm/dma-mapping.c | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 0a75058c11f3..bdf0d236aaee 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -219,25 +219,7 @@ EXPORT_SYMBOL(arm_coherent_dma_ops); static int __dma_supported(struct device *dev, u64 mask, bool warn) { - unsigned long max_dma_pfn; - - /* - * If the mask allows for more memory than we can address, - * and we actually have that much memory, then we must - * indicate that DMA to this device is not supported. - */ - if (sizeof(mask) != sizeof(dma_addr_t) && - mask > (dma_addr_t)~0 && - dma_to_pfn(dev, ~0) < max_pfn - 1) { - if (warn) { - dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n", - mask); - dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n"); - } - return 0; - } - - max_dma_pfn = min(max_pfn, arm_dma_pfn_limit); + unsigned long max_dma_pfn = min(max_pfn, arm_dma_pfn_limit); /* * Translate the device's DMA mask to a PFN limit. This From 34ab03160eda51839be6dd5a939680963266707c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 3 Jun 2019 12:48:57 +0200 Subject: [PATCH 17/29] arm-nommu: remove the partial DMA_ATTR_NON_CONSISTENT support The arm-nommu DMA code supports DMA_ATTR_NON_CONSISTENT allocations, but does not provide a cache_sync operation. This means any user of it will never be able to actually transfer cache ownership and thus cause coherency bugs. Signed-off-by: Christoph Hellwig Reviewed-by: Vladimir Murzin --- arch/arm/mm/dma-mapping-nommu.c | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c index f304b10e23a4..bc003df45546 100644 --- a/arch/arm/mm/dma-mapping-nommu.c +++ b/arch/arm/mm/dma-mapping-nommu.c @@ -39,18 +39,7 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size, unsigned long attrs) { - void *ret; - - /* - * Try generic allocator first if we are advertised that - * consistency is not required. - */ - - if (attrs & DMA_ATTR_NON_CONSISTENT) - return dma_direct_alloc_pages(dev, size, dma_handle, gfp, - attrs); - - ret = dma_alloc_from_global_coherent(size, dma_handle); + void *ret = dma_alloc_from_global_coherent(size, dma_handle); /* * dma_alloc_from_global_coherent() may fail because: @@ -70,16 +59,9 @@ static void arm_nommu_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { - if (attrs & DMA_ATTR_NON_CONSISTENT) { - dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); - } else { - int ret = dma_release_from_global_coherent(get_order(size), - cpu_addr); + int ret = dma_release_from_global_coherent(get_order(size), cpu_addr); - WARN_ON_ONCE(ret == 0); - } - - return; + WARN_ON_ONCE(ret == 0); } static int arm_nommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, From 80e61fcd23946cb222f780a49ab2eeb7ef1d3749 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 3 Jun 2019 12:52:47 +0200 Subject: [PATCH 18/29] arc: remove the partial DMA_ATTR_NON_CONSISTENT support The arc DMA code supports DMA_ATTR_NON_CONSISTENT allocations, but does not provide a cache_sync operation. This means any user of it will never be able to actually transfer cache ownership and thus cause coherency bugs. Signed-off-by: Christoph Hellwig Reviewed-by: Evgeniy Paltsev Tested-by: Evgeniy Paltsev --- arch/arc/mm/dma.c | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c index 1525ac00fd02..9832928f896d 100644 --- a/arch/arc/mm/dma.c +++ b/arch/arc/mm/dma.c @@ -24,7 +24,6 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, struct page *page; phys_addr_t paddr; void *kvaddr; - bool need_coh = !(attrs & DMA_ATTR_NON_CONSISTENT); /* * __GFP_HIGHMEM flag is cleared by upper layer functions @@ -46,14 +45,10 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, * A coherent buffer needs MMU mapping to enforce non-cachability. * kvaddr is kernel Virtual address (0x7000_0000 based). */ - if (need_coh) { - kvaddr = ioremap_nocache(paddr, size); - if (kvaddr == NULL) { - __free_pages(page, order); - return NULL; - } - } else { - kvaddr = (void *)(u32)paddr; + kvaddr = ioremap_nocache(paddr, size); + if (kvaddr == NULL) { + __free_pages(page, order); + return NULL; } /* @@ -66,9 +61,7 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, * Currently flush_cache_vmap nukes the L1 cache completely which * will be optimized as a separate commit */ - if (need_coh) - dma_cache_wback_inv(paddr, size); - + dma_cache_wback_inv(paddr, size); return kvaddr; } @@ -78,9 +71,7 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, phys_addr_t paddr = dma_handle; struct page *page = virt_to_page(paddr); - if (!(attrs & DMA_ATTR_NON_CONSISTENT)) - iounmap((void __force __iomem *)vaddr); - + iounmap((void __force __iomem *)vaddr); __free_pages(page, get_order(size)); } From 961729bfc73e698be19305834805592227bd09e3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 3 Jun 2019 12:54:13 +0200 Subject: [PATCH 19/29] openrisc: remove the partial DMA_ATTR_NON_CONSISTENT support The openrisc DMA code supports DMA_ATTR_NON_CONSISTENT allocations, but does not provide a cache_sync operation. This means any user of it will never be able to actually transfer cache ownership and thus cause coherency bugs. Signed-off-by: Christoph Hellwig Acked-by: Stafford Horne --- arch/openrisc/kernel/dma.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/arch/openrisc/kernel/dma.c b/arch/openrisc/kernel/dma.c index f79457cb3741..9f25fd0fbb5d 100644 --- a/arch/openrisc/kernel/dma.c +++ b/arch/openrisc/kernel/dma.c @@ -98,15 +98,13 @@ arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, va = (unsigned long)page; - if ((attrs & DMA_ATTR_NON_CONSISTENT) == 0) { - /* - * We need to iterate through the pages, clearing the dcache for - * them and setting the cache-inhibit bit. - */ - if (walk_page_range(va, va + size, &walk)) { - free_pages_exact(page, size); - return NULL; - } + /* + * We need to iterate through the pages, clearing the dcache for + * them and setting the cache-inhibit bit. + */ + if (walk_page_range(va, va + size, &walk)) { + free_pages_exact(page, size); + return NULL; } return (void *)va; @@ -122,10 +120,8 @@ arch_dma_free(struct device *dev, size_t size, void *vaddr, .mm = &init_mm }; - if ((attrs & DMA_ATTR_NON_CONSISTENT) == 0) { - /* walk_page_range shouldn't be able to fail here */ - WARN_ON(walk_page_range(va, va + size, &walk)); - } + /* walk_page_range shouldn't be able to fail here */ + WARN_ON(walk_page_range(va, va + size, &walk)); free_pages_exact(vaddr, size); } From 4b85faed211ccfbcc7f3adf1cd62f0b00d1a172b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 14 Jun 2019 16:06:10 +0200 Subject: [PATCH 20/29] dma-mapping: add a dma_alloc_need_uncached helper Check if we need to allocate uncached memory for a device given the allocation flags. Switch over the uncached segment check to this helper to deal with architectures that do not support the dma_cache_sync operation and thus should not returned cacheable memory for DMA_ATTR_NON_CONSISTENT allocations. Signed-off-by: Christoph Hellwig --- include/linux/dma-noncoherent.h | 14 ++++++++++++++ kernel/dma/direct.c | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 7e0126a04e02..732919ac5c11 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -20,6 +20,20 @@ static inline bool dev_is_dma_coherent(struct device *dev) } #endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */ +/* + * Check if an allocation needs to be marked uncached to be coherent. + */ +static inline bool dma_alloc_need_uncached(struct device *dev, + unsigned long attrs) +{ + if (dev_is_dma_coherent(dev)) + return false; + if (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) && + (attrs & DMA_ATTR_NON_CONSISTENT)) + return false; + return true; +} + void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index b67f0aa08aa3..c2893713bf80 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -160,7 +160,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, memset(ret, 0, size); if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && - !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) { + dma_alloc_need_uncached(dev, attrs)) { arch_dma_prep_coherent(page, size); ret = uncached_kernel_address(ret); } @@ -182,7 +182,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order); if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && - !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) + dma_alloc_need_uncached(dev, attrs)) cpu_addr = cached_kernel_address(cpu_addr); __dma_direct_free_pages(dev, size, virt_to_page(cpu_addr)); } From c2f2124e0d447ad02a41a92361b3734366797680 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 14 Jun 2019 15:59:14 +0200 Subject: [PATCH 21/29] dma-direct: handle DMA_ATTR_NON_CONSISTENT in common code Only call into arch_dma_alloc if we require an uncached mapping, and remove the parisc code manually doing normal cached DMA_ATTR_NON_CONSISTENT allocations. Signed-off-by: Christoph Hellwig Acked-by: Helge Deller # parisc --- arch/parisc/kernel/pci-dma.c | 48 ++++++++++-------------------------- kernel/dma/direct.c | 4 +-- 2 files changed, 15 insertions(+), 37 deletions(-) diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index 239162355b58..ca35d9a76e50 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -394,17 +394,20 @@ pcxl_dma_init(void) __initcall(pcxl_dma_init); -static void *pcxl_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) +void *arch_dma_alloc(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { unsigned long vaddr; unsigned long paddr; int order; + if (boot_cpu_data.cpu_type != pcxl2 && boot_cpu_data.cpu_type != pcxl) + return NULL; + order = get_order(size); size = 1 << (order + PAGE_SHIFT); vaddr = pcxl_alloc_range(size); - paddr = __get_free_pages(flag | __GFP_ZERO, order); + paddr = __get_free_pages(gfp | __GFP_ZERO, order); flush_kernel_dcache_range(paddr, size); paddr = __pa(paddr); map_uncached_pages(vaddr, size, paddr); @@ -421,44 +424,19 @@ static void *pcxl_dma_alloc(struct device *dev, size_t size, return (void *)vaddr; } -static void *pcx_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) -{ - void *addr; - - if ((attrs & DMA_ATTR_NON_CONSISTENT) == 0) - return NULL; - - addr = (void *)__get_free_pages(flag | __GFP_ZERO, get_order(size)); - if (addr) - *dma_handle = (dma_addr_t)virt_to_phys(addr); - - return addr; -} - -void *arch_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) -{ - - if (boot_cpu_data.cpu_type == pcxl2 || boot_cpu_data.cpu_type == pcxl) - return pcxl_dma_alloc(dev, size, dma_handle, gfp, attrs); - else - return pcx_dma_alloc(dev, size, dma_handle, gfp, attrs); -} - void arch_dma_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { int order = get_order(size); - if (boot_cpu_data.cpu_type == pcxl2 || boot_cpu_data.cpu_type == pcxl) { - size = 1 << (order + PAGE_SHIFT); - unmap_uncached_pages((unsigned long)vaddr, size); - pcxl_free_range((unsigned long)vaddr, size); + WARN_ON_ONCE(boot_cpu_data.cpu_type != pcxl2 && + boot_cpu_data.cpu_type != pcxl); - vaddr = __va(dma_handle); - } - free_pages((unsigned long)vaddr, get_order(size)); + size = 1 << (order + PAGE_SHIFT); + unmap_uncached_pages((unsigned long)vaddr, size); + pcxl_free_range((unsigned long)vaddr, size); + + free_pages((unsigned long)__va(dma_handle), order); } void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index c2893713bf80..fc354f4f490b 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -191,7 +191,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && - !dev_is_dma_coherent(dev)) + dma_alloc_need_uncached(dev, attrs)) return arch_dma_alloc(dev, size, dma_handle, gfp, attrs); return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); } @@ -200,7 +200,7 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && - !dev_is_dma_coherent(dev)) + dma_alloc_need_uncached(dev, attrs)) arch_dma_free(dev, size, cpu_addr, dma_addr, attrs); else dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); From d98849aff87911013aadb730138ab728b52fc547 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 14 Jun 2019 16:17:27 +0200 Subject: [PATCH 22/29] dma-direct: handle DMA_ATTR_NO_KERNEL_MAPPING in common code DMA_ATTR_NO_KERNEL_MAPPING is generally implemented by allocating normal cacheable pages or CMA memory, and then returning the page pointer as the opaque handle. Lift that code from the xtensa and generic dma remapping implementations into the generic dma-direct code so that we don't even call arch_dma_alloc for these allocations. Signed-off-by: Christoph Hellwig --- arch/xtensa/kernel/pci-dma.c | 8 +------- include/linux/dma-noncoherent.h | 2 ++ kernel/dma/direct.c | 14 ++++++++++++++ kernel/dma/remap.c | 13 ++----------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/arch/xtensa/kernel/pci-dma.c b/arch/xtensa/kernel/pci-dma.c index 9171bff76fc4..206771277dff 100644 --- a/arch/xtensa/kernel/pci-dma.c +++ b/arch/xtensa/kernel/pci-dma.c @@ -167,10 +167,6 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, *handle = phys_to_dma(dev, page_to_phys(page)); - if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { - return page; - } - #ifdef CONFIG_MMU if (PageHighMem(page)) { void *p; @@ -196,9 +192,7 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; struct page *page; - if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { - page = vaddr; - } else if (platform_vaddr_uncached(vaddr)) { + if (platform_vaddr_uncached(vaddr)) { page = virt_to_page(platform_vaddr_to_cached(vaddr)); } else { #ifdef CONFIG_MMU diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 732919ac5c11..53ee36ecdf37 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -28,6 +28,8 @@ static inline bool dma_alloc_need_uncached(struct device *dev, { if (dev_is_dma_coherent(dev)) return false; + if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) + return false; if (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) && (attrs & DMA_ATTR_NON_CONSISTENT)) return false; diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index fc354f4f490b..b90e1aede743 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -138,6 +138,14 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, if (!page) return NULL; + if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { + /* remove any dirty cache lines on the kernel alias */ + if (!PageHighMem(page)) + arch_dma_prep_coherent(page, size); + /* return the page pointer as the opaque cookie */ + return page; + } + if (PageHighMem(page)) { /* * Depending on the cma= arguments and per-arch setup @@ -178,6 +186,12 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, { unsigned int page_order = get_order(size); + if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { + /* cpu_addr is a struct page cookie, not a kernel address */ + __dma_direct_free_pages(dev, size, cpu_addr); + return; + } + if (force_dma_unencrypted()) set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order); diff --git a/kernel/dma/remap.c b/kernel/dma/remap.c index 0207e3764d52..a594aec07882 100644 --- a/kernel/dma/remap.c +++ b/kernel/dma/remap.c @@ -202,8 +202,7 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, size = PAGE_ALIGN(size); - if (!gfpflags_allow_blocking(flags) && - !(attrs & DMA_ATTR_NO_KERNEL_MAPPING)) { + if (!gfpflags_allow_blocking(flags)) { ret = dma_alloc_from_pool(size, &page, flags); if (!ret) return NULL; @@ -217,11 +216,6 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, /* remove any dirty cache lines on the kernel alias */ arch_dma_prep_coherent(page, size); - if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { - ret = page; /* opaque cookie */ - goto done; - } - /* create a coherent mapping */ ret = dma_common_contiguous_remap(page, size, VM_USERMAP, arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs), @@ -240,10 +234,7 @@ done: void arch_dma_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { - if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { - /* vaddr is a struct page cookie, not a kernel address */ - __dma_direct_free_pages(dev, size, vaddr); - } else if (!dma_free_from_pool(vaddr, PAGE_ALIGN(size))) { + if (!dma_free_from_pool(vaddr, PAGE_ALIGN(size))) { phys_addr_t phys = dma_to_phys(dev, dma_handle); struct page *page = pfn_to_page(__phys_to_pfn(phys)); From f73c904534393133e7ddbbe5c36bb007f9c2fb7f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 14 Jun 2019 16:26:41 +0200 Subject: [PATCH 23/29] arc: use the generic remapping allocator for coherent DMA allocations Replace the code that sets up uncached PTEs with the generic vmap based remapping code. It also provides an atomic pool for allocations from non-blocking context, which we not properly supported by the existing arc code. Signed-off-by: Christoph Hellwig Reviewed-by: Evgeniy Paltsev Tested-by: Evgeniy Paltsev --- arch/arc/Kconfig | 2 ++ arch/arc/mm/dma.c | 62 ++++++++--------------------------------------- 2 files changed, 12 insertions(+), 52 deletions(-) diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 23e063df5d2c..cdad7d30ff1d 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -10,6 +10,7 @@ config ARC def_bool y select ARC_TIMERS select ARCH_HAS_DMA_COHERENT_TO_PFN + select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SETUP_DMA_OPS select ARCH_HAS_SYNC_DMA_FOR_CPU @@ -19,6 +20,7 @@ config ARC select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS select COMMON_CLK + select DMA_DIRECT_REMAP select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC) select GENERIC_CLOCKEVENTS select GENERIC_FIND_FIRST_BIT diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c index 9832928f896d..0fa850709fac 100644 --- a/arch/arc/mm/dma.c +++ b/arch/arc/mm/dma.c @@ -11,46 +11,15 @@ #include /* - * ARCH specific callbacks for generic noncoherent DMA ops (dma/noncoherent.c) + * ARCH specific callbacks for generic noncoherent DMA ops * - hardware IOC not available (or "dma-coherent" not set for device in DT) * - But still handle both coherent and non-coherent requests from caller * * For DMA coherent hardware (IOC) generic code suffices */ -void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, - gfp_t gfp, unsigned long attrs) + +void arch_dma_prep_coherent(struct page *page, size_t size) { - unsigned long order = get_order(size); - struct page *page; - phys_addr_t paddr; - void *kvaddr; - - /* - * __GFP_HIGHMEM flag is cleared by upper layer functions - * (in include/linux/dma-mapping.h) so we should never get a - * __GFP_HIGHMEM here. - */ - BUG_ON(gfp & __GFP_HIGHMEM); - - page = alloc_pages(gfp | __GFP_ZERO, order); - if (!page) - return NULL; - - /* This is linear addr (0x8000_0000 based) */ - paddr = page_to_phys(page); - - *dma_handle = paddr; - - /* - * A coherent buffer needs MMU mapping to enforce non-cachability. - * kvaddr is kernel Virtual address (0x7000_0000 based). - */ - kvaddr = ioremap_nocache(paddr, size); - if (kvaddr == NULL) { - __free_pages(page, order); - return NULL; - } - /* * Evict any existing L1 and/or L2 lines for the backing page * in case it was used earlier as a normal "cached" page. @@ -61,24 +30,7 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, * Currently flush_cache_vmap nukes the L1 cache completely which * will be optimized as a separate commit */ - dma_cache_wback_inv(paddr, size); - return kvaddr; -} - -void arch_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, unsigned long attrs) -{ - phys_addr_t paddr = dma_handle; - struct page *page = virt_to_page(paddr); - - iounmap((void __force __iomem *)vaddr); - __free_pages(page, get_order(size)); -} - -long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, - dma_addr_t dma_addr) -{ - return __phys_to_pfn(dma_addr); + dma_cache_wback_inv(page_to_phys(page), size); } /* @@ -155,3 +107,9 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, dev_info(dev, "use %sncoherent DMA ops\n", dev->dma_coherent ? "" : "non"); } + +static int __init atomic_pool_init(void) +{ + return dma_atomic_pool_init(GFP_KERNEL, pgprot_noncached(PAGE_KERNEL)); +} +postcore_initcall(atomic_pool_init); From 6309513c1be47cd3805a6ceae60ab70285df36ae Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 28 Apr 2019 14:28:38 -0500 Subject: [PATCH 24/29] nds32: use the generic remapping allocator for coherent DMA allocations Replace the code that sets up uncached PTEs with the generic vmap based remapping code. It also provides an atomic pool for allocations from non-blocking context, which we not properly supported by the existing nds32 code. Signed-off-by: Christoph Hellwig Tested-by: Greentime Hu Reviewed-by: Greentime Hu --- arch/nds32/Kconfig | 2 + arch/nds32/kernel/dma.c | 325 ++-------------------------------------- 2 files changed, 13 insertions(+), 314 deletions(-) diff --git a/arch/nds32/Kconfig b/arch/nds32/Kconfig index 3299e287a477..643ea6b4bfa2 100644 --- a/arch/nds32/Kconfig +++ b/arch/nds32/Kconfig @@ -7,12 +7,14 @@ config NDS32 def_bool y select ARCH_32BIT_OFF_T + select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_WANT_FRAME_POINTERS if FTRACE select CLKSRC_MMIO select CLONE_BACKWARDS select COMMON_CLK + select DMA_DIRECT_REMAP select GENERIC_ATOMIC64 select GENERIC_CPU_DEVICES select GENERIC_CLOCKEVENTS diff --git a/arch/nds32/kernel/dma.c b/arch/nds32/kernel/dma.c index d0dbd4fe9645..490e3720d694 100644 --- a/arch/nds32/kernel/dma.c +++ b/arch/nds32/kernel/dma.c @@ -3,327 +3,13 @@ #include #include -#include #include -#include #include #include -#include #include #include #include -/* - * This is the page table (2MB) covering uncached, DMA consistent allocations - */ -static pte_t *consistent_pte; -static DEFINE_RAW_SPINLOCK(consistent_lock); - -/* - * VM region handling support. - * - * This should become something generic, handling VM region allocations for - * vmalloc and similar (ioremap, module space, etc). - * - * I envisage vmalloc()'s supporting vm_struct becoming: - * - * struct vm_struct { - * struct vm_region region; - * unsigned long flags; - * struct page **pages; - * unsigned int nr_pages; - * unsigned long phys_addr; - * }; - * - * get_vm_area() would then call vm_region_alloc with an appropriate - * struct vm_region head (eg): - * - * struct vm_region vmalloc_head = { - * .vm_list = LIST_HEAD_INIT(vmalloc_head.vm_list), - * .vm_start = VMALLOC_START, - * .vm_end = VMALLOC_END, - * }; - * - * However, vmalloc_head.vm_start is variable (typically, it is dependent on - * the amount of RAM found at boot time.) I would imagine that get_vm_area() - * would have to initialise this each time prior to calling vm_region_alloc(). - */ -struct arch_vm_region { - struct list_head vm_list; - unsigned long vm_start; - unsigned long vm_end; - struct page *vm_pages; -}; - -static struct arch_vm_region consistent_head = { - .vm_list = LIST_HEAD_INIT(consistent_head.vm_list), - .vm_start = CONSISTENT_BASE, - .vm_end = CONSISTENT_END, -}; - -static struct arch_vm_region *vm_region_alloc(struct arch_vm_region *head, - size_t size, int gfp) -{ - unsigned long addr = head->vm_start, end = head->vm_end - size; - unsigned long flags; - struct arch_vm_region *c, *new; - - new = kmalloc(sizeof(struct arch_vm_region), gfp); - if (!new) - goto out; - - raw_spin_lock_irqsave(&consistent_lock, flags); - - list_for_each_entry(c, &head->vm_list, vm_list) { - if ((addr + size) < addr) - goto nospc; - if ((addr + size) <= c->vm_start) - goto found; - addr = c->vm_end; - if (addr > end) - goto nospc; - } - -found: - /* - * Insert this entry _before_ the one we found. - */ - list_add_tail(&new->vm_list, &c->vm_list); - new->vm_start = addr; - new->vm_end = addr + size; - - raw_spin_unlock_irqrestore(&consistent_lock, flags); - return new; - -nospc: - raw_spin_unlock_irqrestore(&consistent_lock, flags); - kfree(new); -out: - return NULL; -} - -static struct arch_vm_region *vm_region_find(struct arch_vm_region *head, - unsigned long addr) -{ - struct arch_vm_region *c; - - list_for_each_entry(c, &head->vm_list, vm_list) { - if (c->vm_start == addr) - goto out; - } - c = NULL; -out: - return c; -} - -void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t gfp, unsigned long attrs) -{ - struct page *page; - struct arch_vm_region *c; - unsigned long order; - u64 mask = ~0ULL, limit; - pgprot_t prot = pgprot_noncached(PAGE_KERNEL); - - if (!consistent_pte) { - pr_err("%s: not initialized\n", __func__); - dump_stack(); - return NULL; - } - - if (dev) { - mask = dev->coherent_dma_mask; - - /* - * Sanity check the DMA mask - it must be non-zero, and - * must be able to be satisfied by a DMA allocation. - */ - if (mask == 0) { - dev_warn(dev, "coherent DMA mask is unset\n"); - goto no_page; - } - - } - - /* - * Sanity check the allocation size. - */ - size = PAGE_ALIGN(size); - limit = (mask + 1) & ~mask; - if ((limit && size >= limit) || - size >= (CONSISTENT_END - CONSISTENT_BASE)) { - pr_warn("coherent allocation too big " - "(requested %#x mask %#llx)\n", size, mask); - goto no_page; - } - - order = get_order(size); - - if (mask != 0xffffffff) - gfp |= GFP_DMA; - - page = alloc_pages(gfp, order); - if (!page) - goto no_page; - - /* - * Invalidate any data that might be lurking in the - * kernel direct-mapped region for device DMA. - */ - { - unsigned long kaddr = (unsigned long)page_address(page); - memset(page_address(page), 0, size); - cpu_dma_wbinval_range(kaddr, kaddr + size); - } - - /* - * Allocate a virtual address in the consistent mapping region. - */ - c = vm_region_alloc(&consistent_head, size, - gfp & ~(__GFP_DMA | __GFP_HIGHMEM)); - if (c) { - pte_t *pte = consistent_pte + CONSISTENT_OFFSET(c->vm_start); - struct page *end = page + (1 << order); - - c->vm_pages = page; - - /* - * Set the "dma handle" - */ - *handle = page_to_phys(page); - - do { - BUG_ON(!pte_none(*pte)); - - /* - * x86 does not mark the pages reserved... - */ - SetPageReserved(page); - set_pte(pte, mk_pte(page, prot)); - page++; - pte++; - } while (size -= PAGE_SIZE); - - /* - * Free the otherwise unused pages. - */ - while (page < end) { - __free_page(page); - page++; - } - - return (void *)c->vm_start; - } - - if (page) - __free_pages(page, order); -no_page: - *handle = ~0; - return NULL; -} - -void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs) -{ - struct arch_vm_region *c; - unsigned long flags, addr; - pte_t *ptep; - - size = PAGE_ALIGN(size); - - raw_spin_lock_irqsave(&consistent_lock, flags); - - c = vm_region_find(&consistent_head, (unsigned long)cpu_addr); - if (!c) - goto no_area; - - if ((c->vm_end - c->vm_start) != size) { - pr_err("%s: freeing wrong coherent size (%ld != %d)\n", - __func__, c->vm_end - c->vm_start, size); - dump_stack(); - size = c->vm_end - c->vm_start; - } - - ptep = consistent_pte + CONSISTENT_OFFSET(c->vm_start); - addr = c->vm_start; - do { - pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep); - unsigned long pfn; - - ptep++; - addr += PAGE_SIZE; - - if (!pte_none(pte) && pte_present(pte)) { - pfn = pte_pfn(pte); - - if (pfn_valid(pfn)) { - struct page *page = pfn_to_page(pfn); - - /* - * x86 does not mark the pages reserved... - */ - ClearPageReserved(page); - - __free_page(page); - continue; - } - } - - pr_crit("%s: bad page in kernel page table\n", __func__); - } while (size -= PAGE_SIZE); - - flush_tlb_kernel_range(c->vm_start, c->vm_end); - - list_del(&c->vm_list); - - raw_spin_unlock_irqrestore(&consistent_lock, flags); - - kfree(c); - return; - -no_area: - raw_spin_unlock_irqrestore(&consistent_lock, flags); - pr_err("%s: trying to free invalid coherent area: %p\n", - __func__, cpu_addr); - dump_stack(); -} - -/* - * Initialise the consistent memory allocation. - */ -static int __init consistent_init(void) -{ - pgd_t *pgd; - pmd_t *pmd; - pte_t *pte; - int ret = 0; - - do { - pgd = pgd_offset(&init_mm, CONSISTENT_BASE); - pmd = pmd_alloc(&init_mm, pgd, CONSISTENT_BASE); - if (!pmd) { - pr_err("%s: no pmd tables\n", __func__); - ret = -ENOMEM; - break; - } - /* The first level mapping may be created in somewhere. - * It's not necessary to warn here. */ - /* WARN_ON(!pmd_none(*pmd)); */ - - pte = pte_alloc_kernel(pmd, CONSISTENT_BASE); - if (!pte) { - ret = -ENOMEM; - break; - } - - consistent_pte = pte; - } while (0); - - return ret; -} - -core_initcall(consistent_init); - static inline void cache_op(phys_addr_t paddr, size_t size, void (*fn)(unsigned long start, unsigned long end)) { @@ -389,3 +75,14 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, BUG(); } } + +void arch_dma_prep_coherent(struct page *page, size_t size) +{ + cache_op(page_to_phys(page), size, cpu_dma_wbinval_range); +} + +static int __init atomic_pool_init(void) +{ + return dma_atomic_pool_init(GFP_KERNEL, pgprot_noncached(PAGE_KERNEL)); +} +postcore_initcall(atomic_pool_init); From b1acd4b8a8942f614053e516c56c88e1716562d6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 28 Apr 2019 14:00:52 -0500 Subject: [PATCH 25/29] nios2: use the generic uncached segment support in dma-direct Stop providing our own arch alloc/free hooks and just expose the segment offset and use the generic dma-direct allocator. Signed-off-by: Christoph Hellwig Acked-by: Ley Foon Tan --- arch/nios2/Kconfig | 1 + arch/nios2/include/asm/page.h | 6 ------ arch/nios2/mm/dma-mapping.c | 40 ++++++++++++++++------------------- 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index 26a9c760a98b..44b5da37e8bd 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -4,6 +4,7 @@ config NIOS2 select ARCH_32BIT_OFF_T select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE + select ARCH_HAS_UNCACHED_SEGMENT select ARCH_NO_SWAP select TIMER_OF select GENERIC_ATOMIC64 diff --git a/arch/nios2/include/asm/page.h b/arch/nios2/include/asm/page.h index f1fbdc47bdaf..79fcac61f6ef 100644 --- a/arch/nios2/include/asm/page.h +++ b/arch/nios2/include/asm/page.h @@ -101,12 +101,6 @@ static inline bool pfn_valid(unsigned long pfn) # define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | \ VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) -# define UNCAC_ADDR(addr) \ - ((void *)((unsigned)(addr) | CONFIG_NIOS2_IO_REGION_BASE)) -# define CAC_ADDR(addr) \ - ((void *)(((unsigned)(addr) & ~CONFIG_NIOS2_IO_REGION_BASE) | \ - CONFIG_NIOS2_KERNEL_REGION_BASE)) - #include #include diff --git a/arch/nios2/mm/dma-mapping.c b/arch/nios2/mm/dma-mapping.c index 4af9e5b5ba1c..9cb238664584 100644 --- a/arch/nios2/mm/dma-mapping.c +++ b/arch/nios2/mm/dma-mapping.c @@ -60,32 +60,28 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, } } -void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, - gfp_t gfp, unsigned long attrs) +void arch_dma_prep_coherent(struct page *page, size_t size) { - void *ret; + unsigned long start = (unsigned long)page_address(page); - /* optimized page clearing */ - gfp |= __GFP_ZERO; - - if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff)) - gfp |= GFP_DMA; - - ret = (void *) __get_free_pages(gfp, get_order(size)); - if (ret != NULL) { - *dma_handle = virt_to_phys(ret); - flush_dcache_range((unsigned long) ret, - (unsigned long) ret + size); - ret = UNCAC_ADDR(ret); - } - - return ret; + flush_dcache_range(start, start + size); } -void arch_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, unsigned long attrs) +void *uncached_kernel_address(void *ptr) { - unsigned long addr = (unsigned long) CAC_ADDR((unsigned long) vaddr); + unsigned long addr = (unsigned long)ptr; - free_pages(addr, get_order(size)); + addr |= CONFIG_NIOS2_IO_REGION_BASE; + + return (void *)ptr; +} + +void *cached_kernel_address(void *ptr) +{ + unsigned long addr = (unsigned long)ptr; + + addr &= ~CONFIG_NIOS2_IO_REGION_BASE; + addr |= CONFIG_NIOS2_KERNEL_REGION_BASE; + + return (void *)ptr; } From cf394fc5f7155c24efb584979e81427575ab3539 Mon Sep 17 00:00:00 2001 From: Fredrik Noring Date: Tue, 25 Jun 2019 17:05:58 +0200 Subject: [PATCH 26/29] lib/genalloc.c: Add algorithm, align and zeroed family of DMA allocators Provide the algorithm option to DMA allocators as well, along with convenience variants for zeroed and aligned memory. The following four functions are added: - gen_pool_dma_alloc_algo() - gen_pool_dma_alloc_align() - gen_pool_dma_zalloc_algo() - gen_pool_dma_zalloc_align() Signed-off-by: Fredrik Noring Tested-by: Guenter Roeck Signed-off-by: Christoph Hellwig --- include/linux/genalloc.h | 10 +++- lib/genalloc.c | 100 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/include/linux/genalloc.h b/include/linux/genalloc.h index 6c62eeca754f..ed641337df87 100644 --- a/include/linux/genalloc.h +++ b/include/linux/genalloc.h @@ -121,7 +121,15 @@ extern unsigned long gen_pool_alloc_algo(struct gen_pool *, size_t, genpool_algo_t algo, void *data); extern void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); -void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); +extern void *gen_pool_dma_alloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data); +extern void *gen_pool_dma_alloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align); +extern void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); +extern void *gen_pool_dma_zalloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data); +extern void *gen_pool_dma_zalloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align); extern void gen_pool_free(struct gen_pool *, unsigned long, size_t); extern void gen_pool_for_each_chunk(struct gen_pool *, void (*)(struct gen_pool *, struct gen_pool_chunk *, void *), void *); diff --git a/lib/genalloc.c b/lib/genalloc.c index 5db43476a19b..512623fbac51 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -347,13 +347,35 @@ EXPORT_SYMBOL(gen_pool_alloc_algo); * Return: virtual address of the allocated memory, or %NULL on failure */ void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) +{ + return gen_pool_dma_alloc_algo(pool, size, dma, pool->algo, pool->data); +} +EXPORT_SYMBOL(gen_pool_dma_alloc); + +/** + * gen_pool_dma_alloc_algo - allocate special memory from the pool for DMA + * usage with the given pool algorithm + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @algo: algorithm passed from caller + * @data: data passed to algorithm + * + * Allocate the requested number of bytes from the specified pool. Uses the + * given pool allocation function. Can not be used in NMI handler on + * architectures without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated memory, or %NULL on failure + */ +void *gen_pool_dma_alloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data) { unsigned long vaddr; if (!pool) return NULL; - vaddr = gen_pool_alloc(pool, size); + vaddr = gen_pool_alloc_algo(pool, size, algo, data); if (!vaddr) return NULL; @@ -362,7 +384,31 @@ void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) return (void *)vaddr; } -EXPORT_SYMBOL(gen_pool_dma_alloc); +EXPORT_SYMBOL(gen_pool_dma_alloc_algo); + +/** + * gen_pool_dma_alloc_align - allocate special memory from the pool for DMA + * usage with the given alignment + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @align: alignment in bytes for starting address + * + * Allocate the requested number bytes from the specified pool, with the given + * alignment restriction. Can not be used in NMI handler on architectures + * without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated memory, or %NULL on failure + */ +void *gen_pool_dma_alloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align) +{ + struct genpool_data_align data = { .align = align }; + + return gen_pool_dma_alloc_algo(pool, size, dma, + gen_pool_first_fit_align, &data); +} +EXPORT_SYMBOL(gen_pool_dma_alloc_align); /** * gen_pool_dma_zalloc - allocate special zeroed memory from the pool for @@ -380,14 +426,60 @@ EXPORT_SYMBOL(gen_pool_dma_alloc); */ void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) { - void *vaddr = gen_pool_dma_alloc(pool, size, dma); + return gen_pool_dma_zalloc_algo(pool, size, dma, pool->algo, pool->data); +} +EXPORT_SYMBOL(gen_pool_dma_zalloc); + +/** + * gen_pool_dma_zalloc_algo - allocate special zeroed memory from the pool for + * DMA usage with the given pool algorithm + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @algo: algorithm passed from caller + * @data: data passed to algorithm + * + * Allocate the requested number of zeroed bytes from the specified pool. Uses + * the given pool allocation function. Can not be used in NMI handler on + * architectures without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated zeroed memory, or %NULL on failure + */ +void *gen_pool_dma_zalloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data) +{ + void *vaddr = gen_pool_dma_alloc_algo(pool, size, dma, algo, data); if (vaddr) memset(vaddr, 0, size); return vaddr; } -EXPORT_SYMBOL(gen_pool_dma_zalloc); +EXPORT_SYMBOL(gen_pool_dma_zalloc_algo); + +/** + * gen_pool_dma_zalloc_align - allocate special zeroed memory from the pool for + * DMA usage with the given alignment + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @align: alignment in bytes for starting address + * + * Allocate the requested number of zeroed bytes from the specified pool, + * with the given alignment restriction. Can not be used in NMI handler on + * architectures without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated zeroed memory, or %NULL on failure + */ +void *gen_pool_dma_zalloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align) +{ + struct genpool_data_align data = { .align = align }; + + return gen_pool_dma_zalloc_algo(pool, size, dma, + gen_pool_first_fit_align, &data); +} +EXPORT_SYMBOL(gen_pool_dma_zalloc_align); /** * gen_pool_free - free allocated special memory back to the pool From ff2437befd8fe52046e0db949347b5bcfab6b097 Mon Sep 17 00:00:00 2001 From: Fredrik Noring Date: Tue, 25 Jun 2019 17:08:23 +0200 Subject: [PATCH 27/29] usb: host: Fix excessive alignment restriction for local memory allocations The PAGE_SHIFT alignment restriction to devm_gen_pool_create() quickly exhaust local memory because most allocations are much smaller than PAGE_SIZE. This causes USB device failures such as usb 1-2.1: reset full-speed USB device number 4 using sm501-usb sd 1:0:0:0: [sda] tag#0 UNKNOWN(0x2003) Result: hostbyte=0x03 driverbyte=0x00 sd 1:0:0:0: [sda] tag#0 CDB: opcode=0x28 28 00 00 00 08 7c 00 00 f0 00 print_req_error: I/O error, dev sda, sector 2172 flags 80700 when trying to boot from the SM501 USB controller on SH4 with QEMU. Align allocations as required but not necessarily much more than that. The HCCA, TD and ED structures align with 256, 32 and 16 byte memory boundaries, as specified by the Open HCI[1]. The min_alloc_order argument to devm_gen_pool_create is now somewhat arbitrarily set to 4 (16 bytes). Perhaps it could be somewhat lower for general buffer allocations. Reference: [1] "Open Host Controller Interface Specification for USB", release 1.0a, Compaq, Microsoft, National Semiconductor, 1999, pp. 16, 19, 33. Reported-by: Guenter Roeck Signed-off-by: Fredrik Noring Tested-by: Guenter Roeck Signed-off-by: Christoph Hellwig --- drivers/usb/core/hcd.c | 2 +- drivers/usb/host/ohci-hcd.c | 4 ++-- drivers/usb/host/ohci-mem.c | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index fe631d18c1ed..88533938ce19 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -3044,7 +3044,7 @@ int usb_hcd_setup_local_mem(struct usb_hcd *hcd, phys_addr_t phys_addr, int err; void *local_mem; - hcd->localmem_pool = devm_gen_pool_create(hcd->self.sysdev, PAGE_SHIFT, + hcd->localmem_pool = devm_gen_pool_create(hcd->self.sysdev, 4, dev_to_node(hcd->self.sysdev), dev_name(hcd->self.sysdev)); if (IS_ERR(hcd->localmem_pool)) diff --git a/drivers/usb/host/ohci-hcd.c b/drivers/usb/host/ohci-hcd.c index 5801858d867e..b457fdaff297 100644 --- a/drivers/usb/host/ohci-hcd.c +++ b/drivers/usb/host/ohci-hcd.c @@ -507,9 +507,9 @@ static int ohci_init (struct ohci_hcd *ohci) ohci->prev_frame_no = IO_WATCHDOG_OFF; if (hcd->localmem_pool) - ohci->hcca = gen_pool_dma_alloc(hcd->localmem_pool, + ohci->hcca = gen_pool_dma_alloc_align(hcd->localmem_pool, sizeof(*ohci->hcca), - &ohci->hcca_dma); + &ohci->hcca_dma, 256); else ohci->hcca = dma_alloc_coherent(hcd->self.controller, sizeof(*ohci->hcca), diff --git a/drivers/usb/host/ohci-mem.c b/drivers/usb/host/ohci-mem.c index 4afe27cc7e46..1425335c6baf 100644 --- a/drivers/usb/host/ohci-mem.c +++ b/drivers/usb/host/ohci-mem.c @@ -94,7 +94,8 @@ td_alloc (struct ohci_hcd *hc, gfp_t mem_flags) struct usb_hcd *hcd = ohci_to_hcd(hc); if (hcd->localmem_pool) - td = gen_pool_dma_zalloc(hcd->localmem_pool, sizeof(*td), &dma); + td = gen_pool_dma_zalloc_align(hcd->localmem_pool, + sizeof(*td), &dma, 32); else td = dma_pool_zalloc(hc->td_cache, mem_flags, &dma); if (td) { @@ -137,7 +138,8 @@ ed_alloc (struct ohci_hcd *hc, gfp_t mem_flags) struct usb_hcd *hcd = ohci_to_hcd(hc); if (hcd->localmem_pool) - ed = gen_pool_dma_zalloc(hcd->localmem_pool, sizeof(*ed), &dma); + ed = gen_pool_dma_zalloc_align(hcd->localmem_pool, + sizeof(*ed), &dma, 16); else ed = dma_pool_zalloc(hc->ed_cache, mem_flags, &dma); if (ed) { From 2ee7a4ef98e309a1f496be2b0bf9307b5a7a31e5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 30 Jun 2019 18:43:47 +0200 Subject: [PATCH 28/29] MIPS: only select ARCH_HAS_UNCACHED_SEGMENT for non-coherent platforms While mips might architecturally have the uncached segment all the time, the infrastructure to use it is only need on platforms where DMA is at least partially incoherent. Only select it for those configuration to fix a build failure as the arch_dma_prep_coherent symbol is also only provided for non-coherent platforms. Fixes: 2e96e04d25ca ("MIPS: use the generic uncached segment support in dma-direct") Reported-by: Guenter Roeck Signed-off-by: Christoph Hellwig Acked-by: Paul Burton Tested-by: Guenter Roeck --- arch/mips/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 61a390c2f2c1..caf480275a31 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -8,7 +8,6 @@ config MIPS select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN_SANITIZE_ALL - select ARCH_HAS_UNCACHED_SEGMENT select ARCH_SUPPORTS_UPROBES select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF if 64BIT @@ -1120,6 +1119,7 @@ config DMA_NONCOHERENT bool select ARCH_HAS_DMA_MMAP_PGPROT select ARCH_HAS_SYNC_DMA_FOR_DEVICE + select ARCH_HAS_UNCACHED_SEGMENT select NEED_DMA_MAP_STATE select ARCH_HAS_DMA_COHERENT_TO_PFN select DMA_NONCOHERENT_CACHE_SYNC From 15ffe5e1acf5fe1512e98b20702e46ce9f25e2f7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 8 Jul 2019 12:55:27 -0700 Subject: [PATCH 29/29] dma-mapping: mark dma_alloc_need_uncached as __always_inline Without the __always_inline at least i386 configs that have CONFIG_OPTIMIZE_INLINING set seem fail to inline dma_alloc_need_uncached, leading to a linker error because of undefined symbols. Reported-by: Randy Dunlap Signed-off-by: Christoph Hellwig Acked-by: Randy Dunlap # build-tested --- include/linux/dma-noncoherent.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 53ee36ecdf37..3813211a9aad 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -23,7 +23,7 @@ static inline bool dev_is_dma_coherent(struct device *dev) /* * Check if an allocation needs to be marked uncached to be coherent. */ -static inline bool dma_alloc_need_uncached(struct device *dev, +static __always_inline bool dma_alloc_need_uncached(struct device *dev, unsigned long attrs) { if (dev_is_dma_coherent(dev))