2005-04-16 16:20:36 -06:00
|
|
|
/*
|
2008-09-25 08:59:19 -06:00
|
|
|
* linux/arch/arm/mm/dma-mapping.c
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* Copyright (C) 2000-2004 Russell King
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* DMA uncached mapping support.
|
|
|
|
*/
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/mm.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 02:04:11 -06:00
|
|
|
#include <linux/gfp.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/list.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/device.h>
|
|
|
|
#include <linux/dma-mapping.h>
|
2011-12-29 05:09:51 -07:00
|
|
|
#include <linux/dma-contiguous.h>
|
2010-12-15 13:14:45 -07:00
|
|
|
#include <linux/highmem.h>
|
2011-12-29 05:09:51 -07:00
|
|
|
#include <linux/memblock.h>
|
2011-08-02 10:28:27 -06:00
|
|
|
#include <linux/slab.h>
|
2012-05-16 07:48:21 -06:00
|
|
|
#include <linux/iommu.h>
|
2012-07-30 01:11:33 -06:00
|
|
|
#include <linux/io.h>
|
2012-05-16 07:48:21 -06:00
|
|
|
#include <linux/vmalloc.h>
|
2012-06-24 05:46:26 -06:00
|
|
|
#include <linux/sizes.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-04-01 16:07:39 -07:00
|
|
|
#include <asm/memory.h>
|
2009-03-12 20:52:09 -06:00
|
|
|
#include <asm/highmem.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <asm/cacheflush.h>
|
|
|
|
#include <asm/tlbflush.h>
|
2011-08-02 10:28:27 -06:00
|
|
|
#include <asm/mach/arch.h>
|
2012-05-16 07:48:21 -06:00
|
|
|
#include <asm/dma-iommu.h>
|
2011-12-29 05:09:51 -07:00
|
|
|
#include <asm/mach/map.h>
|
|
|
|
#include <asm/system_info.h>
|
|
|
|
#include <asm/dma-contiguous.h>
|
2006-01-12 09:12:21 -07:00
|
|
|
|
2011-07-08 14:26:59 -06:00
|
|
|
#include "mm.h"
|
|
|
|
|
2012-02-10 11:55:20 -07:00
|
|
|
/*
|
|
|
|
* The DMA API is built upon the notion of "buffer ownership". A buffer
|
|
|
|
* is either exclusively owned by the CPU (and therefore may be accessed
|
|
|
|
* by it) or exclusively owned by the DMA device. These helper functions
|
|
|
|
* represent the transitions between these two ownership states.
|
|
|
|
*
|
|
|
|
* Note, however, that on later ARMs, this notion does not work due to
|
|
|
|
* speculative prefetches. We model our approach on the assumption that
|
|
|
|
* the CPU does do speculative prefetches, which means we clean caches
|
|
|
|
* before transfers and delay cache invalidation until transfer completion.
|
|
|
|
*
|
|
|
|
*/
|
2012-02-10 11:55:20 -07:00
|
|
|
static void __dma_page_cpu_to_dev(struct page *, unsigned long,
|
2012-02-10 11:55:20 -07:00
|
|
|
size_t, enum dma_data_direction);
|
2012-02-10 11:55:20 -07:00
|
|
|
static void __dma_page_dev_to_cpu(struct page *, unsigned long,
|
2012-02-10 11:55:20 -07:00
|
|
|
size_t, enum dma_data_direction);
|
|
|
|
|
2012-02-10 11:55:20 -07:00
|
|
|
/**
|
|
|
|
* arm_dma_map_page - map a portion of a page for streaming DMA
|
|
|
|
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
|
|
|
|
* @page: page that buffer resides in
|
|
|
|
* @offset: offset into page for start of buffer
|
|
|
|
* @size: size of buffer to map
|
|
|
|
* @dir: DMA transfer direction
|
|
|
|
*
|
|
|
|
* Ensure that any data held in the cache is appropriately discarded
|
|
|
|
* or written back.
|
|
|
|
*
|
|
|
|
* The device owns this memory once this call has completed. The CPU
|
|
|
|
* can regain ownership by calling dma_unmap_page().
|
|
|
|
*/
|
2012-02-10 11:55:20 -07:00
|
|
|
static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
|
2012-02-10 11:55:20 -07:00
|
|
|
unsigned long offset, size_t size, enum dma_data_direction dir,
|
|
|
|
struct dma_attrs *attrs)
|
|
|
|
{
|
2012-06-06 06:50:56 -06:00
|
|
|
if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
|
2012-02-10 11:55:20 -07:00
|
|
|
__dma_page_cpu_to_dev(page, offset, size, dir);
|
|
|
|
return pfn_to_dma(dev, page_to_pfn(page)) + offset;
|
2012-02-10 11:55:20 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
|
|
|
|
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
|
|
|
|
* @handle: DMA address of buffer
|
|
|
|
* @size: size of buffer (same as passed to dma_map_page)
|
|
|
|
* @dir: DMA transfer direction (same as passed to dma_map_page)
|
|
|
|
*
|
|
|
|
* Unmap a page streaming mode DMA translation. The handle and size
|
|
|
|
* must match what was provided in the previous dma_map_page() call.
|
|
|
|
* All other usages are undefined.
|
|
|
|
*
|
|
|
|
* After this call, reads by the CPU to the buffer are guaranteed to see
|
|
|
|
* whatever the device wrote there.
|
|
|
|
*/
|
2012-02-10 11:55:20 -07:00
|
|
|
static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
|
2012-02-10 11:55:20 -07:00
|
|
|
size_t size, enum dma_data_direction dir,
|
|
|
|
struct dma_attrs *attrs)
|
|
|
|
{
|
2012-06-06 06:50:56 -06:00
|
|
|
if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
|
2012-02-10 11:55:20 -07:00
|
|
|
__dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
|
|
|
|
handle & ~PAGE_MASK, size, dir);
|
2012-02-10 11:55:20 -07:00
|
|
|
}
|
|
|
|
|
2012-02-10 11:55:20 -07:00
|
|
|
static void arm_dma_sync_single_for_cpu(struct device *dev,
|
2012-02-10 11:55:20 -07:00
|
|
|
dma_addr_t handle, size_t size, enum dma_data_direction dir)
|
|
|
|
{
|
|
|
|
unsigned int offset = handle & (PAGE_SIZE - 1);
|
|
|
|
struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
|
2012-02-10 11:55:20 -07:00
|
|
|
if (!arch_is_coherent())
|
|
|
|
__dma_page_dev_to_cpu(page, offset, size, dir);
|
2012-02-10 11:55:20 -07:00
|
|
|
}
|
|
|
|
|
2012-02-10 11:55:20 -07:00
|
|
|
static void arm_dma_sync_single_for_device(struct device *dev,
|
2012-02-10 11:55:20 -07:00
|
|
|
dma_addr_t handle, size_t size, enum dma_data_direction dir)
|
|
|
|
{
|
|
|
|
unsigned int offset = handle & (PAGE_SIZE - 1);
|
|
|
|
struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
|
2012-02-10 11:55:20 -07:00
|
|
|
if (!arch_is_coherent())
|
|
|
|
__dma_page_cpu_to_dev(page, offset, size, dir);
|
2012-02-10 11:55:20 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static int arm_dma_set_mask(struct device *dev, u64 dma_mask);
|
|
|
|
|
|
|
|
struct dma_map_ops arm_dma_ops = {
|
2012-05-16 10:31:23 -06:00
|
|
|
.alloc = arm_dma_alloc,
|
|
|
|
.free = arm_dma_free,
|
|
|
|
.mmap = arm_dma_mmap,
|
2012-06-13 02:01:15 -06:00
|
|
|
.get_sgtable = arm_dma_get_sgtable,
|
2012-02-10 11:55:20 -07:00
|
|
|
.map_page = arm_dma_map_page,
|
|
|
|
.unmap_page = arm_dma_unmap_page,
|
|
|
|
.map_sg = arm_dma_map_sg,
|
|
|
|
.unmap_sg = arm_dma_unmap_sg,
|
|
|
|
.sync_single_for_cpu = arm_dma_sync_single_for_cpu,
|
|
|
|
.sync_single_for_device = arm_dma_sync_single_for_device,
|
|
|
|
.sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
|
|
|
|
.sync_sg_for_device = arm_dma_sync_sg_for_device,
|
|
|
|
.set_dma_mask = arm_dma_set_mask,
|
|
|
|
};
|
|
|
|
EXPORT_SYMBOL(arm_dma_ops);
|
|
|
|
|
2009-07-24 05:35:02 -06:00
|
|
|
static u64 get_coherent_dma_mask(struct device *dev)
|
|
|
|
{
|
2011-07-08 14:26:59 -06:00
|
|
|
u64 mask = (u64)arm_dma_limit;
|
2009-07-24 05:35:02 -06:00
|
|
|
|
|
|
|
if (dev) {
|
|
|
|
mask = dev->coherent_dma_mask;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Sanity check the DMA mask - it must be non-zero, and
|
|
|
|
* must be able to be satisfied by a DMA allocation.
|
|
|
|
*/
|
|
|
|
if (mask == 0) {
|
|
|
|
dev_warn(dev, "coherent DMA mask is unset\n");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-07-08 14:26:59 -06:00
|
|
|
if ((~mask) & (u64)arm_dma_limit) {
|
2009-07-24 05:35:02 -06:00
|
|
|
dev_warn(dev, "coherent DMA mask %#llx is smaller "
|
|
|
|
"than system GFP_DMA mask %#llx\n",
|
2011-07-08 14:26:59 -06:00
|
|
|
mask, (u64)arm_dma_limit);
|
2009-07-24 05:35:02 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-07-24 05:35:02 -06:00
|
|
|
return mask;
|
|
|
|
}
|
|
|
|
|
2011-12-29 05:09:51 -07:00
|
|
|
static void __dma_clear_buffer(struct page *page, size_t size)
|
|
|
|
{
|
|
|
|
void *ptr;
|
|
|
|
/*
|
|
|
|
* Ensure that the allocated pages are zeroed, and that any data
|
|
|
|
* lurking in the kernel direct-mapped region is invalidated.
|
|
|
|
*/
|
|
|
|
ptr = page_address(page);
|
2012-05-16 07:48:21 -06:00
|
|
|
if (ptr) {
|
|
|
|
memset(ptr, 0, size);
|
|
|
|
dmac_flush_range(ptr, ptr + size);
|
|
|
|
outer_flush_range(__pa(ptr), __pa(ptr) + size);
|
|
|
|
}
|
2011-12-29 05:09:51 -07:00
|
|
|
}
|
|
|
|
|
2009-11-19 08:31:07 -07:00
|
|
|
/*
|
|
|
|
* Allocate a DMA buffer for 'dev' of size 'size' using the
|
|
|
|
* specified gfp mask. Note that 'size' must be page aligned.
|
|
|
|
*/
|
|
|
|
static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
|
|
|
|
{
|
|
|
|
unsigned long order = get_order(size);
|
|
|
|
struct page *page, *p, *e;
|
|
|
|
|
|
|
|
page = alloc_pages(gfp, order);
|
|
|
|
if (!page)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Now split the huge page and free the excess pages
|
|
|
|
*/
|
|
|
|
split_page(page, order);
|
|
|
|
for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
|
|
|
|
__free_page(p);
|
|
|
|
|
2011-12-29 05:09:51 -07:00
|
|
|
__dma_clear_buffer(page, size);
|
2009-11-19 08:31:07 -07:00
|
|
|
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Free a DMA buffer. 'size' must be page aligned.
|
|
|
|
*/
|
|
|
|
static void __dma_free_buffer(struct page *page, size_t size)
|
|
|
|
{
|
|
|
|
struct page *e = page + (size >> PAGE_SHIFT);
|
|
|
|
|
|
|
|
while (page < e) {
|
|
|
|
__free_page(page);
|
|
|
|
page++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-07-24 05:35:02 -06:00
|
|
|
#ifdef CONFIG_MMU
|
2012-07-30 01:11:33 -06:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
#error ARM Coherent DMA allocator does not (yet) support huge TLB
|
|
|
|
#endif
|
2010-06-21 08:09:06 -06:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
static void *__alloc_from_contiguous(struct device *dev, size_t size,
|
|
|
|
pgprot_t prot, struct page **ret_page);
|
2011-08-02 10:28:27 -06:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
|
|
|
|
pgprot_t prot, struct page **ret_page,
|
|
|
|
const void *caller);
|
2011-08-02 10:28:27 -06:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
static void *
|
|
|
|
__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
|
|
|
|
const void *caller)
|
2011-08-02 10:28:27 -06:00
|
|
|
{
|
2012-07-30 01:11:33 -06:00
|
|
|
struct vm_struct *area;
|
|
|
|
unsigned long addr;
|
2011-08-02 10:28:27 -06:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
/*
|
|
|
|
* DMA allocation can be mapped to user space, so lets
|
|
|
|
* set VM_USERMAP flags too.
|
|
|
|
*/
|
|
|
|
area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
|
|
|
|
caller);
|
|
|
|
if (!area)
|
|
|
|
return NULL;
|
|
|
|
addr = (unsigned long)area->addr;
|
|
|
|
area->phys_addr = __pfn_to_phys(page_to_pfn(page));
|
2011-08-02 10:28:27 -06:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) {
|
|
|
|
vunmap((void *)addr);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
return (void *)addr;
|
2011-08-02 10:28:27 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
static void __dma_free_remap(void *cpu_addr, size_t size)
|
2009-11-19 09:46:02 -07:00
|
|
|
{
|
2012-07-30 01:11:33 -06:00
|
|
|
unsigned int flags = VM_ARM_DMA_CONSISTENT | VM_USERMAP;
|
|
|
|
struct vm_struct *area = find_vm_area(cpu_addr);
|
|
|
|
if (!area || (area->flags & flags) != flags) {
|
|
|
|
WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
|
|
|
|
return;
|
2011-08-02 10:28:27 -06:00
|
|
|
}
|
2012-07-30 01:11:33 -06:00
|
|
|
unmap_kernel_range((unsigned long)cpu_addr, size);
|
|
|
|
vunmap(cpu_addr);
|
2009-11-19 09:46:02 -07:00
|
|
|
}
|
|
|
|
|
2012-08-20 03:19:25 -06:00
|
|
|
#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
struct dma_pool {
|
|
|
|
size_t size;
|
|
|
|
spinlock_t lock;
|
|
|
|
unsigned long *bitmap;
|
|
|
|
unsigned long nr_pages;
|
|
|
|
void *vaddr;
|
2012-08-27 23:13:01 -06:00
|
|
|
struct page **pages;
|
2011-12-29 05:09:51 -07:00
|
|
|
};
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
static struct dma_pool atomic_pool = {
|
2012-08-20 03:19:25 -06:00
|
|
|
.size = DEFAULT_DMA_COHERENT_POOL_SIZE,
|
2012-07-30 01:11:33 -06:00
|
|
|
};
|
2011-12-29 05:09:51 -07:00
|
|
|
|
|
|
|
static int __init early_coherent_pool(char *p)
|
|
|
|
{
|
2012-07-30 01:11:33 -06:00
|
|
|
atomic_pool.size = memparse(p, &p);
|
2011-12-29 05:09:51 -07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("coherent_pool", early_coherent_pool);
|
|
|
|
|
2012-08-20 03:19:25 -06:00
|
|
|
void __init init_dma_coherent_pool_size(unsigned long size)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Catch any attempt to set the pool size too late.
|
|
|
|
*/
|
|
|
|
BUG_ON(atomic_pool.vaddr);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set architecture specific coherent pool size only if
|
|
|
|
* it has not been changed by kernel command line parameter.
|
|
|
|
*/
|
|
|
|
if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
|
|
|
|
atomic_pool.size = size;
|
|
|
|
}
|
|
|
|
|
2011-12-29 05:09:51 -07:00
|
|
|
/*
|
|
|
|
* Initialise the coherent pool for atomic allocations.
|
|
|
|
*/
|
2012-07-30 01:11:33 -06:00
|
|
|
static int __init atomic_pool_init(void)
|
2011-12-29 05:09:51 -07:00
|
|
|
{
|
2012-07-30 01:11:33 -06:00
|
|
|
struct dma_pool *pool = &atomic_pool;
|
2011-12-29 05:09:51 -07:00
|
|
|
pgprot_t prot = pgprot_dmacoherent(pgprot_kernel);
|
2012-07-30 01:11:33 -06:00
|
|
|
unsigned long nr_pages = pool->size >> PAGE_SHIFT;
|
|
|
|
unsigned long *bitmap;
|
2011-12-29 05:09:51 -07:00
|
|
|
struct page *page;
|
2012-08-27 23:13:01 -06:00
|
|
|
struct page **pages;
|
2011-12-29 05:09:51 -07:00
|
|
|
void *ptr;
|
2012-07-30 01:11:33 -06:00
|
|
|
int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
|
2011-12-29 05:09:51 -07:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
bitmap = kzalloc(bitmap_size, GFP_KERNEL);
|
|
|
|
if (!bitmap)
|
|
|
|
goto no_bitmap;
|
2011-12-29 05:09:51 -07:00
|
|
|
|
2012-08-27 23:13:01 -06:00
|
|
|
pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
|
|
|
|
if (!pages)
|
|
|
|
goto no_pages;
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
if (IS_ENABLED(CONFIG_CMA))
|
|
|
|
ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page);
|
|
|
|
else
|
|
|
|
ptr = __alloc_remap_buffer(NULL, pool->size, GFP_KERNEL, prot,
|
|
|
|
&page, NULL);
|
2011-12-29 05:09:51 -07:00
|
|
|
if (ptr) {
|
2012-08-27 23:13:01 -06:00
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < nr_pages; i++)
|
|
|
|
pages[i] = page + i;
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
spin_lock_init(&pool->lock);
|
|
|
|
pool->vaddr = ptr;
|
2012-08-27 23:13:01 -06:00
|
|
|
pool->pages = pages;
|
2012-07-30 01:11:33 -06:00
|
|
|
pool->bitmap = bitmap;
|
|
|
|
pool->nr_pages = nr_pages;
|
|
|
|
pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
|
|
|
|
(unsigned)pool->size / 1024);
|
2011-12-29 05:09:51 -07:00
|
|
|
return 0;
|
|
|
|
}
|
2012-09-24 00:35:03 -06:00
|
|
|
|
|
|
|
kfree(pages);
|
2012-08-27 23:13:01 -06:00
|
|
|
no_pages:
|
2012-07-30 01:11:33 -06:00
|
|
|
kfree(bitmap);
|
|
|
|
no_bitmap:
|
|
|
|
pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
|
|
|
|
(unsigned)pool->size / 1024);
|
2011-12-29 05:09:51 -07:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* CMA is activated by core_initcall, so we must be called after it.
|
|
|
|
*/
|
2012-07-30 01:11:33 -06:00
|
|
|
postcore_initcall(atomic_pool_init);
|
2011-12-29 05:09:51 -07:00
|
|
|
|
|
|
|
struct dma_contig_early_reserve {
|
|
|
|
phys_addr_t base;
|
|
|
|
unsigned long size;
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
|
|
|
|
|
|
|
|
static int dma_mmu_remap_num __initdata;
|
|
|
|
|
|
|
|
void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
|
|
|
|
{
|
|
|
|
dma_mmu_remap[dma_mmu_remap_num].base = base;
|
|
|
|
dma_mmu_remap[dma_mmu_remap_num].size = size;
|
|
|
|
dma_mmu_remap_num++;
|
|
|
|
}
|
|
|
|
|
|
|
|
void __init dma_contiguous_remap(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
for (i = 0; i < dma_mmu_remap_num; i++) {
|
|
|
|
phys_addr_t start = dma_mmu_remap[i].base;
|
|
|
|
phys_addr_t end = start + dma_mmu_remap[i].size;
|
|
|
|
struct map_desc map;
|
|
|
|
unsigned long addr;
|
|
|
|
|
|
|
|
if (end > arm_lowmem_limit)
|
|
|
|
end = arm_lowmem_limit;
|
|
|
|
if (start >= end)
|
2012-08-07 06:01:14 -06:00
|
|
|
continue;
|
2011-12-29 05:09:51 -07:00
|
|
|
|
|
|
|
map.pfn = __phys_to_pfn(start);
|
|
|
|
map.virtual = __phys_to_virt(start);
|
|
|
|
map.length = end - start;
|
|
|
|
map.type = MT_MEMORY_DMA_READY;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Clear previous low-memory mapping
|
|
|
|
*/
|
|
|
|
for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
|
2012-05-14 11:49:56 -06:00
|
|
|
addr += PMD_SIZE)
|
2011-12-29 05:09:51 -07:00
|
|
|
pmd_clear(pmd_off_k(addr));
|
|
|
|
|
|
|
|
iotable_init(&map, 1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,
|
|
|
|
void *data)
|
|
|
|
{
|
|
|
|
struct page *page = virt_to_page(addr);
|
|
|
|
pgprot_t prot = *(pgprot_t *)data;
|
|
|
|
|
|
|
|
set_pte_ext(pte, mk_pte(page, prot), 0);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
|
|
|
|
{
|
|
|
|
unsigned long start = (unsigned long) page_address(page);
|
|
|
|
unsigned end = start + size;
|
|
|
|
|
|
|
|
apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
|
|
|
|
dsb();
|
|
|
|
flush_tlb_kernel_range(start, end);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
|
|
|
|
pgprot_t prot, struct page **ret_page,
|
|
|
|
const void *caller)
|
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
void *ptr;
|
|
|
|
page = __dma_alloc_buffer(dev, size, gfp);
|
|
|
|
if (!page)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
|
|
|
|
if (!ptr) {
|
|
|
|
__dma_free_buffer(page, size);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
*ret_page = page;
|
|
|
|
return ptr;
|
|
|
|
}
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
static void *__alloc_from_pool(size_t size, struct page **ret_page)
|
2011-12-29 05:09:51 -07:00
|
|
|
{
|
2012-07-30 01:11:33 -06:00
|
|
|
struct dma_pool *pool = &atomic_pool;
|
|
|
|
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
|
|
|
unsigned int pageno;
|
|
|
|
unsigned long flags;
|
|
|
|
void *ptr = NULL;
|
2012-08-07 06:39:25 -06:00
|
|
|
unsigned long align_mask;
|
2011-12-29 05:09:51 -07:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
if (!pool->vaddr) {
|
|
|
|
WARN(1, "coherent pool not initialised!\n");
|
2011-12-29 05:09:51 -07:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Align the region allocation - allocations from pool are rather
|
|
|
|
* small, so align them to their order in pages, minimum is a page
|
|
|
|
* size. This helps reduce fragmentation of the DMA space.
|
|
|
|
*/
|
2012-08-07 06:39:25 -06:00
|
|
|
align_mask = (1 << get_order(size)) - 1;
|
2012-07-30 01:11:33 -06:00
|
|
|
|
|
|
|
spin_lock_irqsave(&pool->lock, flags);
|
|
|
|
pageno = bitmap_find_next_zero_area(pool->bitmap, pool->nr_pages,
|
2012-08-07 06:39:25 -06:00
|
|
|
0, count, align_mask);
|
2012-07-30 01:11:33 -06:00
|
|
|
if (pageno < pool->nr_pages) {
|
|
|
|
bitmap_set(pool->bitmap, pageno, count);
|
|
|
|
ptr = pool->vaddr + PAGE_SIZE * pageno;
|
2012-08-27 23:13:01 -06:00
|
|
|
*ret_page = pool->pages[pageno];
|
2012-08-22 06:50:42 -06:00
|
|
|
} else {
|
|
|
|
pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
|
|
|
|
"Please increase it with coherent_pool= kernel parameter!\n",
|
|
|
|
(unsigned)pool->size / 1024);
|
2011-12-29 05:09:51 -07:00
|
|
|
}
|
2012-07-30 01:11:33 -06:00
|
|
|
spin_unlock_irqrestore(&pool->lock, flags);
|
|
|
|
|
|
|
|
return ptr;
|
2011-12-29 05:09:51 -07:00
|
|
|
}
|
|
|
|
|
2012-08-27 23:13:02 -06:00
|
|
|
static bool __in_atomic_pool(void *start, size_t size)
|
|
|
|
{
|
|
|
|
struct dma_pool *pool = &atomic_pool;
|
|
|
|
void *end = start + size;
|
|
|
|
void *pool_start = pool->vaddr;
|
|
|
|
void *pool_end = pool->vaddr + pool->size;
|
|
|
|
|
2012-09-10 08:14:16 -06:00
|
|
|
if (start < pool_start || start >= pool_end)
|
2012-08-27 23:13:02 -06:00
|
|
|
return false;
|
|
|
|
|
|
|
|
if (end <= pool_end)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
|
|
|
|
start, end - 1, pool_start, pool_end - 1);
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
static int __free_from_pool(void *start, size_t size)
|
2011-12-29 05:09:51 -07:00
|
|
|
{
|
2012-07-30 01:11:33 -06:00
|
|
|
struct dma_pool *pool = &atomic_pool;
|
|
|
|
unsigned long pageno, count;
|
|
|
|
unsigned long flags;
|
2011-12-29 05:09:51 -07:00
|
|
|
|
2012-08-27 23:13:02 -06:00
|
|
|
if (!__in_atomic_pool(start, size))
|
2011-12-29 05:09:51 -07:00
|
|
|
return 0;
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
pageno = (start - pool->vaddr) >> PAGE_SHIFT;
|
|
|
|
count = size >> PAGE_SHIFT;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&pool->lock, flags);
|
|
|
|
bitmap_clear(pool->bitmap, pageno, count);
|
|
|
|
spin_unlock_irqrestore(&pool->lock, flags);
|
|
|
|
|
2011-12-29 05:09:51 -07:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *__alloc_from_contiguous(struct device *dev, size_t size,
|
|
|
|
pgprot_t prot, struct page **ret_page)
|
|
|
|
{
|
|
|
|
unsigned long order = get_order(size);
|
|
|
|
size_t count = size >> PAGE_SHIFT;
|
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
page = dma_alloc_from_contiguous(dev, count, order);
|
|
|
|
if (!page)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
__dma_clear_buffer(page, size);
|
|
|
|
__dma_remap(page, size, prot);
|
|
|
|
|
|
|
|
*ret_page = page;
|
|
|
|
return page_address(page);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __free_from_contiguous(struct device *dev, struct page *page,
|
|
|
|
size_t size)
|
|
|
|
{
|
|
|
|
__dma_remap(page, size, pgprot_kernel);
|
|
|
|
dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
|
|
|
|
}
|
|
|
|
|
2012-05-16 10:31:23 -06:00
|
|
|
static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
|
|
|
|
{
|
|
|
|
prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
|
|
|
|
pgprot_writecombine(prot) :
|
|
|
|
pgprot_dmacoherent(prot);
|
|
|
|
return prot;
|
|
|
|
}
|
|
|
|
|
2011-12-29 05:09:51 -07:00
|
|
|
#define nommu() 0
|
|
|
|
|
2009-07-24 05:35:02 -06:00
|
|
|
#else /* !CONFIG_MMU */
|
2009-11-19 09:31:39 -07:00
|
|
|
|
2011-12-29 05:09:51 -07:00
|
|
|
#define nommu() 1
|
|
|
|
|
2012-05-16 10:31:23 -06:00
|
|
|
#define __get_dma_pgprot(attrs, prot) __pgprot(0)
|
2011-12-29 05:09:51 -07:00
|
|
|
#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL
|
2012-07-30 01:11:33 -06:00
|
|
|
#define __alloc_from_pool(size, ret_page) NULL
|
2011-12-29 05:09:51 -07:00
|
|
|
#define __alloc_from_contiguous(dev, size, prot, ret) NULL
|
|
|
|
#define __free_from_pool(cpu_addr, size) 0
|
|
|
|
#define __free_from_contiguous(dev, page, size) do { } while (0)
|
|
|
|
#define __dma_free_remap(cpu_addr, size) do { } while (0)
|
2009-11-19 14:12:17 -07:00
|
|
|
|
|
|
|
#endif /* CONFIG_MMU */
|
|
|
|
|
2011-12-29 05:09:51 -07:00
|
|
|
static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
|
|
|
|
struct page **ret_page)
|
2009-07-24 05:35:02 -06:00
|
|
|
{
|
2011-12-29 05:09:51 -07:00
|
|
|
struct page *page;
|
|
|
|
page = __dma_alloc_buffer(dev, size, gfp);
|
|
|
|
if (!page)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
*ret_page = page;
|
|
|
|
return page_address(page);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
|
|
|
|
gfp_t gfp, pgprot_t prot, const void *caller)
|
|
|
|
{
|
|
|
|
u64 mask = get_coherent_dma_mask(dev);
|
2009-11-19 08:54:45 -07:00
|
|
|
struct page *page;
|
2009-11-19 14:12:17 -07:00
|
|
|
void *addr;
|
2009-07-24 05:35:02 -06:00
|
|
|
|
2011-12-29 05:09:51 -07:00
|
|
|
#ifdef CONFIG_DMA_API_DEBUG
|
|
|
|
u64 limit = (mask + 1) & ~mask;
|
|
|
|
if (limit && size >= limit) {
|
|
|
|
dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
|
|
|
|
size, mask);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (!mask)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (mask < 0xffffffffULL)
|
|
|
|
gfp |= GFP_DMA;
|
|
|
|
|
2011-11-23 16:47:12 -07:00
|
|
|
/*
|
|
|
|
* Following is a work-around (a.k.a. hack) to prevent pages
|
|
|
|
* with __GFP_COMP being passed to split_page() which cannot
|
|
|
|
* handle them. The real problem is that this flag probably
|
|
|
|
* should be 0 on ARM as it is not supported on this
|
|
|
|
* platform; see CONFIG_HUGETLBFS.
|
|
|
|
*/
|
|
|
|
gfp &= ~(__GFP_COMP);
|
|
|
|
|
2012-02-29 06:45:28 -07:00
|
|
|
*handle = DMA_ERROR_CODE;
|
2009-11-19 08:54:45 -07:00
|
|
|
size = PAGE_ALIGN(size);
|
2009-07-24 05:35:02 -06:00
|
|
|
|
2011-12-29 05:09:51 -07:00
|
|
|
if (arch_is_coherent() || nommu())
|
|
|
|
addr = __alloc_simple_buffer(dev, size, gfp, &page);
|
2012-07-30 01:11:33 -06:00
|
|
|
else if (gfp & GFP_ATOMIC)
|
|
|
|
addr = __alloc_from_pool(size, &page);
|
2012-05-30 02:48:29 -06:00
|
|
|
else if (!IS_ENABLED(CONFIG_CMA))
|
2011-12-29 05:09:51 -07:00
|
|
|
addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);
|
2009-11-19 14:12:17 -07:00
|
|
|
else
|
2011-12-29 05:09:51 -07:00
|
|
|
addr = __alloc_from_contiguous(dev, size, prot, &page);
|
2009-11-19 09:31:39 -07:00
|
|
|
|
2009-11-19 14:12:17 -07:00
|
|
|
if (addr)
|
2011-01-02 17:00:17 -07:00
|
|
|
*handle = pfn_to_dma(dev, page_to_pfn(page));
|
2009-11-19 09:31:39 -07:00
|
|
|
|
2009-11-19 14:12:17 -07:00
|
|
|
return addr;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate DMA-coherent memory space and return both the kernel remapped
|
|
|
|
* virtual and bus address for that space.
|
|
|
|
*/
|
2012-05-16 10:31:23 -06:00
|
|
|
void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
|
|
|
|
gfp_t gfp, struct dma_attrs *attrs)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2012-05-16 10:31:23 -06:00
|
|
|
pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
|
2008-07-18 03:30:14 -06:00
|
|
|
void *memory;
|
|
|
|
|
|
|
|
if (dma_alloc_from_coherent(dev, size, handle, &memory))
|
|
|
|
return memory;
|
|
|
|
|
2012-05-16 10:31:23 -06:00
|
|
|
return __dma_alloc(dev, size, handle, gfp, prot,
|
2012-01-12 16:08:07 -07:00
|
|
|
__builtin_return_address(0));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2012-05-16 10:31:23 -06:00
|
|
|
* Create userspace mapping for the DMA-coherent memory.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2012-05-16 10:31:23 -06:00
|
|
|
int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
|
|
|
|
void *cpu_addr, dma_addr_t dma_addr, size_t size,
|
|
|
|
struct dma_attrs *attrs)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2009-07-24 05:35:02 -06:00
|
|
|
int ret = -ENXIO;
|
|
|
|
#ifdef CONFIG_MMU
|
2012-07-30 01:35:26 -06:00
|
|
|
unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
|
|
|
|
unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
2011-12-29 05:09:51 -07:00
|
|
|
unsigned long pfn = dma_to_pfn(dev, dma_addr);
|
2012-07-30 01:35:26 -06:00
|
|
|
unsigned long off = vma->vm_pgoff;
|
|
|
|
|
2012-05-16 10:31:23 -06:00
|
|
|
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
|
|
|
|
|
2012-05-15 11:04:13 -06:00
|
|
|
if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
|
|
|
|
return ret;
|
|
|
|
|
2012-07-30 01:35:26 -06:00
|
|
|
if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
|
|
|
|
ret = remap_pfn_range(vma, vma->vm_start,
|
|
|
|
pfn + off,
|
|
|
|
vma->vm_end - vma->vm_start,
|
|
|
|
vma->vm_page_prot);
|
|
|
|
}
|
2009-07-24 05:35:02 -06:00
|
|
|
#endif /* CONFIG_MMU */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2011-12-29 05:09:51 -07:00
|
|
|
* Free a buffer as defined by the above mapping.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2012-05-16 10:31:23 -06:00
|
|
|
void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
|
|
|
|
dma_addr_t handle, struct dma_attrs *attrs)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2011-12-29 05:09:51 -07:00
|
|
|
struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
|
2005-11-25 08:52:51 -07:00
|
|
|
|
2008-07-18 03:30:14 -06:00
|
|
|
if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
|
|
|
|
return;
|
|
|
|
|
2009-11-19 08:38:12 -07:00
|
|
|
size = PAGE_ALIGN(size);
|
|
|
|
|
2011-12-29 05:09:51 -07:00
|
|
|
if (arch_is_coherent() || nommu()) {
|
|
|
|
__dma_free_buffer(page, size);
|
2012-08-07 06:44:05 -06:00
|
|
|
} else if (__free_from_pool(cpu_addr, size)) {
|
|
|
|
return;
|
2012-05-30 02:48:29 -06:00
|
|
|
} else if (!IS_ENABLED(CONFIG_CMA)) {
|
2009-11-19 09:31:39 -07:00
|
|
|
__dma_free_remap(cpu_addr, size);
|
2011-12-29 05:09:51 -07:00
|
|
|
__dma_free_buffer(page, size);
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Non-atomic allocations cannot be freed with IRQs disabled
|
|
|
|
*/
|
|
|
|
WARN_ON(irqs_disabled());
|
|
|
|
__free_from_contiguous(dev, page, size);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2008-09-25 09:30:57 -06:00
|
|
|
|
2012-06-13 02:01:15 -06:00
|
|
|
int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
|
|
|
|
void *cpu_addr, dma_addr_t handle, size_t size,
|
|
|
|
struct dma_attrs *attrs)
|
|
|
|
{
|
|
|
|
struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
|
|
|
|
if (unlikely(ret))
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-11-24 09:27:17 -07:00
|
|
|
static void dma_cache_maint_page(struct page *page, unsigned long offset,
|
2009-11-26 09:19:58 -07:00
|
|
|
size_t size, enum dma_data_direction dir,
|
|
|
|
void (*op)(const void *, size_t, int))
|
2009-03-12 20:52:09 -06:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* A single sg entry may refer to multiple physically contiguous
|
|
|
|
* pages. But we still need to process highmem pages individually.
|
|
|
|
* If highmem is not configured then the bulk of this loop gets
|
|
|
|
* optimized out.
|
|
|
|
*/
|
|
|
|
size_t left = size;
|
|
|
|
do {
|
|
|
|
size_t len = left;
|
2009-11-24 07:41:01 -07:00
|
|
|
void *vaddr;
|
|
|
|
|
|
|
|
if (PageHighMem(page)) {
|
|
|
|
if (len + offset > PAGE_SIZE) {
|
|
|
|
if (offset >= PAGE_SIZE) {
|
|
|
|
page += offset / PAGE_SIZE;
|
|
|
|
offset %= PAGE_SIZE;
|
|
|
|
}
|
|
|
|
len = PAGE_SIZE - offset;
|
|
|
|
}
|
|
|
|
vaddr = kmap_high_get(page);
|
|
|
|
if (vaddr) {
|
|
|
|
vaddr += offset;
|
2009-11-26 09:19:58 -07:00
|
|
|
op(vaddr, len, dir);
|
2009-11-24 07:41:01 -07:00
|
|
|
kunmap_high(page);
|
2010-03-29 14:46:02 -06:00
|
|
|
} else if (cache_is_vipt()) {
|
2010-12-15 13:14:45 -07:00
|
|
|
/* unmapped pages might still be cached */
|
|
|
|
vaddr = kmap_atomic(page);
|
2010-03-29 14:46:02 -06:00
|
|
|
op(vaddr + offset, len, dir);
|
2010-12-15 13:14:45 -07:00
|
|
|
kunmap_atomic(vaddr);
|
2009-03-12 20:52:09 -06:00
|
|
|
}
|
2009-11-24 07:41:01 -07:00
|
|
|
} else {
|
|
|
|
vaddr = page_address(page) + offset;
|
2009-11-26 09:19:58 -07:00
|
|
|
op(vaddr, len, dir);
|
2009-03-12 20:52:09 -06:00
|
|
|
}
|
|
|
|
offset = 0;
|
|
|
|
page++;
|
|
|
|
left -= len;
|
|
|
|
} while (left);
|
|
|
|
}
|
2009-11-24 09:27:17 -07:00
|
|
|
|
2012-02-10 11:55:20 -07:00
|
|
|
/*
|
|
|
|
* Make an area consistent for devices.
|
|
|
|
* Note: Drivers should NOT use this function directly, as it will break
|
|
|
|
* platforms with CONFIG_DMABOUNCE.
|
|
|
|
* Use the driver DMA support - see dma-mapping.h (dma_sync_*)
|
|
|
|
*/
|
|
|
|
static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
|
2009-11-24 09:27:17 -07:00
|
|
|
size_t size, enum dma_data_direction dir)
|
|
|
|
{
|
2009-11-24 10:53:33 -07:00
|
|
|
unsigned long paddr;
|
|
|
|
|
2009-11-26 09:19:58 -07:00
|
|
|
dma_cache_maint_page(page, off, size, dir, dmac_map_area);
|
2009-11-24 10:53:33 -07:00
|
|
|
|
|
|
|
paddr = page_to_phys(page) + off;
|
2009-10-31 10:52:16 -06:00
|
|
|
if (dir == DMA_FROM_DEVICE) {
|
|
|
|
outer_inv_range(paddr, paddr + size);
|
|
|
|
} else {
|
|
|
|
outer_clean_range(paddr, paddr + size);
|
|
|
|
}
|
|
|
|
/* FIXME: non-speculating: flush on bidirectional mappings? */
|
2009-11-24 09:27:17 -07:00
|
|
|
}
|
|
|
|
|
2012-02-10 11:55:20 -07:00
|
|
|
static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
|
2009-11-24 09:27:17 -07:00
|
|
|
size_t size, enum dma_data_direction dir)
|
|
|
|
{
|
2009-10-31 10:52:16 -06:00
|
|
|
unsigned long paddr = page_to_phys(page) + off;
|
|
|
|
|
|
|
|
/* FIXME: non-speculating: not required */
|
|
|
|
/* don't bother invalidating if DMA to device */
|
|
|
|
if (dir != DMA_TO_DEVICE)
|
|
|
|
outer_inv_range(paddr, paddr + size);
|
|
|
|
|
2009-11-26 09:19:58 -07:00
|
|
|
dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
|
2010-09-13 08:57:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Mark the D-cache clean for this page to avoid extra flushing.
|
|
|
|
*/
|
|
|
|
if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
|
|
|
|
set_bit(PG_dcache_clean, &page->flags);
|
2009-11-24 09:27:17 -07:00
|
|
|
}
|
2009-03-12 20:52:09 -06:00
|
|
|
|
2008-09-25 09:30:57 -06:00
|
|
|
/**
|
2012-02-10 11:55:20 -07:00
|
|
|
* arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
|
2008-09-25 09:30:57 -06:00
|
|
|
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
|
|
|
|
* @sg: list of buffers
|
|
|
|
* @nents: number of buffers to map
|
|
|
|
* @dir: DMA transfer direction
|
|
|
|
*
|
|
|
|
* Map a set of buffers described by scatterlist in streaming mode for DMA.
|
|
|
|
* This is the scatter-gather version of the dma_map_single interface.
|
|
|
|
* Here the scatter gather list elements are each tagged with the
|
|
|
|
* appropriate dma address and length. They are obtained via
|
|
|
|
* sg_dma_{address,length}.
|
|
|
|
*
|
|
|
|
* Device ownership issues as mentioned for dma_map_single are the same
|
|
|
|
* here.
|
|
|
|
*/
|
2012-02-10 11:55:20 -07:00
|
|
|
int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
|
|
|
|
enum dma_data_direction dir, struct dma_attrs *attrs)
|
2008-09-25 09:30:57 -06:00
|
|
|
{
|
2012-02-10 11:55:20 -07:00
|
|
|
struct dma_map_ops *ops = get_dma_ops(dev);
|
2008-09-25 09:30:57 -06:00
|
|
|
struct scatterlist *s;
|
2008-09-25 14:05:02 -06:00
|
|
|
int i, j;
|
2008-09-25 09:30:57 -06:00
|
|
|
|
|
|
|
for_each_sg(sg, s, nents, i) {
|
2012-05-16 07:48:21 -06:00
|
|
|
#ifdef CONFIG_NEED_SG_DMA_LENGTH
|
|
|
|
s->dma_length = s->length;
|
|
|
|
#endif
|
2012-02-10 11:55:20 -07:00
|
|
|
s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
|
|
|
|
s->length, dir, attrs);
|
2008-09-25 14:05:02 -06:00
|
|
|
if (dma_mapping_error(dev, s->dma_address))
|
|
|
|
goto bad_mapping;
|
2008-09-25 09:30:57 -06:00
|
|
|
}
|
|
|
|
return nents;
|
2008-09-25 14:05:02 -06:00
|
|
|
|
|
|
|
bad_mapping:
|
|
|
|
for_each_sg(sg, s, i, j)
|
2012-02-10 11:55:20 -07:00
|
|
|
ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
|
2008-09-25 14:05:02 -06:00
|
|
|
return 0;
|
2008-09-25 09:30:57 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2012-02-10 11:55:20 -07:00
|
|
|
* arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
|
2008-09-25 09:30:57 -06:00
|
|
|
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
|
|
|
|
* @sg: list of buffers
|
2011-01-12 10:50:37 -07:00
|
|
|
* @nents: number of buffers to unmap (same as was passed to dma_map_sg)
|
2008-09-25 09:30:57 -06:00
|
|
|
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
|
|
|
|
*
|
|
|
|
* Unmap a set of streaming mode DMA translations. Again, CPU access
|
|
|
|
* rules concerning calls here are the same as for dma_unmap_single().
|
|
|
|
*/
|
2012-02-10 11:55:20 -07:00
|
|
|
void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
|
|
|
|
enum dma_data_direction dir, struct dma_attrs *attrs)
|
2008-09-25 09:30:57 -06:00
|
|
|
{
|
2012-02-10 11:55:20 -07:00
|
|
|
struct dma_map_ops *ops = get_dma_ops(dev);
|
2008-09-25 14:05:02 -06:00
|
|
|
struct scatterlist *s;
|
|
|
|
|
|
|
|
int i;
|
2011-01-03 04:29:28 -07:00
|
|
|
|
2008-09-25 14:05:02 -06:00
|
|
|
for_each_sg(sg, s, nents, i)
|
2012-02-10 11:55:20 -07:00
|
|
|
ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
|
2008-09-25 09:30:57 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2012-02-10 11:55:20 -07:00
|
|
|
* arm_dma_sync_sg_for_cpu
|
2008-09-25 09:30:57 -06:00
|
|
|
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
|
|
|
|
* @sg: list of buffers
|
|
|
|
* @nents: number of buffers to map (returned from dma_map_sg)
|
|
|
|
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
|
|
|
|
*/
|
2012-02-10 11:55:20 -07:00
|
|
|
void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
|
2008-09-25 09:30:57 -06:00
|
|
|
int nents, enum dma_data_direction dir)
|
|
|
|
{
|
2012-02-10 11:55:20 -07:00
|
|
|
struct dma_map_ops *ops = get_dma_ops(dev);
|
2008-09-25 09:30:57 -06:00
|
|
|
struct scatterlist *s;
|
|
|
|
int i;
|
|
|
|
|
2012-02-10 11:55:20 -07:00
|
|
|
for_each_sg(sg, s, nents, i)
|
|
|
|
ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
|
|
|
|
dir);
|
2008-09-25 09:30:57 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2012-02-10 11:55:20 -07:00
|
|
|
* arm_dma_sync_sg_for_device
|
2008-09-25 09:30:57 -06:00
|
|
|
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
|
|
|
|
* @sg: list of buffers
|
|
|
|
* @nents: number of buffers to map (returned from dma_map_sg)
|
|
|
|
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
|
|
|
|
*/
|
2012-02-10 11:55:20 -07:00
|
|
|
void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
|
2008-09-25 09:30:57 -06:00
|
|
|
int nents, enum dma_data_direction dir)
|
|
|
|
{
|
2012-02-10 11:55:20 -07:00
|
|
|
struct dma_map_ops *ops = get_dma_ops(dev);
|
2008-09-25 09:30:57 -06:00
|
|
|
struct scatterlist *s;
|
|
|
|
int i;
|
|
|
|
|
2012-02-10 11:55:20 -07:00
|
|
|
for_each_sg(sg, s, nents, i)
|
|
|
|
ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
|
|
|
|
dir);
|
2008-09-25 09:30:57 -06:00
|
|
|
}
|
2011-01-03 04:29:28 -07:00
|
|
|
|
2011-07-08 14:26:59 -06:00
|
|
|
/*
|
|
|
|
* Return whether the given device DMA address mask can be supported
|
|
|
|
* properly. For example, if your device can only drive the low 24-bits
|
|
|
|
* during bus mastering, then you would pass 0x00ffffff as the mask
|
|
|
|
* to this function.
|
|
|
|
*/
|
|
|
|
int dma_supported(struct device *dev, u64 mask)
|
|
|
|
{
|
|
|
|
if (mask < (u64)arm_dma_limit)
|
|
|
|
return 0;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_supported);
|
|
|
|
|
2012-02-10 11:55:20 -07:00
|
|
|
static int arm_dma_set_mask(struct device *dev, u64 dma_mask)
|
2011-07-08 14:26:59 -06:00
|
|
|
{
|
|
|
|
if (!dev->dma_mask || !dma_supported(dev, dma_mask))
|
|
|
|
return -EIO;
|
|
|
|
|
|
|
|
*dev->dma_mask = dma_mask;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-01-03 04:29:28 -07:00
|
|
|
#define PREALLOC_DMA_DEBUG_ENTRIES 4096
|
|
|
|
|
|
|
|
static int __init dma_debug_do_init(void)
|
|
|
|
{
|
|
|
|
dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
fs_initcall(dma_debug_do_init);
|
2012-05-16 07:48:21 -06:00
|
|
|
|
|
|
|
#ifdef CONFIG_ARM_DMA_USE_IOMMU
|
|
|
|
|
|
|
|
/* IOMMU */
|
|
|
|
|
|
|
|
static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
|
|
|
|
size_t size)
|
|
|
|
{
|
|
|
|
unsigned int order = get_order(size);
|
|
|
|
unsigned int align = 0;
|
|
|
|
unsigned int count, start;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) +
|
|
|
|
(1 << mapping->order) - 1) >> mapping->order;
|
|
|
|
|
|
|
|
if (order > mapping->order)
|
|
|
|
align = (1 << (order - mapping->order)) - 1;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&mapping->lock, flags);
|
|
|
|
start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0,
|
|
|
|
count, align);
|
|
|
|
if (start > mapping->bits) {
|
|
|
|
spin_unlock_irqrestore(&mapping->lock, flags);
|
|
|
|
return DMA_ERROR_CODE;
|
|
|
|
}
|
|
|
|
|
|
|
|
bitmap_set(mapping->bitmap, start, count);
|
|
|
|
spin_unlock_irqrestore(&mapping->lock, flags);
|
|
|
|
|
|
|
|
return mapping->base + (start << (mapping->order + PAGE_SHIFT));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void __free_iova(struct dma_iommu_mapping *mapping,
|
|
|
|
dma_addr_t addr, size_t size)
|
|
|
|
{
|
|
|
|
unsigned int start = (addr - mapping->base) >>
|
|
|
|
(mapping->order + PAGE_SHIFT);
|
|
|
|
unsigned int count = ((size >> PAGE_SHIFT) +
|
|
|
|
(1 << mapping->order) - 1) >> mapping->order;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&mapping->lock, flags);
|
|
|
|
bitmap_clear(mapping->bitmap, start, count);
|
|
|
|
spin_unlock_irqrestore(&mapping->lock, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
|
|
|
|
{
|
|
|
|
struct page **pages;
|
|
|
|
int count = size >> PAGE_SHIFT;
|
|
|
|
int array_size = count * sizeof(struct page *);
|
|
|
|
int i = 0;
|
|
|
|
|
|
|
|
if (array_size <= PAGE_SIZE)
|
|
|
|
pages = kzalloc(array_size, gfp);
|
|
|
|
else
|
|
|
|
pages = vzalloc(array_size);
|
|
|
|
if (!pages)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
while (count) {
|
2012-06-21 03:48:11 -06:00
|
|
|
int j, order = __fls(count);
|
2012-05-16 07:48:21 -06:00
|
|
|
|
|
|
|
pages[i] = alloc_pages(gfp | __GFP_NOWARN, order);
|
|
|
|
while (!pages[i] && order)
|
|
|
|
pages[i] = alloc_pages(gfp | __GFP_NOWARN, --order);
|
|
|
|
if (!pages[i])
|
|
|
|
goto error;
|
|
|
|
|
2012-09-10 23:39:39 -06:00
|
|
|
if (order) {
|
2012-05-16 07:48:21 -06:00
|
|
|
split_page(pages[i], order);
|
2012-09-10 23:39:39 -06:00
|
|
|
j = 1 << order;
|
|
|
|
while (--j)
|
|
|
|
pages[i + j] = pages[i] + j;
|
|
|
|
}
|
2012-05-16 07:48:21 -06:00
|
|
|
|
|
|
|
__dma_clear_buffer(pages[i], PAGE_SIZE << order);
|
|
|
|
i += 1 << order;
|
|
|
|
count -= 1 << order;
|
|
|
|
}
|
|
|
|
|
|
|
|
return pages;
|
|
|
|
error:
|
2012-07-27 09:12:50 -06:00
|
|
|
while (i--)
|
2012-05-16 07:48:21 -06:00
|
|
|
if (pages[i])
|
|
|
|
__free_pages(pages[i], 0);
|
2012-07-16 00:59:55 -06:00
|
|
|
if (array_size <= PAGE_SIZE)
|
2012-05-16 07:48:21 -06:00
|
|
|
kfree(pages);
|
|
|
|
else
|
|
|
|
vfree(pages);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __iommu_free_buffer(struct device *dev, struct page **pages, size_t size)
|
|
|
|
{
|
|
|
|
int count = size >> PAGE_SHIFT;
|
|
|
|
int array_size = count * sizeof(struct page *);
|
|
|
|
int i;
|
|
|
|
for (i = 0; i < count; i++)
|
|
|
|
if (pages[i])
|
|
|
|
__free_pages(pages[i], 0);
|
2012-07-16 00:59:55 -06:00
|
|
|
if (array_size <= PAGE_SIZE)
|
2012-05-16 07:48:21 -06:00
|
|
|
kfree(pages);
|
|
|
|
else
|
|
|
|
vfree(pages);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create a CPU mapping for a specified pages
|
|
|
|
*/
|
|
|
|
static void *
|
2012-07-30 01:11:33 -06:00
|
|
|
__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
|
|
|
|
const void *caller)
|
2012-05-16 07:48:21 -06:00
|
|
|
{
|
2012-07-30 01:11:33 -06:00
|
|
|
unsigned int i, nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
|
|
|
struct vm_struct *area;
|
|
|
|
unsigned long p;
|
2012-05-16 07:48:21 -06:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
|
|
|
|
caller);
|
|
|
|
if (!area)
|
2012-05-16 07:48:21 -06:00
|
|
|
return NULL;
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
area->pages = pages;
|
|
|
|
area->nr_pages = nr_pages;
|
|
|
|
p = (unsigned long)area->addr;
|
2012-05-16 07:48:21 -06:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
for (i = 0; i < nr_pages; i++) {
|
|
|
|
phys_addr_t phys = __pfn_to_phys(page_to_pfn(pages[i]));
|
|
|
|
if (ioremap_page_range(p, p + PAGE_SIZE, phys, prot))
|
|
|
|
goto err;
|
|
|
|
p += PAGE_SIZE;
|
2012-05-16 07:48:21 -06:00
|
|
|
}
|
2012-07-30 01:11:33 -06:00
|
|
|
return area->addr;
|
|
|
|
err:
|
|
|
|
unmap_kernel_range((unsigned long)area->addr, size);
|
|
|
|
vunmap(area->addr);
|
2012-05-16 07:48:21 -06:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create a mapping in device IO address space for specified pages
|
|
|
|
*/
|
|
|
|
static dma_addr_t
|
|
|
|
__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
|
|
|
|
{
|
|
|
|
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
|
|
|
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
|
|
|
dma_addr_t dma_addr, iova;
|
|
|
|
int i, ret = DMA_ERROR_CODE;
|
|
|
|
|
|
|
|
dma_addr = __alloc_iova(mapping, size);
|
|
|
|
if (dma_addr == DMA_ERROR_CODE)
|
|
|
|
return dma_addr;
|
|
|
|
|
|
|
|
iova = dma_addr;
|
|
|
|
for (i = 0; i < count; ) {
|
|
|
|
unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
|
|
|
|
phys_addr_t phys = page_to_phys(pages[i]);
|
|
|
|
unsigned int len, j;
|
|
|
|
|
|
|
|
for (j = i + 1; j < count; j++, next_pfn++)
|
|
|
|
if (page_to_pfn(pages[j]) != next_pfn)
|
|
|
|
break;
|
|
|
|
|
|
|
|
len = (j - i) << PAGE_SHIFT;
|
|
|
|
ret = iommu_map(mapping->domain, iova, phys, len, 0);
|
|
|
|
if (ret < 0)
|
|
|
|
goto fail;
|
|
|
|
iova += len;
|
|
|
|
i = j;
|
|
|
|
}
|
|
|
|
return dma_addr;
|
|
|
|
fail:
|
|
|
|
iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
|
|
|
|
__free_iova(mapping, dma_addr, size);
|
|
|
|
return DMA_ERROR_CODE;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
|
|
|
|
{
|
|
|
|
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* add optional in-page offset from iova to size and align
|
|
|
|
* result to page size
|
|
|
|
*/
|
|
|
|
size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
|
|
|
|
iova &= PAGE_MASK;
|
|
|
|
|
|
|
|
iommu_unmap(mapping->domain, iova, size);
|
|
|
|
__free_iova(mapping, iova, size);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-08-27 23:13:03 -06:00
|
|
|
static struct page **__atomic_get_pages(void *addr)
|
|
|
|
{
|
|
|
|
struct dma_pool *pool = &atomic_pool;
|
|
|
|
struct page **pages = pool->pages;
|
|
|
|
int offs = (addr - pool->vaddr) >> PAGE_SHIFT;
|
|
|
|
|
|
|
|
return pages + offs;
|
|
|
|
}
|
|
|
|
|
2012-05-16 12:38:58 -06:00
|
|
|
static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
|
2012-07-30 01:11:33 -06:00
|
|
|
{
|
|
|
|
struct vm_struct *area;
|
|
|
|
|
2012-08-27 23:13:03 -06:00
|
|
|
if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
|
|
|
|
return __atomic_get_pages(cpu_addr);
|
|
|
|
|
2012-05-16 12:38:58 -06:00
|
|
|
if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
|
|
|
|
return cpu_addr;
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
area = find_vm_area(cpu_addr);
|
|
|
|
if (area && (area->flags & VM_ARM_DMA_CONSISTENT))
|
|
|
|
return area->pages;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2012-08-27 23:13:04 -06:00
|
|
|
static void *__iommu_alloc_atomic(struct device *dev, size_t size,
|
|
|
|
dma_addr_t *handle)
|
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
void *addr;
|
|
|
|
|
|
|
|
addr = __alloc_from_pool(size, &page);
|
|
|
|
if (!addr)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
*handle = __iommu_create_mapping(dev, &page, size);
|
|
|
|
if (*handle == DMA_ERROR_CODE)
|
|
|
|
goto err_mapping;
|
|
|
|
|
|
|
|
return addr;
|
|
|
|
|
|
|
|
err_mapping:
|
|
|
|
__free_from_pool(addr, size);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __iommu_free_atomic(struct device *dev, struct page **pages,
|
|
|
|
dma_addr_t handle, size_t size)
|
|
|
|
{
|
|
|
|
__iommu_remove_mapping(dev, handle, size);
|
|
|
|
__free_from_pool(page_address(pages[0]), size);
|
|
|
|
}
|
|
|
|
|
2012-05-16 07:48:21 -06:00
|
|
|
static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
|
|
|
|
dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
|
|
|
|
{
|
|
|
|
pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
|
|
|
|
struct page **pages;
|
|
|
|
void *addr = NULL;
|
|
|
|
|
|
|
|
*handle = DMA_ERROR_CODE;
|
|
|
|
size = PAGE_ALIGN(size);
|
|
|
|
|
2012-08-27 23:13:04 -06:00
|
|
|
if (gfp & GFP_ATOMIC)
|
|
|
|
return __iommu_alloc_atomic(dev, size, handle);
|
|
|
|
|
2012-05-16 07:48:21 -06:00
|
|
|
pages = __iommu_alloc_buffer(dev, size, gfp);
|
|
|
|
if (!pages)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
*handle = __iommu_create_mapping(dev, pages, size);
|
|
|
|
if (*handle == DMA_ERROR_CODE)
|
|
|
|
goto err_buffer;
|
|
|
|
|
2012-05-16 12:38:58 -06:00
|
|
|
if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
|
|
|
|
return pages;
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
addr = __iommu_alloc_remap(pages, size, gfp, prot,
|
|
|
|
__builtin_return_address(0));
|
2012-05-16 07:48:21 -06:00
|
|
|
if (!addr)
|
|
|
|
goto err_mapping;
|
|
|
|
|
|
|
|
return addr;
|
|
|
|
|
|
|
|
err_mapping:
|
|
|
|
__iommu_remove_mapping(dev, *handle, size);
|
|
|
|
err_buffer:
|
|
|
|
__iommu_free_buffer(dev, pages, size);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
|
|
|
|
void *cpu_addr, dma_addr_t dma_addr, size_t size,
|
|
|
|
struct dma_attrs *attrs)
|
|
|
|
{
|
2012-07-30 01:11:33 -06:00
|
|
|
unsigned long uaddr = vma->vm_start;
|
|
|
|
unsigned long usize = vma->vm_end - vma->vm_start;
|
2012-05-16 12:38:58 -06:00
|
|
|
struct page **pages = __iommu_get_pages(cpu_addr, attrs);
|
2012-05-16 07:48:21 -06:00
|
|
|
|
|
|
|
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
if (!pages)
|
|
|
|
return -ENXIO;
|
2012-05-16 07:48:21 -06:00
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
do {
|
|
|
|
int ret = vm_insert_page(vma, uaddr, *pages++);
|
|
|
|
if (ret) {
|
|
|
|
pr_err("Remapping memory failed: %d\n", ret);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
uaddr += PAGE_SIZE;
|
|
|
|
usize -= PAGE_SIZE;
|
|
|
|
} while (usize > 0);
|
2012-05-16 07:48:21 -06:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* free a page as defined by the above mapping.
|
|
|
|
* Must not be called with IRQs disabled.
|
|
|
|
*/
|
|
|
|
void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
|
|
|
|
dma_addr_t handle, struct dma_attrs *attrs)
|
|
|
|
{
|
2012-05-16 12:38:58 -06:00
|
|
|
struct page **pages = __iommu_get_pages(cpu_addr, attrs);
|
2012-05-16 07:48:21 -06:00
|
|
|
size = PAGE_ALIGN(size);
|
|
|
|
|
2012-07-30 01:11:33 -06:00
|
|
|
if (!pages) {
|
|
|
|
WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
|
|
|
|
return;
|
2012-05-16 07:48:21 -06:00
|
|
|
}
|
2012-07-30 01:11:33 -06:00
|
|
|
|
2012-08-27 23:13:04 -06:00
|
|
|
if (__in_atomic_pool(cpu_addr, size)) {
|
|
|
|
__iommu_free_atomic(dev, pages, handle, size);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2012-05-16 12:38:58 -06:00
|
|
|
if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
|
|
|
|
unmap_kernel_range((unsigned long)cpu_addr, size);
|
|
|
|
vunmap(cpu_addr);
|
|
|
|
}
|
2012-07-30 01:11:33 -06:00
|
|
|
|
|
|
|
__iommu_remove_mapping(dev, handle, size);
|
|
|
|
__iommu_free_buffer(dev, pages, size);
|
2012-05-16 07:48:21 -06:00
|
|
|
}
|
|
|
|
|
2012-06-13 02:01:15 -06:00
|
|
|
static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
|
|
|
|
void *cpu_addr, dma_addr_t dma_addr,
|
|
|
|
size_t size, struct dma_attrs *attrs)
|
|
|
|
{
|
|
|
|
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
|
|
|
struct page **pages = __iommu_get_pages(cpu_addr, attrs);
|
|
|
|
|
|
|
|
if (!pages)
|
|
|
|
return -ENXIO;
|
|
|
|
|
|
|
|
return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
|
|
|
|
GFP_KERNEL);
|
2012-05-16 07:48:21 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Map a part of the scatter-gather list into contiguous io address space
|
|
|
|
*/
|
|
|
|
static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
|
|
|
|
size_t size, dma_addr_t *handle,
|
2012-06-06 06:50:56 -06:00
|
|
|
enum dma_data_direction dir, struct dma_attrs *attrs)
|
2012-05-16 07:48:21 -06:00
|
|
|
{
|
|
|
|
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
|
|
|
dma_addr_t iova, iova_base;
|
|
|
|
int ret = 0;
|
|
|
|
unsigned int count;
|
|
|
|
struct scatterlist *s;
|
|
|
|
|
|
|
|
size = PAGE_ALIGN(size);
|
|
|
|
*handle = DMA_ERROR_CODE;
|
|
|
|
|
|
|
|
iova_base = iova = __alloc_iova(mapping, size);
|
|
|
|
if (iova == DMA_ERROR_CODE)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
|
|
|
|
phys_addr_t phys = page_to_phys(sg_page(s));
|
|
|
|
unsigned int len = PAGE_ALIGN(s->offset + s->length);
|
|
|
|
|
2012-06-06 06:50:56 -06:00
|
|
|
if (!arch_is_coherent() &&
|
|
|
|
!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
|
2012-05-16 07:48:21 -06:00
|
|
|
__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
|
|
|
|
|
|
|
|
ret = iommu_map(mapping->domain, iova, phys, len, 0);
|
|
|
|
if (ret < 0)
|
|
|
|
goto fail;
|
|
|
|
count += len >> PAGE_SHIFT;
|
|
|
|
iova += len;
|
|
|
|
}
|
|
|
|
*handle = iova_base;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
fail:
|
|
|
|
iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
|
|
|
|
__free_iova(mapping, iova_base, size);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
|
|
|
|
* @dev: valid struct device pointer
|
|
|
|
* @sg: list of buffers
|
|
|
|
* @nents: number of buffers to map
|
|
|
|
* @dir: DMA transfer direction
|
|
|
|
*
|
|
|
|
* Map a set of buffers described by scatterlist in streaming mode for DMA.
|
|
|
|
* The scatter gather list elements are merged together (if possible) and
|
|
|
|
* tagged with the appropriate dma address and length. They are obtained via
|
|
|
|
* sg_dma_{address,length}.
|
|
|
|
*/
|
|
|
|
int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
|
|
|
|
enum dma_data_direction dir, struct dma_attrs *attrs)
|
|
|
|
{
|
|
|
|
struct scatterlist *s = sg, *dma = sg, *start = sg;
|
|
|
|
int i, count = 0;
|
|
|
|
unsigned int offset = s->offset;
|
|
|
|
unsigned int size = s->offset + s->length;
|
|
|
|
unsigned int max = dma_get_max_seg_size(dev);
|
|
|
|
|
|
|
|
for (i = 1; i < nents; i++) {
|
|
|
|
s = sg_next(s);
|
|
|
|
|
|
|
|
s->dma_address = DMA_ERROR_CODE;
|
|
|
|
s->dma_length = 0;
|
|
|
|
|
|
|
|
if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
|
|
|
|
if (__map_sg_chunk(dev, start, size, &dma->dma_address,
|
2012-06-06 06:50:56 -06:00
|
|
|
dir, attrs) < 0)
|
2012-05-16 07:48:21 -06:00
|
|
|
goto bad_mapping;
|
|
|
|
|
|
|
|
dma->dma_address += offset;
|
|
|
|
dma->dma_length = size - offset;
|
|
|
|
|
|
|
|
size = offset = s->offset;
|
|
|
|
start = s;
|
|
|
|
dma = sg_next(dma);
|
|
|
|
count += 1;
|
|
|
|
}
|
|
|
|
size += s->length;
|
|
|
|
}
|
2012-06-06 06:50:56 -06:00
|
|
|
if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs) < 0)
|
2012-05-16 07:48:21 -06:00
|
|
|
goto bad_mapping;
|
|
|
|
|
|
|
|
dma->dma_address += offset;
|
|
|
|
dma->dma_length = size - offset;
|
|
|
|
|
|
|
|
return count+1;
|
|
|
|
|
|
|
|
bad_mapping:
|
|
|
|
for_each_sg(sg, s, count, i)
|
|
|
|
__iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
|
|
|
|
* @dev: valid struct device pointer
|
|
|
|
* @sg: list of buffers
|
|
|
|
* @nents: number of buffers to unmap (same as was passed to dma_map_sg)
|
|
|
|
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
|
|
|
|
*
|
|
|
|
* Unmap a set of streaming mode DMA translations. Again, CPU access
|
|
|
|
* rules concerning calls here are the same as for dma_unmap_single().
|
|
|
|
*/
|
|
|
|
void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
|
|
|
|
enum dma_data_direction dir, struct dma_attrs *attrs)
|
|
|
|
{
|
|
|
|
struct scatterlist *s;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_sg(sg, s, nents, i) {
|
|
|
|
if (sg_dma_len(s))
|
|
|
|
__iommu_remove_mapping(dev, sg_dma_address(s),
|
|
|
|
sg_dma_len(s));
|
2012-06-06 06:50:56 -06:00
|
|
|
if (!arch_is_coherent() &&
|
|
|
|
!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
|
2012-05-16 07:48:21 -06:00
|
|
|
__dma_page_dev_to_cpu(sg_page(s), s->offset,
|
|
|
|
s->length, dir);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* arm_iommu_sync_sg_for_cpu
|
|
|
|
* @dev: valid struct device pointer
|
|
|
|
* @sg: list of buffers
|
|
|
|
* @nents: number of buffers to map (returned from dma_map_sg)
|
|
|
|
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
|
|
|
|
*/
|
|
|
|
void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
|
|
|
|
int nents, enum dma_data_direction dir)
|
|
|
|
{
|
|
|
|
struct scatterlist *s;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_sg(sg, s, nents, i)
|
|
|
|
if (!arch_is_coherent())
|
|
|
|
__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* arm_iommu_sync_sg_for_device
|
|
|
|
* @dev: valid struct device pointer
|
|
|
|
* @sg: list of buffers
|
|
|
|
* @nents: number of buffers to map (returned from dma_map_sg)
|
|
|
|
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
|
|
|
|
*/
|
|
|
|
void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
|
|
|
|
int nents, enum dma_data_direction dir)
|
|
|
|
{
|
|
|
|
struct scatterlist *s;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_sg(sg, s, nents, i)
|
|
|
|
if (!arch_is_coherent())
|
|
|
|
__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* arm_iommu_map_page
|
|
|
|
* @dev: valid struct device pointer
|
|
|
|
* @page: page that buffer resides in
|
|
|
|
* @offset: offset into page for start of buffer
|
|
|
|
* @size: size of buffer to map
|
|
|
|
* @dir: DMA transfer direction
|
|
|
|
*
|
|
|
|
* IOMMU aware version of arm_dma_map_page()
|
|
|
|
*/
|
|
|
|
static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
|
|
|
|
unsigned long offset, size_t size, enum dma_data_direction dir,
|
|
|
|
struct dma_attrs *attrs)
|
|
|
|
{
|
|
|
|
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
|
|
|
dma_addr_t dma_addr;
|
|
|
|
int ret, len = PAGE_ALIGN(size + offset);
|
|
|
|
|
2012-06-06 06:50:56 -06:00
|
|
|
if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
|
2012-05-16 07:48:21 -06:00
|
|
|
__dma_page_cpu_to_dev(page, offset, size, dir);
|
|
|
|
|
|
|
|
dma_addr = __alloc_iova(mapping, len);
|
|
|
|
if (dma_addr == DMA_ERROR_CODE)
|
|
|
|
return dma_addr;
|
|
|
|
|
|
|
|
ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0);
|
|
|
|
if (ret < 0)
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
return dma_addr + offset;
|
|
|
|
fail:
|
|
|
|
__free_iova(mapping, dma_addr, len);
|
|
|
|
return DMA_ERROR_CODE;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* arm_iommu_unmap_page
|
|
|
|
* @dev: valid struct device pointer
|
|
|
|
* @handle: DMA address of buffer
|
|
|
|
* @size: size of buffer (same as passed to dma_map_page)
|
|
|
|
* @dir: DMA transfer direction (same as passed to dma_map_page)
|
|
|
|
*
|
|
|
|
* IOMMU aware version of arm_dma_unmap_page()
|
|
|
|
*/
|
|
|
|
static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
|
|
|
|
size_t size, enum dma_data_direction dir,
|
|
|
|
struct dma_attrs *attrs)
|
|
|
|
{
|
|
|
|
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
|
|
|
dma_addr_t iova = handle & PAGE_MASK;
|
|
|
|
struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
|
|
|
|
int offset = handle & ~PAGE_MASK;
|
|
|
|
int len = PAGE_ALIGN(size + offset);
|
|
|
|
|
|
|
|
if (!iova)
|
|
|
|
return;
|
|
|
|
|
2012-06-06 06:50:56 -06:00
|
|
|
if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
|
2012-05-16 07:48:21 -06:00
|
|
|
__dma_page_dev_to_cpu(page, offset, size, dir);
|
|
|
|
|
|
|
|
iommu_unmap(mapping->domain, iova, len);
|
|
|
|
__free_iova(mapping, iova, len);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void arm_iommu_sync_single_for_cpu(struct device *dev,
|
|
|
|
dma_addr_t handle, size_t size, enum dma_data_direction dir)
|
|
|
|
{
|
|
|
|
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
|
|
|
dma_addr_t iova = handle & PAGE_MASK;
|
|
|
|
struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
|
|
|
|
unsigned int offset = handle & ~PAGE_MASK;
|
|
|
|
|
|
|
|
if (!iova)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (!arch_is_coherent())
|
|
|
|
__dma_page_dev_to_cpu(page, offset, size, dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void arm_iommu_sync_single_for_device(struct device *dev,
|
|
|
|
dma_addr_t handle, size_t size, enum dma_data_direction dir)
|
|
|
|
{
|
|
|
|
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
|
|
|
dma_addr_t iova = handle & PAGE_MASK;
|
|
|
|
struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
|
|
|
|
unsigned int offset = handle & ~PAGE_MASK;
|
|
|
|
|
|
|
|
if (!iova)
|
|
|
|
return;
|
|
|
|
|
|
|
|
__dma_page_cpu_to_dev(page, offset, size, dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct dma_map_ops iommu_ops = {
|
|
|
|
.alloc = arm_iommu_alloc_attrs,
|
|
|
|
.free = arm_iommu_free_attrs,
|
|
|
|
.mmap = arm_iommu_mmap_attrs,
|
2012-06-13 02:01:15 -06:00
|
|
|
.get_sgtable = arm_iommu_get_sgtable,
|
2012-05-16 07:48:21 -06:00
|
|
|
|
|
|
|
.map_page = arm_iommu_map_page,
|
|
|
|
.unmap_page = arm_iommu_unmap_page,
|
|
|
|
.sync_single_for_cpu = arm_iommu_sync_single_for_cpu,
|
|
|
|
.sync_single_for_device = arm_iommu_sync_single_for_device,
|
|
|
|
|
|
|
|
.map_sg = arm_iommu_map_sg,
|
|
|
|
.unmap_sg = arm_iommu_unmap_sg,
|
|
|
|
.sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu,
|
|
|
|
.sync_sg_for_device = arm_iommu_sync_sg_for_device,
|
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* arm_iommu_create_mapping
|
|
|
|
* @bus: pointer to the bus holding the client device (for IOMMU calls)
|
|
|
|
* @base: start address of the valid IO address space
|
|
|
|
* @size: size of the valid IO address space
|
|
|
|
* @order: accuracy of the IO addresses allocations
|
|
|
|
*
|
|
|
|
* Creates a mapping structure which holds information about used/unused
|
|
|
|
* IO address ranges, which is required to perform memory allocation and
|
|
|
|
* mapping with IOMMU aware functions.
|
|
|
|
*
|
|
|
|
* The client device need to be attached to the mapping with
|
|
|
|
* arm_iommu_attach_device function.
|
|
|
|
*/
|
|
|
|
struct dma_iommu_mapping *
|
|
|
|
arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size,
|
|
|
|
int order)
|
|
|
|
{
|
|
|
|
unsigned int count = size >> (PAGE_SHIFT + order);
|
|
|
|
unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long);
|
|
|
|
struct dma_iommu_mapping *mapping;
|
|
|
|
int err = -ENOMEM;
|
|
|
|
|
|
|
|
if (!count)
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
|
|
|
|
if (!mapping)
|
|
|
|
goto err;
|
|
|
|
|
|
|
|
mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
|
|
|
|
if (!mapping->bitmap)
|
|
|
|
goto err2;
|
|
|
|
|
|
|
|
mapping->base = base;
|
|
|
|
mapping->bits = BITS_PER_BYTE * bitmap_size;
|
|
|
|
mapping->order = order;
|
|
|
|
spin_lock_init(&mapping->lock);
|
|
|
|
|
|
|
|
mapping->domain = iommu_domain_alloc(bus);
|
|
|
|
if (!mapping->domain)
|
|
|
|
goto err3;
|
|
|
|
|
|
|
|
kref_init(&mapping->kref);
|
|
|
|
return mapping;
|
|
|
|
err3:
|
|
|
|
kfree(mapping->bitmap);
|
|
|
|
err2:
|
|
|
|
kfree(mapping);
|
|
|
|
err:
|
|
|
|
return ERR_PTR(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void release_iommu_mapping(struct kref *kref)
|
|
|
|
{
|
|
|
|
struct dma_iommu_mapping *mapping =
|
|
|
|
container_of(kref, struct dma_iommu_mapping, kref);
|
|
|
|
|
|
|
|
iommu_domain_free(mapping->domain);
|
|
|
|
kfree(mapping->bitmap);
|
|
|
|
kfree(mapping);
|
|
|
|
}
|
|
|
|
|
|
|
|
void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
|
|
|
|
{
|
|
|
|
if (mapping)
|
|
|
|
kref_put(&mapping->kref, release_iommu_mapping);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* arm_iommu_attach_device
|
|
|
|
* @dev: valid struct device pointer
|
|
|
|
* @mapping: io address space mapping structure (returned from
|
|
|
|
* arm_iommu_create_mapping)
|
|
|
|
*
|
|
|
|
* Attaches specified io address space mapping to the provided device,
|
|
|
|
* this replaces the dma operations (dma_map_ops pointer) with the
|
|
|
|
* IOMMU aware version. More than one client might be attached to
|
|
|
|
* the same io address space mapping.
|
|
|
|
*/
|
|
|
|
int arm_iommu_attach_device(struct device *dev,
|
|
|
|
struct dma_iommu_mapping *mapping)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = iommu_attach_device(mapping->domain, dev);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
kref_get(&mapping->kref);
|
|
|
|
dev->archdata.mapping = mapping;
|
|
|
|
set_dma_ops(dev, &iommu_ops);
|
|
|
|
|
|
|
|
pr_info("Attached IOMMU controller to %s device.\n", dev_name(dev));
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|