2005-04-16 16:20:36 -06:00
|
|
|
/*
|
2005-12-13 19:10:10 -07:00
|
|
|
* arch/powerpc/sysdev/dart_iommu.c
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
2005-11-21 01:12:32 -07:00
|
|
|
* Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
|
2005-12-13 19:10:10 -07:00
|
|
|
* Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>,
|
|
|
|
* IBM Corporation
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* Based on pSeries_iommu.c:
|
|
|
|
* Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
|
2005-11-21 01:12:32 -07:00
|
|
|
* Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
2005-12-13 19:10:10 -07:00
|
|
|
* Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu.
|
|
|
|
*
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
|
|
* (at your option) any later version.
|
2005-12-13 19:10:10 -07:00
|
|
|
*
|
2005-04-16 16:20:36 -06:00
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
2005-12-13 19:10:10 -07:00
|
|
|
*
|
2005-04-16 16:20:36 -06:00
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/pci.h>
|
|
|
|
#include <linux/dma-mapping.h>
|
|
|
|
#include <linux/vmalloc.h>
|
2007-05-03 06:28:32 -06:00
|
|
|
#include <linux/suspend.h>
|
2010-07-11 22:36:09 -06:00
|
|
|
#include <linux/memblock.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 02:04:11 -06:00
|
|
|
#include <linux/gfp.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <asm/io.h>
|
|
|
|
#include <asm/prom.h>
|
|
|
|
#include <asm/iommu.h>
|
|
|
|
#include <asm/pci-bridge.h>
|
|
|
|
#include <asm/machdep.h>
|
|
|
|
#include <asm/cacheflush.h>
|
2005-09-27 10:50:25 -06:00
|
|
|
#include <asm/ppc-pci.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2005-11-01 21:13:20 -07:00
|
|
|
#include "dart.h"
|
|
|
|
|
2016-07-04 23:03:47 -06:00
|
|
|
/* DART table address and size */
|
|
|
|
static u32 *dart_tablebase;
|
2005-04-16 16:20:36 -06:00
|
|
|
static unsigned long dart_tablesize;
|
|
|
|
|
|
|
|
/* Mapped base address for the dart */
|
2006-02-01 05:28:02 -07:00
|
|
|
static unsigned int __iomem *dart;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Dummy val that entries are set to when unused */
|
|
|
|
static unsigned int dart_emptyval;
|
|
|
|
|
2005-12-13 19:10:10 -07:00
|
|
|
static struct iommu_table iommu_table_dart;
|
|
|
|
static int iommu_table_dart_inited;
|
2005-04-16 16:20:36 -06:00
|
|
|
static int dart_dirty;
|
2005-12-13 19:10:10 -07:00
|
|
|
static int dart_is_u4;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-08-30 13:24:18 -06:00
|
|
|
#define DART_U4_BYPASS_BASE 0x8000000000ull
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#define DBG(...)
|
|
|
|
|
2012-10-03 12:57:10 -06:00
|
|
|
static DEFINE_SPINLOCK(invalidate_lock);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static inline void dart_tlb_invalidate_all(void)
|
|
|
|
{
|
|
|
|
unsigned long l = 0;
|
2005-12-13 19:10:10 -07:00
|
|
|
unsigned int reg, inv_bit;
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned long limit;
|
2012-10-03 12:57:10 -06:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&invalidate_lock, flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
DBG("dart: flush\n");
|
|
|
|
|
|
|
|
/* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the
|
|
|
|
* control register and wait for it to clear.
|
|
|
|
*
|
|
|
|
* Gotcha: Sometimes, the DART won't detect that the bit gets
|
|
|
|
* set. If so, clear it and set it again.
|
2005-12-13 19:10:10 -07:00
|
|
|
*/
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
limit = 0;
|
|
|
|
|
2005-12-13 19:10:10 -07:00
|
|
|
inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB;
|
2005-04-16 16:20:36 -06:00
|
|
|
retry:
|
|
|
|
l = 0;
|
2005-12-13 19:10:10 -07:00
|
|
|
reg = DART_IN(DART_CNTL);
|
|
|
|
reg |= inv_bit;
|
|
|
|
DART_OUT(DART_CNTL, reg);
|
|
|
|
|
|
|
|
while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit))
|
2005-04-16 16:20:36 -06:00
|
|
|
l++;
|
2005-12-13 19:10:10 -07:00
|
|
|
if (l == (1L << limit)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
if (limit < 4) {
|
|
|
|
limit++;
|
2006-06-28 03:50:36 -06:00
|
|
|
reg = DART_IN(DART_CNTL);
|
|
|
|
reg &= ~inv_bit;
|
2005-12-13 19:10:10 -07:00
|
|
|
DART_OUT(DART_CNTL, reg);
|
2005-04-16 16:20:36 -06:00
|
|
|
goto retry;
|
|
|
|
} else
|
2005-12-13 19:10:10 -07:00
|
|
|
panic("DART: TLB did not flush after waiting a long "
|
2005-04-16 16:20:36 -06:00
|
|
|
"time. Buggy U3 ?");
|
|
|
|
}
|
2012-10-03 12:57:10 -06:00
|
|
|
|
|
|
|
spin_unlock_irqrestore(&invalidate_lock, flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-06-28 03:50:36 -06:00
|
|
|
static inline void dart_tlb_invalidate_one(unsigned long bus_rpn)
|
|
|
|
{
|
|
|
|
unsigned int reg;
|
|
|
|
unsigned int l, limit;
|
2012-10-03 12:57:10 -06:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&invalidate_lock, flags);
|
2006-06-28 03:50:36 -06:00
|
|
|
|
|
|
|
reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE |
|
|
|
|
(bus_rpn & DART_CNTL_U4_IONE_MASK);
|
|
|
|
DART_OUT(DART_CNTL, reg);
|
|
|
|
|
|
|
|
limit = 0;
|
|
|
|
wait_more:
|
|
|
|
l = 0;
|
|
|
|
while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) {
|
|
|
|
rmb();
|
|
|
|
l++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (l == (1L << limit)) {
|
|
|
|
if (limit < 4) {
|
|
|
|
limit++;
|
|
|
|
goto wait_more;
|
|
|
|
} else
|
|
|
|
panic("DART: TLB did not flush after waiting a long "
|
|
|
|
"time. Buggy U4 ?");
|
|
|
|
}
|
2012-10-03 12:57:10 -06:00
|
|
|
|
|
|
|
spin_unlock_irqrestore(&invalidate_lock, flags);
|
2006-06-28 03:50:36 -06:00
|
|
|
}
|
|
|
|
|
2016-07-04 23:03:47 -06:00
|
|
|
static void dart_cache_sync(unsigned int *base, unsigned int count)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We add 1 to the number of entries to flush, following a
|
|
|
|
* comment in Darwin indicating that the memory controller
|
|
|
|
* can prefetch unmapped memory under some circumstances.
|
|
|
|
*/
|
|
|
|
unsigned long start = (unsigned long)base;
|
|
|
|
unsigned long end = start + (count + 1) * sizeof(unsigned int);
|
|
|
|
unsigned int tmp;
|
|
|
|
|
|
|
|
/* Perform a standard cache flush */
|
|
|
|
flush_inval_dcache_range(start, end);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Perform the sequence described in the CPC925 manual to
|
|
|
|
* ensure all the data gets to a point the cache incoherent
|
|
|
|
* DART hardware will see.
|
|
|
|
*/
|
|
|
|
asm volatile(" sync;"
|
|
|
|
" isync;"
|
|
|
|
" dcbf 0,%1;"
|
|
|
|
" sync;"
|
|
|
|
" isync;"
|
|
|
|
" lwz %0,0(%1);"
|
|
|
|
" isync" : "=r" (tmp) : "r" (end) : "memory");
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static void dart_flush(struct iommu_table *tbl)
|
|
|
|
{
|
2006-09-13 06:12:52 -06:00
|
|
|
mb();
|
2006-06-28 03:50:36 -06:00
|
|
|
if (dart_dirty) {
|
2005-04-16 16:20:36 -06:00
|
|
|
dart_tlb_invalidate_all();
|
2006-06-28 03:50:36 -06:00
|
|
|
dart_dirty = 0;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-07-23 12:31:16 -06:00
|
|
|
static int dart_build(struct iommu_table *tbl, long index,
|
2005-04-16 16:20:36 -06:00
|
|
|
long npages, unsigned long uaddr,
|
2008-07-15 13:51:47 -06:00
|
|
|
enum dma_data_direction direction,
|
2016-08-03 14:46:00 -06:00
|
|
|
unsigned long attrs)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2016-07-04 23:03:47 -06:00
|
|
|
unsigned int *dp, *orig_dp;
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned int rpn;
|
2006-06-28 03:50:36 -06:00
|
|
|
long l;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr);
|
|
|
|
|
2016-07-04 23:03:47 -06:00
|
|
|
orig_dp = dp = ((unsigned int*)tbl->it_base) + index;
|
2005-12-13 19:10:10 -07:00
|
|
|
|
tree-wide: fix assorted typos all over the place
That is "success", "unknown", "through", "performance", "[re|un]mapping"
, "access", "default", "reasonable", "[con]currently", "temperature"
, "channel", "[un]used", "application", "example","hierarchy", "therefore"
, "[over|under]flow", "contiguous", "threshold", "enough" and others.
Signed-off-by: André Goddard Rosa <andre.goddard@gmail.com>
Signed-off-by: Jiri Kosina <jkosina@suse.cz>
2009-11-14 08:09:05 -07:00
|
|
|
/* On U3, all memory is contiguous, so we can move this
|
2005-04-16 16:20:36 -06:00
|
|
|
* out of the loop.
|
|
|
|
*/
|
2006-06-28 03:50:36 -06:00
|
|
|
l = npages;
|
|
|
|
while (l--) {
|
2012-07-25 15:19:52 -06:00
|
|
|
rpn = __pa(uaddr) >> DART_PAGE_SHIFT;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
*(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK);
|
|
|
|
|
2005-09-19 21:46:44 -06:00
|
|
|
uaddr += DART_PAGE_SIZE;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2016-07-04 23:03:47 -06:00
|
|
|
dart_cache_sync(orig_dp, npages);
|
2006-09-13 06:12:52 -06:00
|
|
|
|
2006-06-28 03:50:36 -06:00
|
|
|
if (dart_is_u4) {
|
|
|
|
rpn = index;
|
|
|
|
while (npages--)
|
|
|
|
dart_tlb_invalidate_one(rpn++);
|
|
|
|
} else {
|
|
|
|
dart_dirty = 1;
|
|
|
|
}
|
2008-07-23 12:31:16 -06:00
|
|
|
return 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static void dart_free(struct iommu_table *tbl, long index, long npages)
|
|
|
|
{
|
2016-07-04 23:03:47 -06:00
|
|
|
unsigned int *dp, *orig_dp;
|
|
|
|
long orig_npages = npages;
|
2005-12-13 19:10:10 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* We don't worry about flushing the TLB cache. The only drawback of
|
|
|
|
* not doing it is that we won't catch buggy device drivers doing
|
|
|
|
* bad DMAs, but then no 32-bit architecture ever does either.
|
|
|
|
*/
|
|
|
|
|
|
|
|
DBG("dart: free at: %lx, %lx\n", index, npages);
|
|
|
|
|
2016-07-04 23:03:47 -06:00
|
|
|
orig_dp = dp = ((unsigned int *)tbl->it_base) + index;
|
2005-12-13 19:10:10 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
while (npages--)
|
|
|
|
*(dp++) = dart_emptyval;
|
|
|
|
|
2016-07-04 23:03:47 -06:00
|
|
|
dart_cache_sync(orig_dp, orig_npages);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2016-07-04 23:03:47 -06:00
|
|
|
static void allocate_dart(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2016-07-04 23:03:47 -06:00
|
|
|
unsigned long tmp;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2016-07-04 23:03:47 -06:00
|
|
|
/* 512 pages (2MB) is max DART tablesize. */
|
|
|
|
dart_tablesize = 1UL << 21;
|
2005-12-13 19:10:10 -07:00
|
|
|
|
2016-07-04 23:03:47 -06:00
|
|
|
/*
|
|
|
|
* 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we
|
|
|
|
* will blow up an entire large page anyway in the kernel mapping.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2016-07-04 23:03:47 -06:00
|
|
|
dart_tablebase = __va(memblock_alloc_base(1UL<<24,
|
|
|
|
1UL<<24, 0x80000000L));
|
|
|
|
|
|
|
|
/* There is no point scanning the DART space for leaks*/
|
|
|
|
kmemleak_no_scan((void *)dart_tablebase);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Allocate a spare page to map all invalid DART pages. We need to do
|
|
|
|
* that to work around what looks like a problem with the HT bridge
|
|
|
|
* prefetching into invalid pages and corrupting data
|
|
|
|
*/
|
2010-07-11 22:36:09 -06:00
|
|
|
tmp = memblock_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE);
|
2005-12-13 19:10:10 -07:00
|
|
|
dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) &
|
|
|
|
DARTMAP_RPNMASK);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2016-07-04 23:03:47 -06:00
|
|
|
printk(KERN_INFO "DART table allocated at: %p\n", dart_tablebase);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init dart_init(struct device_node *dart_node)
|
|
|
|
{
|
|
|
|
unsigned int i;
|
|
|
|
unsigned long base, size;
|
|
|
|
struct resource r;
|
|
|
|
|
|
|
|
/* IOMMU disabled by the user ? bail out */
|
|
|
|
if (iommu_is_off)
|
|
|
|
return -ENODEV;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Only use the DART if the machine has more than 1GB of RAM
|
|
|
|
* or if requested with iommu=on on cmdline.
|
|
|
|
*
|
|
|
|
* 1GB of RAM is picked as limit because some default devices
|
|
|
|
* (i.e. Airport Extreme) have 30 bit address range limits.
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (!iommu_force_on && memblock_end_of_DRAM() <= 0x40000000ull)
|
|
|
|
return -ENODEV;
|
|
|
|
|
|
|
|
/* Get DART registers */
|
|
|
|
if (of_address_to_resource(dart_node, 0, &r))
|
|
|
|
panic("DART: can't get register base ! ");
|
|
|
|
|
2005-12-13 19:10:10 -07:00
|
|
|
/* Map in DART registers */
|
2011-06-09 10:13:32 -06:00
|
|
|
dart = ioremap(r.start, resource_size(&r));
|
2005-04-16 16:20:36 -06:00
|
|
|
if (dart == NULL)
|
2005-12-13 19:10:10 -07:00
|
|
|
panic("DART: Cannot map registers!");
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2016-07-04 23:03:47 -06:00
|
|
|
/* Allocate the DART and dummy page */
|
|
|
|
allocate_dart();
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Fill initial table */
|
|
|
|
for (i = 0; i < dart_tablesize/4; i++)
|
2016-07-04 23:03:47 -06:00
|
|
|
dart_tablebase[i] = dart_emptyval;
|
|
|
|
|
|
|
|
/* Push to memory */
|
|
|
|
dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Initialize DART with table base and enable it. */
|
2016-07-04 23:03:47 -06:00
|
|
|
base = ((unsigned long)dart_tablebase) >> DART_PAGE_SHIFT;
|
2005-12-13 19:10:10 -07:00
|
|
|
size = dart_tablesize >> DART_PAGE_SHIFT;
|
|
|
|
if (dart_is_u4) {
|
2005-12-18 22:49:07 -07:00
|
|
|
size &= DART_SIZE_U4_SIZE_MASK;
|
2005-12-13 19:10:10 -07:00
|
|
|
DART_OUT(DART_BASE_U4, base);
|
|
|
|
DART_OUT(DART_SIZE_U4, size);
|
|
|
|
DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE);
|
|
|
|
} else {
|
2005-12-18 22:49:07 -07:00
|
|
|
size &= DART_CNTL_U3_SIZE_MASK;
|
2005-12-13 19:10:10 -07:00
|
|
|
DART_OUT(DART_CNTL,
|
|
|
|
DART_CNTL_U3_ENABLE |
|
|
|
|
(base << DART_CNTL_U3_BASE_SHIFT) |
|
|
|
|
(size << DART_CNTL_U3_SIZE_SHIFT));
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Invalidate DART to get rid of possible stale TLBs */
|
|
|
|
dart_tlb_invalidate_all();
|
|
|
|
|
2005-12-13 19:10:10 -07:00
|
|
|
printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n",
|
|
|
|
dart_is_u4 ? "U4" : "U3");
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-06-05 00:35:06 -06:00
|
|
|
static struct iommu_table_ops iommu_dart_ops = {
|
|
|
|
.set = dart_build,
|
|
|
|
.clear = dart_free,
|
|
|
|
.flush = dart_flush,
|
|
|
|
};
|
|
|
|
|
2005-12-13 19:10:10 -07:00
|
|
|
static void iommu_table_dart_setup(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2005-12-13 19:10:10 -07:00
|
|
|
iommu_table_dart.it_busno = 0;
|
|
|
|
iommu_table_dart.it_offset = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
/* it_size is in number of entries */
|
2006-10-29 22:15:59 -07:00
|
|
|
iommu_table_dart.it_size = dart_tablesize / sizeof(u32);
|
2014-01-28 21:20:12 -07:00
|
|
|
iommu_table_dart.it_page_shift = IOMMU_PAGE_SHIFT_4K;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Initialize the common IOMMU code */
|
2016-07-04 23:03:47 -06:00
|
|
|
iommu_table_dart.it_base = (unsigned long)dart_tablebase;
|
2005-12-13 19:10:10 -07:00
|
|
|
iommu_table_dart.it_index = 0;
|
|
|
|
iommu_table_dart.it_blocksize = 1;
|
2015-06-05 00:35:06 -06:00
|
|
|
iommu_table_dart.it_ops = &iommu_dart_ops;
|
2006-06-10 04:58:08 -06:00
|
|
|
iommu_init_table(&iommu_table_dart, -1);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Reserve the last page of the DART to avoid possible prefetch
|
|
|
|
* past the DART mapped area
|
|
|
|
*/
|
2005-12-13 19:10:10 -07:00
|
|
|
set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2010-08-30 13:24:18 -06:00
|
|
|
static void pci_dma_dev_setup_dart(struct pci_dev *dev)
|
|
|
|
{
|
2015-06-23 23:25:27 -06:00
|
|
|
if (dart_is_u4)
|
|
|
|
set_dma_offset(&dev->dev, DART_U4_BYPASS_BASE);
|
|
|
|
set_iommu_table_base(&dev->dev, &iommu_table_dart);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-11-10 23:25:02 -07:00
|
|
|
static void pci_dma_bus_setup_dart(struct pci_bus *bus)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2005-12-13 19:10:10 -07:00
|
|
|
if (!iommu_table_dart_inited) {
|
|
|
|
iommu_table_dart_inited = 1;
|
|
|
|
iommu_table_dart_setup();
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-08-30 13:24:18 -06:00
|
|
|
static bool dart_device_on_pcie(struct device *dev)
|
|
|
|
{
|
|
|
|
struct device_node *np = of_node_get(dev->of_node);
|
|
|
|
|
|
|
|
while(np) {
|
|
|
|
if (of_device_is_compatible(np, "U4-pcie") ||
|
|
|
|
of_device_is_compatible(np, "u4-pcie")) {
|
|
|
|
of_node_put(np);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
np = of_get_next_parent(np);
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int dart_dma_set_mask(struct device *dev, u64 dma_mask)
|
|
|
|
{
|
|
|
|
if (!dev->dma_mask || !dma_supported(dev, dma_mask))
|
|
|
|
return -EIO;
|
|
|
|
|
|
|
|
/* U4 supports a DART bypass, we use it for 64-bit capable
|
|
|
|
* devices to improve performances. However, that only works
|
|
|
|
* for devices connected to U4 own PCIe interface, not bridged
|
|
|
|
* through hypertransport. We need the device to support at
|
|
|
|
* least 40 bits of addresses.
|
|
|
|
*/
|
|
|
|
if (dart_device_on_pcie(dev) && dma_mask >= DMA_BIT_MASK(40)) {
|
|
|
|
dev_info(dev, "Using 64-bit DMA iommu bypass\n");
|
|
|
|
set_dma_ops(dev, &dma_direct_ops);
|
|
|
|
} else {
|
|
|
|
dev_info(dev, "Using 32-bit DMA via iommu\n");
|
|
|
|
set_dma_ops(dev, &dma_iommu_ops);
|
|
|
|
}
|
|
|
|
|
|
|
|
*dev->dma_mask = dma_mask;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-03-30 23:00:48 -06:00
|
|
|
void __init iommu_init_early_dart(struct pci_controller_ops *controller_ops)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct device_node *dn;
|
|
|
|
|
|
|
|
/* Find the DART in the device-tree */
|
|
|
|
dn = of_find_compatible_node(NULL, "dart", "u3-dart");
|
2005-12-13 19:10:10 -07:00
|
|
|
if (dn == NULL) {
|
|
|
|
dn = of_find_compatible_node(NULL, "dart", "u4-dart");
|
|
|
|
if (dn == NULL)
|
2010-10-18 01:27:02 -06:00
|
|
|
return; /* use default direct_dma_ops */
|
2005-12-13 19:10:10 -07:00
|
|
|
dart_is_u4 = 1;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-08-30 13:24:18 -06:00
|
|
|
/* Initialize the DART HW */
|
|
|
|
if (dart_init(dn) != 0)
|
|
|
|
goto bail;
|
|
|
|
|
|
|
|
/* Setup bypass if supported */
|
|
|
|
if (dart_is_u4)
|
|
|
|
ppc_md.dma_set_mask = dart_dma_set_mask;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2015-03-30 23:00:57 -06:00
|
|
|
controller_ops->dma_dev_setup = pci_dma_dev_setup_dart;
|
|
|
|
controller_ops->dma_bus_setup = pci_dma_bus_setup_dart;
|
|
|
|
|
2010-08-30 13:24:18 -06:00
|
|
|
/* Setup pci_dma ops */
|
|
|
|
set_pci_dma_ops(&dma_iommu_ops);
|
|
|
|
return;
|
2005-12-13 19:10:10 -07:00
|
|
|
|
|
|
|
bail:
|
|
|
|
/* If init failed, use direct iommu and null setup functions */
|
2015-03-30 23:00:57 -06:00
|
|
|
controller_ops->dma_dev_setup = NULL;
|
|
|
|
controller_ops->dma_bus_setup = NULL;
|
2005-12-13 19:10:10 -07:00
|
|
|
|
|
|
|
/* Setup pci_dma ops */
|
2007-03-03 22:58:39 -07:00
|
|
|
set_pci_dma_ops(&dma_direct_ops);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2007-05-03 06:28:32 -06:00
|
|
|
#ifdef CONFIG_PM
|
|
|
|
static void iommu_dart_restore(void)
|
|
|
|
{
|
2016-07-04 23:03:47 -06:00
|
|
|
dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32));
|
2007-05-03 06:28:32 -06:00
|
|
|
dart_tlb_invalidate_all();
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init iommu_init_late_dart(void)
|
|
|
|
{
|
|
|
|
if (!dart_tablebase)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
ppc_md.iommu_restore = iommu_dart_restore;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
late_initcall(iommu_init_late_dart);
|
2016-07-04 23:03:47 -06:00
|
|
|
#endif /* CONFIG_PM */
|