![Linus Torvalds](/assets/img/avatar_default.png)
Bart Van Assche noted that the ib DMA mapping code was significantly similar enough to the core DMA mapping code that with a few changes it was possible to remove the IB DMA mapping code entirely and switch the RDMA stack to use the core DMA mapping code. This resulted in a nice set of cleanups, but touched the entire tree. This branch will be submitted separately to Linus at the end of the merge window as per normal practice for tree wide changes like this. -----BEGIN PGP SIGNATURE----- iQIcBAABAgAGBQJYo06oAAoJELgmozMOVy/d9Z8QALedWHdu98St1L0u2c8sxnR9 2zo/4sF5Vb9u7FpmdIX32L4SQ9s9KhPE8Qp8NtZLf9v10zlDebIRJDpXknXtKooV CAXxX4sxBXV27/UrhbZEfXiPrmm6ccJFyIfRnMU6NlMqh2AtAsRa5AC2/RMp8oUD Med97PFiF0o6TD22/UH1VFbRpX1zjaKyqm7a3as5sJfzNA+UGIZAQ7Euz8000DKZ xCgVLTEwS0FmOujtBkCst7xa9TjuqR1HLOB4DdGvAhP6BHdz2yamM7Qmh9NN+NEX 0BtjsuXomtn6j6AszGC+bpipCZh3NUigcwoFAARXCYFHibBvo4DPdFeGsraFgXdy 1+KyR8CCeQG3Aly5Vwr264RFPGkGpwMj8PsBlXgQVtrlg4rriaCzOJNmIIbfdADw ftqhxBOzReZw77aH2s+9p2ILRfcAmPqhynLvFGFo9LBvsik8LVso7YgZN0xGxwcI IjI/XGC8UskPVsIZBIYA6sl2bYzgOjtBIHiXjRrPlW3uhduIXLrvKFfLPP/5XLAG ehLXK+J0bfsyY9ClmlNS8oH/WdLhXAyy/KNmnj5bRRm9qg6BRJR3bsOBhZJODuoC XgEXFfF6/7roNESWxowff7pK0rTkRg/m/Pa4VQpeO+6NWHE7kgZhL6kyIp5nKcwS 3e7mgpcwC+3XfA/6vU3F =e0Si -----END PGP SIGNATURE----- Merge tag 'for-next-dma_ops' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma Pull rdma DMA mapping updates from Doug Ledford: "Drop IB DMA mapping code and use core DMA code instead. Bart Van Assche noted that the ib DMA mapping code was significantly similar enough to the core DMA mapping code that with a few changes it was possible to remove the IB DMA mapping code entirely and switch the RDMA stack to use the core DMA mapping code. This resulted in a nice set of cleanups, but touched the entire tree and has been kept separate for that reason." * tag 'for-next-dma_ops' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma: (37 commits) IB/rxe, IB/rdmavt: Use dma_virt_ops instead of duplicating it IB/core: Remove ib_device.dma_device nvme-rdma: Switch from dma_device to dev.parent RDS: net: Switch from dma_device to dev.parent IB/srpt: Modify a debug statement IB/srp: Switch from dma_device to dev.parent IB/iser: Switch from dma_device to dev.parent IB/IPoIB: Switch from dma_device to dev.parent IB/rxe: Switch from dma_device to dev.parent IB/vmw_pvrdma: Switch from dma_device to dev.parent IB/usnic: Switch from dma_device to dev.parent IB/qib: Switch from dma_device to dev.parent IB/qedr: Switch from dma_device to dev.parent IB/ocrdma: Switch from dma_device to dev.parent IB/nes: Remove a superfluous assignment statement IB/mthca: Switch from dma_device to dev.parent IB/mlx5: Switch from dma_device to dev.parent IB/mlx4: Switch from dma_device to dev.parent IB/i40iw: Remove a superfluous assignment statement IB/hns: Switch from dma_device to dev.parent ...
358 lines
9.7 KiB
C
358 lines
9.7 KiB
C
/*
|
|
* This file is subject to the terms and conditions of the GNU General Public
|
|
* License. See the file "COPYING" in the main directory of this archive
|
|
* for more details.
|
|
*
|
|
* Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
|
|
* Copyright (C) 2000, 2001 Ralf Baechle <ralf@gnu.org>
|
|
* Copyright (C) 2005 Ilya A. Volynets-Evenbakh <ilya@total-knowledge.com>
|
|
* swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
|
|
* IP32 changes by Ilya.
|
|
* Copyright (C) 2010 Cavium Networks, Inc.
|
|
*/
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/scatterlist.h>
|
|
#include <linux/bootmem.h>
|
|
#include <linux/export.h>
|
|
#include <linux/swiotlb.h>
|
|
#include <linux/types.h>
|
|
#include <linux/init.h>
|
|
#include <linux/mm.h>
|
|
|
|
#include <asm/bootinfo.h>
|
|
|
|
#include <asm/octeon/octeon.h>
|
|
|
|
#ifdef CONFIG_PCI
|
|
#include <asm/octeon/pci-octeon.h>
|
|
#include <asm/octeon/cvmx-npi-defs.h>
|
|
#include <asm/octeon/cvmx-pci-defs.h>
|
|
|
|
static dma_addr_t octeon_hole_phys_to_dma(phys_addr_t paddr)
|
|
{
|
|
if (paddr >= CVMX_PCIE_BAR1_PHYS_BASE && paddr < (CVMX_PCIE_BAR1_PHYS_BASE + CVMX_PCIE_BAR1_PHYS_SIZE))
|
|
return paddr - CVMX_PCIE_BAR1_PHYS_BASE + CVMX_PCIE_BAR1_RC_BASE;
|
|
else
|
|
return paddr;
|
|
}
|
|
|
|
static phys_addr_t octeon_hole_dma_to_phys(dma_addr_t daddr)
|
|
{
|
|
if (daddr >= CVMX_PCIE_BAR1_RC_BASE)
|
|
return daddr + CVMX_PCIE_BAR1_PHYS_BASE - CVMX_PCIE_BAR1_RC_BASE;
|
|
else
|
|
return daddr;
|
|
}
|
|
|
|
static dma_addr_t octeon_gen1_phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
if (paddr >= 0x410000000ull && paddr < 0x420000000ull)
|
|
paddr -= 0x400000000ull;
|
|
return octeon_hole_phys_to_dma(paddr);
|
|
}
|
|
|
|
static phys_addr_t octeon_gen1_dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
{
|
|
daddr = octeon_hole_dma_to_phys(daddr);
|
|
|
|
if (daddr >= 0x10000000ull && daddr < 0x20000000ull)
|
|
daddr += 0x400000000ull;
|
|
|
|
return daddr;
|
|
}
|
|
|
|
static dma_addr_t octeon_gen2_phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
return octeon_hole_phys_to_dma(paddr);
|
|
}
|
|
|
|
static phys_addr_t octeon_gen2_dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
{
|
|
return octeon_hole_dma_to_phys(daddr);
|
|
}
|
|
|
|
static dma_addr_t octeon_big_phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
if (paddr >= 0x410000000ull && paddr < 0x420000000ull)
|
|
paddr -= 0x400000000ull;
|
|
|
|
/* Anything in the BAR1 hole or above goes via BAR2 */
|
|
if (paddr >= 0xf0000000ull)
|
|
paddr = OCTEON_BAR2_PCI_ADDRESS + paddr;
|
|
|
|
return paddr;
|
|
}
|
|
|
|
static phys_addr_t octeon_big_dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
{
|
|
if (daddr >= OCTEON_BAR2_PCI_ADDRESS)
|
|
daddr -= OCTEON_BAR2_PCI_ADDRESS;
|
|
|
|
if (daddr >= 0x10000000ull && daddr < 0x20000000ull)
|
|
daddr += 0x400000000ull;
|
|
return daddr;
|
|
}
|
|
|
|
static dma_addr_t octeon_small_phys_to_dma(struct device *dev,
|
|
phys_addr_t paddr)
|
|
{
|
|
if (paddr >= 0x410000000ull && paddr < 0x420000000ull)
|
|
paddr -= 0x400000000ull;
|
|
|
|
/* Anything not in the BAR1 range goes via BAR2 */
|
|
if (paddr >= octeon_bar1_pci_phys && paddr < octeon_bar1_pci_phys + 0x8000000ull)
|
|
paddr = paddr - octeon_bar1_pci_phys;
|
|
else
|
|
paddr = OCTEON_BAR2_PCI_ADDRESS + paddr;
|
|
|
|
return paddr;
|
|
}
|
|
|
|
static phys_addr_t octeon_small_dma_to_phys(struct device *dev,
|
|
dma_addr_t daddr)
|
|
{
|
|
if (daddr >= OCTEON_BAR2_PCI_ADDRESS)
|
|
daddr -= OCTEON_BAR2_PCI_ADDRESS;
|
|
else
|
|
daddr += octeon_bar1_pci_phys;
|
|
|
|
if (daddr >= 0x10000000ull && daddr < 0x20000000ull)
|
|
daddr += 0x400000000ull;
|
|
return daddr;
|
|
}
|
|
|
|
#endif /* CONFIG_PCI */
|
|
|
|
static dma_addr_t octeon_dma_map_page(struct device *dev, struct page *page,
|
|
unsigned long offset, size_t size, enum dma_data_direction direction,
|
|
unsigned long attrs)
|
|
{
|
|
dma_addr_t daddr = swiotlb_map_page(dev, page, offset, size,
|
|
direction, attrs);
|
|
mb();
|
|
|
|
return daddr;
|
|
}
|
|
|
|
static int octeon_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
|
int nents, enum dma_data_direction direction, unsigned long attrs)
|
|
{
|
|
int r = swiotlb_map_sg_attrs(dev, sg, nents, direction, attrs);
|
|
mb();
|
|
return r;
|
|
}
|
|
|
|
static void octeon_dma_sync_single_for_device(struct device *dev,
|
|
dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
|
|
{
|
|
swiotlb_sync_single_for_device(dev, dma_handle, size, direction);
|
|
mb();
|
|
}
|
|
|
|
static void octeon_dma_sync_sg_for_device(struct device *dev,
|
|
struct scatterlist *sg, int nelems, enum dma_data_direction direction)
|
|
{
|
|
swiotlb_sync_sg_for_device(dev, sg, nelems, direction);
|
|
mb();
|
|
}
|
|
|
|
static void *octeon_dma_alloc_coherent(struct device *dev, size_t size,
|
|
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
|
|
{
|
|
void *ret;
|
|
|
|
/* ignore region specifiers */
|
|
gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
|
|
|
|
if (IS_ENABLED(CONFIG_ZONE_DMA) && dev == NULL)
|
|
gfp |= __GFP_DMA;
|
|
else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
|
|
dev->coherent_dma_mask <= DMA_BIT_MASK(24))
|
|
gfp |= __GFP_DMA;
|
|
else if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
|
|
dev->coherent_dma_mask <= DMA_BIT_MASK(32))
|
|
gfp |= __GFP_DMA32;
|
|
|
|
/* Don't invoke OOM killer */
|
|
gfp |= __GFP_NORETRY;
|
|
|
|
ret = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
|
|
|
|
mb();
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void octeon_dma_free_coherent(struct device *dev, size_t size,
|
|
void *vaddr, dma_addr_t dma_handle, unsigned long attrs)
|
|
{
|
|
swiotlb_free_coherent(dev, size, vaddr, dma_handle);
|
|
}
|
|
|
|
static dma_addr_t octeon_unity_phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
return paddr;
|
|
}
|
|
|
|
static phys_addr_t octeon_unity_dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
{
|
|
return daddr;
|
|
}
|
|
|
|
struct octeon_dma_map_ops {
|
|
const struct dma_map_ops dma_map_ops;
|
|
dma_addr_t (*phys_to_dma)(struct device *dev, phys_addr_t paddr);
|
|
phys_addr_t (*dma_to_phys)(struct device *dev, dma_addr_t daddr);
|
|
};
|
|
|
|
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
|
|
struct octeon_dma_map_ops,
|
|
dma_map_ops);
|
|
|
|
return ops->phys_to_dma(dev, paddr);
|
|
}
|
|
EXPORT_SYMBOL(phys_to_dma);
|
|
|
|
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
{
|
|
struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
|
|
struct octeon_dma_map_ops,
|
|
dma_map_ops);
|
|
|
|
return ops->dma_to_phys(dev, daddr);
|
|
}
|
|
EXPORT_SYMBOL(dma_to_phys);
|
|
|
|
static struct octeon_dma_map_ops octeon_linear_dma_map_ops = {
|
|
.dma_map_ops = {
|
|
.alloc = octeon_dma_alloc_coherent,
|
|
.free = octeon_dma_free_coherent,
|
|
.map_page = octeon_dma_map_page,
|
|
.unmap_page = swiotlb_unmap_page,
|
|
.map_sg = octeon_dma_map_sg,
|
|
.unmap_sg = swiotlb_unmap_sg_attrs,
|
|
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
|
|
.sync_single_for_device = octeon_dma_sync_single_for_device,
|
|
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
|
|
.sync_sg_for_device = octeon_dma_sync_sg_for_device,
|
|
.mapping_error = swiotlb_dma_mapping_error,
|
|
.dma_supported = swiotlb_dma_supported
|
|
},
|
|
.phys_to_dma = octeon_unity_phys_to_dma,
|
|
.dma_to_phys = octeon_unity_dma_to_phys
|
|
};
|
|
|
|
char *octeon_swiotlb;
|
|
|
|
void __init plat_swiotlb_setup(void)
|
|
{
|
|
int i;
|
|
phys_addr_t max_addr;
|
|
phys_addr_t addr_size;
|
|
size_t swiotlbsize;
|
|
unsigned long swiotlb_nslabs;
|
|
|
|
max_addr = 0;
|
|
addr_size = 0;
|
|
|
|
for (i = 0 ; i < boot_mem_map.nr_map; i++) {
|
|
struct boot_mem_map_entry *e = &boot_mem_map.map[i];
|
|
if (e->type != BOOT_MEM_RAM && e->type != BOOT_MEM_INIT_RAM)
|
|
continue;
|
|
|
|
/* These addresses map low for PCI. */
|
|
if (e->addr > 0x410000000ull && !OCTEON_IS_OCTEON2())
|
|
continue;
|
|
|
|
addr_size += e->size;
|
|
|
|
if (max_addr < e->addr + e->size)
|
|
max_addr = e->addr + e->size;
|
|
|
|
}
|
|
|
|
swiotlbsize = PAGE_SIZE;
|
|
|
|
#ifdef CONFIG_PCI
|
|
/*
|
|
* For OCTEON_DMA_BAR_TYPE_SMALL, size the iotlb at 1/4 memory
|
|
* size to a maximum of 64MB
|
|
*/
|
|
if (OCTEON_IS_MODEL(OCTEON_CN31XX)
|
|
|| OCTEON_IS_MODEL(OCTEON_CN38XX_PASS2)) {
|
|
swiotlbsize = addr_size / 4;
|
|
if (swiotlbsize > 64 * (1<<20))
|
|
swiotlbsize = 64 * (1<<20);
|
|
} else if (max_addr > 0xf0000000ul) {
|
|
/*
|
|
* Otherwise only allocate a big iotlb if there is
|
|
* memory past the BAR1 hole.
|
|
*/
|
|
swiotlbsize = 64 * (1<<20);
|
|
}
|
|
#endif
|
|
#ifdef CONFIG_USB_OHCI_HCD_PLATFORM
|
|
/* OCTEON II ohci is only 32-bit. */
|
|
if (OCTEON_IS_OCTEON2() && max_addr >= 0x100000000ul)
|
|
swiotlbsize = 64 * (1<<20);
|
|
#endif
|
|
swiotlb_nslabs = swiotlbsize >> IO_TLB_SHIFT;
|
|
swiotlb_nslabs = ALIGN(swiotlb_nslabs, IO_TLB_SEGSIZE);
|
|
swiotlbsize = swiotlb_nslabs << IO_TLB_SHIFT;
|
|
|
|
octeon_swiotlb = alloc_bootmem_low_pages(swiotlbsize);
|
|
|
|
if (swiotlb_init_with_tbl(octeon_swiotlb, swiotlb_nslabs, 1) == -ENOMEM)
|
|
panic("Cannot allocate SWIOTLB buffer");
|
|
|
|
mips_dma_map_ops = &octeon_linear_dma_map_ops.dma_map_ops;
|
|
}
|
|
|
|
#ifdef CONFIG_PCI
|
|
static struct octeon_dma_map_ops _octeon_pci_dma_map_ops = {
|
|
.dma_map_ops = {
|
|
.alloc = octeon_dma_alloc_coherent,
|
|
.free = octeon_dma_free_coherent,
|
|
.map_page = octeon_dma_map_page,
|
|
.unmap_page = swiotlb_unmap_page,
|
|
.map_sg = octeon_dma_map_sg,
|
|
.unmap_sg = swiotlb_unmap_sg_attrs,
|
|
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
|
|
.sync_single_for_device = octeon_dma_sync_single_for_device,
|
|
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
|
|
.sync_sg_for_device = octeon_dma_sync_sg_for_device,
|
|
.mapping_error = swiotlb_dma_mapping_error,
|
|
.dma_supported = swiotlb_dma_supported
|
|
},
|
|
};
|
|
|
|
const struct dma_map_ops *octeon_pci_dma_map_ops;
|
|
|
|
void __init octeon_pci_dma_init(void)
|
|
{
|
|
switch (octeon_dma_bar_type) {
|
|
case OCTEON_DMA_BAR_TYPE_PCIE2:
|
|
_octeon_pci_dma_map_ops.phys_to_dma = octeon_gen2_phys_to_dma;
|
|
_octeon_pci_dma_map_ops.dma_to_phys = octeon_gen2_dma_to_phys;
|
|
break;
|
|
case OCTEON_DMA_BAR_TYPE_PCIE:
|
|
_octeon_pci_dma_map_ops.phys_to_dma = octeon_gen1_phys_to_dma;
|
|
_octeon_pci_dma_map_ops.dma_to_phys = octeon_gen1_dma_to_phys;
|
|
break;
|
|
case OCTEON_DMA_BAR_TYPE_BIG:
|
|
_octeon_pci_dma_map_ops.phys_to_dma = octeon_big_phys_to_dma;
|
|
_octeon_pci_dma_map_ops.dma_to_phys = octeon_big_dma_to_phys;
|
|
break;
|
|
case OCTEON_DMA_BAR_TYPE_SMALL:
|
|
_octeon_pci_dma_map_ops.phys_to_dma = octeon_small_phys_to_dma;
|
|
_octeon_pci_dma_map_ops.dma_to_phys = octeon_small_dma_to_phys;
|
|
break;
|
|
default:
|
|
BUG();
|
|
}
|
|
octeon_pci_dma_map_ops = &_octeon_pci_dma_map_ops.dma_map_ops;
|
|
}
|
|
#endif /* CONFIG_PCI */
|