dma/direct: Handle the memory encryption bit in common code
Give the basic phys_to_dma() and dma_to_phys() helpers a __-prefix and add the memory encryption mask to the non-prefixed versions. Use the __-prefixed versions directly instead of clearing the mask again in various places. Tested-by: Tom Lendacky <thomas.lendacky@amd.com> Signed-off-by: Christoph Hellwig <hch@lst.de> Reviewed-by: Thomas Gleixner <tglx@linutronix.de> Cc: David Woodhouse <dwmw2@infradead.org> Cc: Joerg Roedel <joro@8bytes.org> Cc: Jon Mason <jdmason@kudzu.us> Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Muli Ben-Yehuda <mulix@mulix.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: iommu@lists.linux-foundation.org Link: http://lkml.kernel.org/r/20180319103826.12853-13-hch@lst.de Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
e7de6c7cc2
commit
b6e05477c1
|
@ -2,13 +2,13 @@
|
||||||
#ifndef ASM_ARM_DMA_DIRECT_H
|
#ifndef ASM_ARM_DMA_DIRECT_H
|
||||||
#define ASM_ARM_DMA_DIRECT_H 1
|
#define ASM_ARM_DMA_DIRECT_H 1
|
||||||
|
|
||||||
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
|
||||||
{
|
{
|
||||||
unsigned int offset = paddr & ~PAGE_MASK;
|
unsigned int offset = paddr & ~PAGE_MASK;
|
||||||
return pfn_to_dma(dev, __phys_to_pfn(paddr)) + offset;
|
return pfn_to_dma(dev, __phys_to_pfn(paddr)) + offset;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr)
|
static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dev_addr)
|
||||||
{
|
{
|
||||||
unsigned int offset = dev_addr & ~PAGE_MASK;
|
unsigned int offset = dev_addr & ~PAGE_MASK;
|
||||||
return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset;
|
return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset;
|
||||||
|
|
|
@ -10,7 +10,7 @@
|
||||||
* IP32 changes by Ilya.
|
* IP32 changes by Ilya.
|
||||||
* Copyright (C) 2010 Cavium Networks, Inc.
|
* Copyright (C) 2010 Cavium Networks, Inc.
|
||||||
*/
|
*/
|
||||||
#include <linux/dma-mapping.h>
|
#include <linux/dma-direct.h>
|
||||||
#include <linux/scatterlist.h>
|
#include <linux/scatterlist.h>
|
||||||
#include <linux/bootmem.h>
|
#include <linux/bootmem.h>
|
||||||
#include <linux/export.h>
|
#include <linux/export.h>
|
||||||
|
@ -182,7 +182,7 @@ struct octeon_dma_map_ops {
|
||||||
phys_addr_t (*dma_to_phys)(struct device *dev, dma_addr_t daddr);
|
phys_addr_t (*dma_to_phys)(struct device *dev, dma_addr_t daddr);
|
||||||
};
|
};
|
||||||
|
|
||||||
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
|
||||||
{
|
{
|
||||||
struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
|
struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
|
||||||
struct octeon_dma_map_ops,
|
struct octeon_dma_map_ops,
|
||||||
|
@ -190,9 +190,9 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
||||||
|
|
||||||
return ops->phys_to_dma(dev, paddr);
|
return ops->phys_to_dma(dev, paddr);
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(phys_to_dma);
|
EXPORT_SYMBOL(__phys_to_dma);
|
||||||
|
|
||||||
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
|
phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
|
||||||
{
|
{
|
||||||
struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
|
struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
|
||||||
struct octeon_dma_map_ops,
|
struct octeon_dma_map_ops,
|
||||||
|
@ -200,7 +200,7 @@ phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
|
||||||
|
|
||||||
return ops->dma_to_phys(dev, daddr);
|
return ops->dma_to_phys(dev, daddr);
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(dma_to_phys);
|
EXPORT_SYMBOL(__dma_to_phys);
|
||||||
|
|
||||||
static struct octeon_dma_map_ops octeon_linear_dma_map_ops = {
|
static struct octeon_dma_map_ops octeon_linear_dma_map_ops = {
|
||||||
.dma_map_ops = {
|
.dma_map_ops = {
|
||||||
|
|
|
@ -69,8 +69,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
|
||||||
return addr + size - 1 <= *dev->dma_mask;
|
return addr + size - 1 <= *dev->dma_mask;
|
||||||
}
|
}
|
||||||
|
|
||||||
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr);
|
dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr);
|
||||||
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr);
|
phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr);
|
||||||
|
|
||||||
struct dma_map_ops;
|
struct dma_map_ops;
|
||||||
extern const struct dma_map_ops *octeon_pci_dma_map_ops;
|
extern const struct dma_map_ops *octeon_pci_dma_map_ops;
|
||||||
|
|
|
@ -25,13 +25,13 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
|
||||||
return addr + size - 1 <= *dev->dma_mask;
|
return addr + size - 1 <= *dev->dma_mask;
|
||||||
}
|
}
|
||||||
|
|
||||||
extern dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr);
|
extern dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr);
|
||||||
extern phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr);
|
extern phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr);
|
||||||
static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr,
|
static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr,
|
||||||
size_t size)
|
size_t size)
|
||||||
{
|
{
|
||||||
#ifdef CONFIG_CPU_LOONGSON3
|
#ifdef CONFIG_CPU_LOONGSON3
|
||||||
return phys_to_dma(dev, virt_to_phys(addr));
|
return __phys_to_dma(dev, virt_to_phys(addr));
|
||||||
#else
|
#else
|
||||||
return virt_to_phys(addr) | 0x80000000;
|
return virt_to_phys(addr) | 0x80000000;
|
||||||
#endif
|
#endif
|
||||||
|
@ -41,7 +41,7 @@ static inline dma_addr_t plat_map_dma_mem_page(struct device *dev,
|
||||||
struct page *page)
|
struct page *page)
|
||||||
{
|
{
|
||||||
#ifdef CONFIG_CPU_LOONGSON3
|
#ifdef CONFIG_CPU_LOONGSON3
|
||||||
return phys_to_dma(dev, page_to_phys(page));
|
return __phys_to_dma(dev, page_to_phys(page));
|
||||||
#else
|
#else
|
||||||
return page_to_phys(page) | 0x80000000;
|
return page_to_phys(page) | 0x80000000;
|
||||||
#endif
|
#endif
|
||||||
|
@ -51,7 +51,7 @@ static inline unsigned long plat_dma_addr_to_phys(struct device *dev,
|
||||||
dma_addr_t dma_addr)
|
dma_addr_t dma_addr)
|
||||||
{
|
{
|
||||||
#if defined(CONFIG_CPU_LOONGSON3) && defined(CONFIG_64BIT)
|
#if defined(CONFIG_CPU_LOONGSON3) && defined(CONFIG_64BIT)
|
||||||
return dma_to_phys(dev, dma_addr);
|
return __dma_to_phys(dev, dma_addr);
|
||||||
#elif defined(CONFIG_CPU_LOONGSON2F) && defined(CONFIG_64BIT)
|
#elif defined(CONFIG_CPU_LOONGSON2F) && defined(CONFIG_64BIT)
|
||||||
return (dma_addr > 0x8fffffff) ? dma_addr : (dma_addr & 0x0fffffff);
|
return (dma_addr > 0x8fffffff) ? dma_addr : (dma_addr & 0x0fffffff);
|
||||||
#else
|
#else
|
||||||
|
|
|
@ -63,7 +63,7 @@ static int loongson_dma_supported(struct device *dev, u64 mask)
|
||||||
return swiotlb_dma_supported(dev, mask);
|
return swiotlb_dma_supported(dev, mask);
|
||||||
}
|
}
|
||||||
|
|
||||||
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
|
||||||
{
|
{
|
||||||
long nid;
|
long nid;
|
||||||
#ifdef CONFIG_PHYS48_TO_HT40
|
#ifdef CONFIG_PHYS48_TO_HT40
|
||||||
|
@ -75,7 +75,7 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
||||||
return paddr;
|
return paddr;
|
||||||
}
|
}
|
||||||
|
|
||||||
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
|
phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
|
||||||
{
|
{
|
||||||
long nid;
|
long nid;
|
||||||
#ifdef CONFIG_PHYS48_TO_HT40
|
#ifdef CONFIG_PHYS48_TO_HT40
|
||||||
|
|
|
@ -17,12 +17,12 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
|
||||||
return addr + size - 1 <= *dev->dma_mask;
|
return addr + size - 1 <= *dev->dma_mask;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
|
||||||
{
|
{
|
||||||
return paddr + get_dma_offset(dev);
|
return paddr + get_dma_offset(dev);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
|
static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
|
||||||
{
|
{
|
||||||
return daddr - get_dma_offset(dev);
|
return daddr - get_dma_offset(dev);
|
||||||
}
|
}
|
||||||
|
|
|
@ -54,7 +54,6 @@ config X86
|
||||||
select ARCH_HAS_FORTIFY_SOURCE
|
select ARCH_HAS_FORTIFY_SOURCE
|
||||||
select ARCH_HAS_GCOV_PROFILE_ALL
|
select ARCH_HAS_GCOV_PROFILE_ALL
|
||||||
select ARCH_HAS_KCOV if X86_64
|
select ARCH_HAS_KCOV if X86_64
|
||||||
select ARCH_HAS_PHYS_TO_DMA
|
|
||||||
select ARCH_HAS_MEMBARRIER_SYNC_CORE
|
select ARCH_HAS_MEMBARRIER_SYNC_CORE
|
||||||
select ARCH_HAS_PMEM_API if X86_64
|
select ARCH_HAS_PMEM_API if X86_64
|
||||||
select ARCH_HAS_REFCOUNT
|
select ARCH_HAS_REFCOUNT
|
||||||
|
@ -692,6 +691,7 @@ config X86_SUPPORTS_MEMORY_FAILURE
|
||||||
config STA2X11
|
config STA2X11
|
||||||
bool "STA2X11 Companion Chip Support"
|
bool "STA2X11 Companion Chip Support"
|
||||||
depends on X86_32_NON_STANDARD && PCI
|
depends on X86_32_NON_STANDARD && PCI
|
||||||
|
select ARCH_HAS_PHYS_TO_DMA
|
||||||
select X86_DEV_DMA_OPS
|
select X86_DEV_DMA_OPS
|
||||||
select X86_DMA_REMAP
|
select X86_DMA_REMAP
|
||||||
select SWIOTLB
|
select SWIOTLB
|
||||||
|
|
|
@ -2,29 +2,8 @@
|
||||||
#ifndef ASM_X86_DMA_DIRECT_H
|
#ifndef ASM_X86_DMA_DIRECT_H
|
||||||
#define ASM_X86_DMA_DIRECT_H 1
|
#define ASM_X86_DMA_DIRECT_H 1
|
||||||
|
|
||||||
#include <linux/mem_encrypt.h>
|
|
||||||
|
|
||||||
#ifdef CONFIG_X86_DMA_REMAP /* Platform code defines bridge-specific code */
|
|
||||||
bool dma_capable(struct device *dev, dma_addr_t addr, size_t size);
|
bool dma_capable(struct device *dev, dma_addr_t addr, size_t size);
|
||||||
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr);
|
dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr);
|
||||||
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr);
|
phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr);
|
||||||
#else
|
|
||||||
static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
|
|
||||||
{
|
|
||||||
if (!dev->dma_mask)
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
return addr + size - 1 <= *dev->dma_mask;
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
||||||
{
|
|
||||||
return __sme_set(paddr);
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
||||||
{
|
|
||||||
return __sme_clr(daddr);
|
|
||||||
}
|
|
||||||
#endif /* CONFIG_X86_DMA_REMAP */
|
|
||||||
#endif /* ASM_X86_DMA_DIRECT_H */
|
#endif /* ASM_X86_DMA_DIRECT_H */
|
||||||
|
|
|
@ -211,7 +211,7 @@ static void *sev_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
|
||||||
* Since we will be clearing the encryption bit, check the
|
* Since we will be clearing the encryption bit, check the
|
||||||
* mask with it already cleared.
|
* mask with it already cleared.
|
||||||
*/
|
*/
|
||||||
addr = __sme_clr(phys_to_dma(dev, page_to_phys(page)));
|
addr = __phys_to_dma(dev, page_to_phys(page));
|
||||||
if ((addr + size) > dev->coherent_dma_mask) {
|
if ((addr + size) > dev->coherent_dma_mask) {
|
||||||
__free_pages(page, get_order(size));
|
__free_pages(page, get_order(size));
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -207,11 +207,11 @@ bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* phys_to_dma - Return the DMA AMBA address used for this STA2x11 device
|
* __phys_to_dma - Return the DMA AMBA address used for this STA2x11 device
|
||||||
* @dev: device for a PCI device
|
* @dev: device for a PCI device
|
||||||
* @paddr: Physical address
|
* @paddr: Physical address
|
||||||
*/
|
*/
|
||||||
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
|
||||||
{
|
{
|
||||||
if (!dev->archdata.is_sta2x11)
|
if (!dev->archdata.is_sta2x11)
|
||||||
return paddr;
|
return paddr;
|
||||||
|
@ -223,7 +223,7 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
||||||
* @dev: device for a PCI device
|
* @dev: device for a PCI device
|
||||||
* @daddr: STA2x11 AMBA DMA address
|
* @daddr: STA2x11 AMBA DMA address
|
||||||
*/
|
*/
|
||||||
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
|
phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
|
||||||
{
|
{
|
||||||
if (!dev->archdata.is_sta2x11)
|
if (!dev->archdata.is_sta2x11)
|
||||||
return daddr;
|
return daddr;
|
||||||
|
|
|
@ -3,18 +3,19 @@
|
||||||
#define _LINUX_DMA_DIRECT_H 1
|
#define _LINUX_DMA_DIRECT_H 1
|
||||||
|
|
||||||
#include <linux/dma-mapping.h>
|
#include <linux/dma-mapping.h>
|
||||||
|
#include <linux/mem_encrypt.h>
|
||||||
|
|
||||||
#ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA
|
#ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA
|
||||||
#include <asm/dma-direct.h>
|
#include <asm/dma-direct.h>
|
||||||
#else
|
#else
|
||||||
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
|
||||||
{
|
{
|
||||||
dma_addr_t dev_addr = (dma_addr_t)paddr;
|
dma_addr_t dev_addr = (dma_addr_t)paddr;
|
||||||
|
|
||||||
return dev_addr - ((dma_addr_t)dev->dma_pfn_offset << PAGE_SHIFT);
|
return dev_addr - ((dma_addr_t)dev->dma_pfn_offset << PAGE_SHIFT);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr)
|
static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dev_addr)
|
||||||
{
|
{
|
||||||
phys_addr_t paddr = (phys_addr_t)dev_addr;
|
phys_addr_t paddr = (phys_addr_t)dev_addr;
|
||||||
|
|
||||||
|
@ -30,6 +31,22 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
|
||||||
}
|
}
|
||||||
#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
|
#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
|
||||||
|
|
||||||
|
/*
|
||||||
|
* If memory encryption is supported, phys_to_dma will set the memory encryption
|
||||||
|
* bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma
|
||||||
|
* and __dma_to_phys versions should only be used on non-encrypted memory for
|
||||||
|
* special occasions like DMA coherent buffers.
|
||||||
|
*/
|
||||||
|
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
||||||
|
{
|
||||||
|
return __sme_set(__phys_to_dma(dev, paddr));
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
|
||||||
|
{
|
||||||
|
return __sme_clr(__dma_to_phys(dev, daddr));
|
||||||
|
}
|
||||||
|
|
||||||
#ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN
|
#ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN
|
||||||
void dma_mark_clean(void *addr, size_t size);
|
void dma_mark_clean(void *addr, size_t size);
|
||||||
#else
|
#else
|
||||||
|
|
|
@ -157,13 +157,6 @@ unsigned long swiotlb_size_or_default(void)
|
||||||
return size ? size : (IO_TLB_DEFAULT_SIZE);
|
return size ? size : (IO_TLB_DEFAULT_SIZE);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* For swiotlb, clear memory encryption mask from dma addresses */
|
|
||||||
static dma_addr_t swiotlb_phys_to_dma(struct device *hwdev,
|
|
||||||
phys_addr_t address)
|
|
||||||
{
|
|
||||||
return __sme_clr(phys_to_dma(hwdev, address));
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Note that this doesn't work with highmem page */
|
/* Note that this doesn't work with highmem page */
|
||||||
static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,
|
static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,
|
||||||
volatile void *address)
|
volatile void *address)
|
||||||
|
@ -622,7 +615,7 @@ map_single(struct device *hwdev, phys_addr_t phys, size_t size,
|
||||||
return SWIOTLB_MAP_ERROR;
|
return SWIOTLB_MAP_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
start_dma_addr = swiotlb_phys_to_dma(hwdev, io_tlb_start);
|
start_dma_addr = __phys_to_dma(hwdev, io_tlb_start);
|
||||||
return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size,
|
return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size,
|
||||||
dir, attrs);
|
dir, attrs);
|
||||||
}
|
}
|
||||||
|
@ -726,12 +719,12 @@ swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,
|
||||||
goto out_warn;
|
goto out_warn;
|
||||||
|
|
||||||
phys_addr = swiotlb_tbl_map_single(dev,
|
phys_addr = swiotlb_tbl_map_single(dev,
|
||||||
swiotlb_phys_to_dma(dev, io_tlb_start),
|
__phys_to_dma(dev, io_tlb_start),
|
||||||
0, size, DMA_FROM_DEVICE, 0);
|
0, size, DMA_FROM_DEVICE, 0);
|
||||||
if (phys_addr == SWIOTLB_MAP_ERROR)
|
if (phys_addr == SWIOTLB_MAP_ERROR)
|
||||||
goto out_warn;
|
goto out_warn;
|
||||||
|
|
||||||
*dma_handle = swiotlb_phys_to_dma(dev, phys_addr);
|
*dma_handle = __phys_to_dma(dev, phys_addr);
|
||||||
if (dma_coherent_ok(dev, *dma_handle, size))
|
if (dma_coherent_ok(dev, *dma_handle, size))
|
||||||
goto out_unmap;
|
goto out_unmap;
|
||||||
|
|
||||||
|
@ -867,10 +860,10 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
|
||||||
map = map_single(dev, phys, size, dir, attrs);
|
map = map_single(dev, phys, size, dir, attrs);
|
||||||
if (map == SWIOTLB_MAP_ERROR) {
|
if (map == SWIOTLB_MAP_ERROR) {
|
||||||
swiotlb_full(dev, size, dir, 1);
|
swiotlb_full(dev, size, dir, 1);
|
||||||
return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer);
|
return __phys_to_dma(dev, io_tlb_overflow_buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
dev_addr = swiotlb_phys_to_dma(dev, map);
|
dev_addr = __phys_to_dma(dev, map);
|
||||||
|
|
||||||
/* Ensure that the address returned is DMA'ble */
|
/* Ensure that the address returned is DMA'ble */
|
||||||
if (dma_capable(dev, dev_addr, size))
|
if (dma_capable(dev, dev_addr, size))
|
||||||
|
@ -879,7 +872,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
|
||||||
attrs |= DMA_ATTR_SKIP_CPU_SYNC;
|
attrs |= DMA_ATTR_SKIP_CPU_SYNC;
|
||||||
swiotlb_tbl_unmap_single(dev, map, size, dir, attrs);
|
swiotlb_tbl_unmap_single(dev, map, size, dir, attrs);
|
||||||
|
|
||||||
return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer);
|
return __phys_to_dma(dev, io_tlb_overflow_buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -1009,7 +1002,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
|
||||||
sg_dma_len(sgl) = 0;
|
sg_dma_len(sgl) = 0;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
sg->dma_address = swiotlb_phys_to_dma(hwdev, map);
|
sg->dma_address = __phys_to_dma(hwdev, map);
|
||||||
} else
|
} else
|
||||||
sg->dma_address = dev_addr;
|
sg->dma_address = dev_addr;
|
||||||
sg_dma_len(sg) = sg->length;
|
sg_dma_len(sg) = sg->length;
|
||||||
|
@ -1073,7 +1066,7 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
|
||||||
int
|
int
|
||||||
swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
|
swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
|
||||||
{
|
{
|
||||||
return (dma_addr == swiotlb_phys_to_dma(hwdev, io_tlb_overflow_buffer));
|
return (dma_addr == __phys_to_dma(hwdev, io_tlb_overflow_buffer));
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -1085,7 +1078,7 @@ swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
|
||||||
int
|
int
|
||||||
swiotlb_dma_supported(struct device *hwdev, u64 mask)
|
swiotlb_dma_supported(struct device *hwdev, u64 mask)
|
||||||
{
|
{
|
||||||
return swiotlb_phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
|
return __phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef CONFIG_DMA_DIRECT_OPS
|
#ifdef CONFIG_DMA_DIRECT_OPS
|
||||||
|
|
Loading…
Reference in a new issue