iommu/exynos: Add support for v5 SYSMMU
This patch adds support for v5 of SYSMMU controller, found in Samsung Exynos 5433 SoCs. The main difference of v5 is support for 36-bit physical address space and some changes in register layout and core clocks hanging. This patch also adds support for ARM64 architecture, which is used by Exynos 5433 SoCs. Signed-off-by: Marek Szyprowski <m.szyprowski@samsung.com> Signed-off-by: Joerg Roedel <jroedel@suse.de>hifive-unleashed-5.1
Marek Szyprowski
Joerg Roedel
parent
e6802707fb
commit
740a01eee9
|
|
@ -35,9 +35,10 @@ Required properties:
|
|||
- interrupts: An interrupt specifier for interrupt signal of System MMU,
|
||||
according to the format defined by a particular interrupt
|
||||
controller.
|
||||
- clock-names: Should be "sysmmu" if the System MMU is needed to gate its clock.
|
||||
- clock-names: Should be "sysmmu" or a pair of "aclk" and "pclk" to gate
|
||||
SYSMMU core clocks.
|
||||
Optional "master" if the clock to the System MMU is gated by
|
||||
another gate clock other than "sysmmu" (usually main gate clock
|
||||
another gate clock other core (usually main gate clock
|
||||
of peripheral device this SYSMMU belongs to).
|
||||
- clocks: Phandles for respective clocks described by clock-names.
|
||||
- power-domains: Required if the System MMU is needed to gate its power.
|
||||
|
|
|
|||
|
|
@ -243,7 +243,7 @@ config TEGRA_IOMMU_SMMU
|
|||
|
||||
config EXYNOS_IOMMU
|
||||
bool "Exynos IOMMU Support"
|
||||
depends on ARCH_EXYNOS && ARM && MMU
|
||||
depends on ARCH_EXYNOS && MMU
|
||||
select IOMMU_API
|
||||
select ARM_DMA_USE_IOMMU
|
||||
help
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
/* linux/drivers/iommu/exynos_iommu.c
|
||||
*
|
||||
* Copyright (c) 2011 Samsung Electronics Co., Ltd.
|
||||
/*
|
||||
* Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
|
||||
* http://www.samsung.com
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
|
|
@ -55,17 +54,25 @@ typedef u32 sysmmu_pte_t;
|
|||
#define lv2ent_small(pent) ((*(pent) & 2) == 2)
|
||||
#define lv2ent_large(pent) ((*(pent) & 3) == 1)
|
||||
|
||||
static u32 sysmmu_page_offset(sysmmu_iova_t iova, u32 size)
|
||||
{
|
||||
return iova & (size - 1);
|
||||
}
|
||||
/*
|
||||
* v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces
|
||||
* v5.0 introduced support for 36bit physical address space by shifting
|
||||
* all page entry values by 4 bits.
|
||||
* All SYSMMU controllers in the system support the address spaces of the same
|
||||
* size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper
|
||||
* value (0 or 4).
|
||||
*/
|
||||
static short PG_ENT_SHIFT = -1;
|
||||
#define SYSMMU_PG_ENT_SHIFT 0
|
||||
#define SYSMMU_V5_PG_ENT_SHIFT 4
|
||||
|
||||
#define section_phys(sent) (*(sent) & SECT_MASK)
|
||||
#define section_offs(iova) sysmmu_page_offset((iova), SECT_SIZE)
|
||||
#define lpage_phys(pent) (*(pent) & LPAGE_MASK)
|
||||
#define lpage_offs(iova) sysmmu_page_offset((iova), LPAGE_SIZE)
|
||||
#define spage_phys(pent) (*(pent) & SPAGE_MASK)
|
||||
#define spage_offs(iova) sysmmu_page_offset((iova), SPAGE_SIZE)
|
||||
#define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT)
|
||||
#define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK)
|
||||
#define section_offs(iova) (iova & (SECT_SIZE - 1))
|
||||
#define lpage_phys(pent) (sect_to_phys(*(pent)) & LPAGE_MASK)
|
||||
#define lpage_offs(iova) (iova & (LPAGE_SIZE - 1))
|
||||
#define spage_phys(pent) (sect_to_phys(*(pent)) & SPAGE_MASK)
|
||||
#define spage_offs(iova) (iova & (SPAGE_SIZE - 1))
|
||||
|
||||
#define NUM_LV1ENTRIES 4096
|
||||
#define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE)
|
||||
|
|
@ -84,13 +91,12 @@ static u32 lv2ent_offset(sysmmu_iova_t iova)
|
|||
#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t))
|
||||
|
||||
#define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
|
||||
#define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0))
|
||||
|
||||
#define lv2table_base(sent) (*(sent) & 0xFFFFFC00)
|
||||
|
||||
#define mk_lv1ent_sect(pa) ((pa) | 2)
|
||||
#define mk_lv1ent_page(pa) ((pa) | 1)
|
||||
#define mk_lv2ent_lpage(pa) ((pa) | 1)
|
||||
#define mk_lv2ent_spage(pa) ((pa) | 2)
|
||||
#define mk_lv1ent_sect(pa) ((pa >> PG_ENT_SHIFT) | 2)
|
||||
#define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1)
|
||||
#define mk_lv2ent_lpage(pa) ((pa >> PG_ENT_SHIFT) | 1)
|
||||
#define mk_lv2ent_spage(pa) ((pa >> PG_ENT_SHIFT) | 2)
|
||||
|
||||
#define CTRL_ENABLE 0x5
|
||||
#define CTRL_BLOCK 0x7
|
||||
|
|
@ -98,14 +104,23 @@ static u32 lv2ent_offset(sysmmu_iova_t iova)
|
|||
|
||||
#define CFG_LRU 0x1
|
||||
#define CFG_QOS(n) ((n & 0xF) << 7)
|
||||
#define CFG_MASK 0x0150FFFF /* Selecting bit 0-15, 20, 22 and 24 */
|
||||
#define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */
|
||||
#define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */
|
||||
#define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */
|
||||
|
||||
/* common registers */
|
||||
#define REG_MMU_CTRL 0x000
|
||||
#define REG_MMU_CFG 0x004
|
||||
#define REG_MMU_STATUS 0x008
|
||||
#define REG_MMU_VERSION 0x034
|
||||
|
||||
#define MMU_MAJ_VER(val) ((val) >> 7)
|
||||
#define MMU_MIN_VER(val) ((val) & 0x7F)
|
||||
#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */
|
||||
|
||||
#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F))
|
||||
|
||||
/* v1.x - v3.x registers */
|
||||
#define REG_MMU_FLUSH 0x00C
|
||||
#define REG_MMU_FLUSH_ENTRY 0x010
|
||||
#define REG_PT_BASE_ADDR 0x014
|
||||
|
|
@ -117,18 +132,14 @@ static u32 lv2ent_offset(sysmmu_iova_t iova)
|
|||
#define REG_AR_FAULT_ADDR 0x02C
|
||||
#define REG_DEFAULT_SLAVE_ADDR 0x030
|
||||
|
||||
#define REG_MMU_VERSION 0x034
|
||||
|
||||
#define MMU_MAJ_VER(val) ((val) >> 7)
|
||||
#define MMU_MIN_VER(val) ((val) & 0x7F)
|
||||
#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */
|
||||
|
||||
#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F))
|
||||
|
||||
#define REG_PB0_SADDR 0x04C
|
||||
#define REG_PB0_EADDR 0x050
|
||||
#define REG_PB1_SADDR 0x054
|
||||
#define REG_PB1_EADDR 0x058
|
||||
/* v5.x registers */
|
||||
#define REG_V5_PT_BASE_PFN 0x00C
|
||||
#define REG_V5_MMU_FLUSH_ALL 0x010
|
||||
#define REG_V5_MMU_FLUSH_ENTRY 0x014
|
||||
#define REG_V5_INT_STATUS 0x060
|
||||
#define REG_V5_INT_CLEAR 0x064
|
||||
#define REG_V5_FAULT_AR_VA 0x070
|
||||
#define REG_V5_FAULT_AW_VA 0x080
|
||||
|
||||
#define has_sysmmu(dev) (dev->archdata.iommu != NULL)
|
||||
|
||||
|
|
@ -169,6 +180,19 @@ static const struct sysmmu_fault_info sysmmu_faults[] = {
|
|||
{ 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE },
|
||||
};
|
||||
|
||||
static const struct sysmmu_fault_info sysmmu_v5_faults[] = {
|
||||
{ 0, REG_V5_FAULT_AR_VA, "AR PTW", IOMMU_FAULT_READ },
|
||||
{ 1, REG_V5_FAULT_AR_VA, "AR PAGE", IOMMU_FAULT_READ },
|
||||
{ 2, REG_V5_FAULT_AR_VA, "AR MULTI-HIT", IOMMU_FAULT_READ },
|
||||
{ 3, REG_V5_FAULT_AR_VA, "AR ACCESS PROTECTION", IOMMU_FAULT_READ },
|
||||
{ 4, REG_V5_FAULT_AR_VA, "AR SECURITY PROTECTION", IOMMU_FAULT_READ },
|
||||
{ 16, REG_V5_FAULT_AW_VA, "AW PTW", IOMMU_FAULT_WRITE },
|
||||
{ 17, REG_V5_FAULT_AW_VA, "AW PAGE", IOMMU_FAULT_WRITE },
|
||||
{ 18, REG_V5_FAULT_AW_VA, "AW MULTI-HIT", IOMMU_FAULT_WRITE },
|
||||
{ 19, REG_V5_FAULT_AW_VA, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE },
|
||||
{ 20, REG_V5_FAULT_AW_VA, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE },
|
||||
};
|
||||
|
||||
/*
|
||||
* This structure is attached to dev.archdata.iommu of the master device
|
||||
* on device add, contains a list of SYSMMU controllers defined by device tree,
|
||||
|
|
@ -205,6 +229,8 @@ struct sysmmu_drvdata {
|
|||
struct device *master; /* master device (owner) */
|
||||
void __iomem *sfrbase; /* our registers */
|
||||
struct clk *clk; /* SYSMMU's clock */
|
||||
struct clk *aclk; /* SYSMMU's aclk clock */
|
||||
struct clk *pclk; /* SYSMMU's pclk clock */
|
||||
struct clk *clk_master; /* master's device clock */
|
||||
int activations; /* number of calls to sysmmu_enable */
|
||||
spinlock_t lock; /* lock for modyfying state */
|
||||
|
|
@ -262,7 +288,10 @@ static bool sysmmu_block(struct sysmmu_drvdata *data)
|
|||
|
||||
static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data)
|
||||
{
|
||||
__raw_writel(0x1, data->sfrbase + REG_MMU_FLUSH);
|
||||
if (MMU_MAJ_VER(data->version) < 5)
|
||||
__raw_writel(0x1, data->sfrbase + REG_MMU_FLUSH);
|
||||
else
|
||||
__raw_writel(0x1, data->sfrbase + REG_V5_MMU_FLUSH_ALL);
|
||||
}
|
||||
|
||||
static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
|
||||
|
|
@ -271,15 +300,23 @@ static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
|
|||
unsigned int i;
|
||||
|
||||
for (i = 0; i < num_inv; i++) {
|
||||
__raw_writel((iova & SPAGE_MASK) | 1,
|
||||
data->sfrbase + REG_MMU_FLUSH_ENTRY);
|
||||
if (MMU_MAJ_VER(data->version) < 5)
|
||||
__raw_writel((iova & SPAGE_MASK) | 1,
|
||||
data->sfrbase + REG_MMU_FLUSH_ENTRY);
|
||||
else
|
||||
__raw_writel((iova & SPAGE_MASK) | 1,
|
||||
data->sfrbase + REG_V5_MMU_FLUSH_ENTRY);
|
||||
iova += SPAGE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd)
|
||||
{
|
||||
__raw_writel(pgd, data->sfrbase + REG_PT_BASE_ADDR);
|
||||
if (MMU_MAJ_VER(data->version) < 5)
|
||||
__raw_writel(pgd, data->sfrbase + REG_PT_BASE_ADDR);
|
||||
else
|
||||
__raw_writel(pgd >> PAGE_SHIFT,
|
||||
data->sfrbase + REG_V5_PT_BASE_PFN);
|
||||
|
||||
__sysmmu_tlb_invalidate(data);
|
||||
}
|
||||
|
|
@ -290,6 +327,8 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data)
|
|||
|
||||
clk_enable(data->clk_master);
|
||||
clk_enable(data->clk);
|
||||
clk_enable(data->pclk);
|
||||
clk_enable(data->aclk);
|
||||
|
||||
ver = __raw_readl(data->sfrbase + REG_MMU_VERSION);
|
||||
|
||||
|
|
@ -302,6 +341,8 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data)
|
|||
dev_dbg(data->sysmmu, "hardware version: %d.%d\n",
|
||||
MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version));
|
||||
|
||||
clk_disable(data->aclk);
|
||||
clk_disable(data->pclk);
|
||||
clk_disable(data->clk);
|
||||
clk_disable(data->clk_master);
|
||||
}
|
||||
|
|
@ -326,19 +367,31 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
|
|||
{
|
||||
/* SYSMMU is in blocked state when interrupt occurred. */
|
||||
struct sysmmu_drvdata *data = dev_id;
|
||||
const struct sysmmu_fault_info *finfo = sysmmu_faults;
|
||||
int i, n = ARRAY_SIZE(sysmmu_faults);
|
||||
unsigned int itype;
|
||||
const struct sysmmu_fault_info *finfo;
|
||||
unsigned int i, n, itype;
|
||||
sysmmu_iova_t fault_addr = -1;
|
||||
unsigned short reg_status, reg_clear;
|
||||
int ret = -ENOSYS;
|
||||
|
||||
WARN_ON(!is_sysmmu_active(data));
|
||||
|
||||
if (MMU_MAJ_VER(data->version) < 5) {
|
||||
reg_status = REG_INT_STATUS;
|
||||
reg_clear = REG_INT_CLEAR;
|
||||
finfo = sysmmu_faults;
|
||||
n = ARRAY_SIZE(sysmmu_faults);
|
||||
} else {
|
||||
reg_status = REG_V5_INT_STATUS;
|
||||
reg_clear = REG_V5_INT_CLEAR;
|
||||
finfo = sysmmu_v5_faults;
|
||||
n = ARRAY_SIZE(sysmmu_v5_faults);
|
||||
}
|
||||
|
||||
spin_lock(&data->lock);
|
||||
|
||||
clk_enable(data->clk_master);
|
||||
|
||||
itype = __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS));
|
||||
itype = __ffs(__raw_readl(data->sfrbase + reg_status));
|
||||
for (i = 0; i < n; i++, finfo++)
|
||||
if (finfo->bit == itype)
|
||||
break;
|
||||
|
|
@ -355,7 +408,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
|
|||
/* fault is not recovered by fault handler */
|
||||
BUG_ON(ret != 0);
|
||||
|
||||
__raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR);
|
||||
__raw_writel(1 << itype, data->sfrbase + reg_clear);
|
||||
|
||||
sysmmu_unblock(data);
|
||||
|
||||
|
|
@ -373,6 +426,8 @@ static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data)
|
|||
__raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL);
|
||||
__raw_writel(0, data->sfrbase + REG_MMU_CFG);
|
||||
|
||||
clk_disable(data->aclk);
|
||||
clk_disable(data->pclk);
|
||||
clk_disable(data->clk);
|
||||
clk_disable(data->clk_master);
|
||||
}
|
||||
|
|
@ -421,6 +476,8 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
|
|||
{
|
||||
clk_enable(data->clk_master);
|
||||
clk_enable(data->clk);
|
||||
clk_enable(data->pclk);
|
||||
clk_enable(data->aclk);
|
||||
|
||||
__raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
|
||||
|
||||
|
|
@ -544,22 +601,47 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
|
|||
}
|
||||
|
||||
data->clk = devm_clk_get(dev, "sysmmu");
|
||||
if (IS_ERR(data->clk)) {
|
||||
dev_err(dev, "Failed to get clock!\n");
|
||||
return PTR_ERR(data->clk);
|
||||
} else {
|
||||
if (!IS_ERR(data->clk)) {
|
||||
ret = clk_prepare(data->clk);
|
||||
if (ret) {
|
||||
dev_err(dev, "Failed to prepare clk\n");
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
data->clk = NULL;
|
||||
}
|
||||
|
||||
data->aclk = devm_clk_get(dev, "aclk");
|
||||
if (!IS_ERR(data->aclk)) {
|
||||
ret = clk_prepare(data->aclk);
|
||||
if (ret) {
|
||||
dev_err(dev, "Failed to prepare aclk\n");
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
data->aclk = NULL;
|
||||
}
|
||||
|
||||
data->pclk = devm_clk_get(dev, "pclk");
|
||||
if (!IS_ERR(data->pclk)) {
|
||||
ret = clk_prepare(data->pclk);
|
||||
if (ret) {
|
||||
dev_err(dev, "Failed to prepare pclk\n");
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
data->pclk = NULL;
|
||||
}
|
||||
|
||||
if (!data->clk && (!data->aclk || !data->pclk)) {
|
||||
dev_err(dev, "Failed to get device clock(s)!\n");
|
||||
return -ENOSYS;
|
||||
}
|
||||
|
||||
data->clk_master = devm_clk_get(dev, "master");
|
||||
if (!IS_ERR(data->clk_master)) {
|
||||
ret = clk_prepare(data->clk_master);
|
||||
if (ret) {
|
||||
clk_unprepare(data->clk);
|
||||
dev_err(dev, "Failed to prepare master's clk\n");
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -573,6 +655,13 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
|
|||
platform_set_drvdata(pdev, data);
|
||||
|
||||
__sysmmu_get_version(data);
|
||||
if (PG_ENT_SHIFT < 0) {
|
||||
if (MMU_MAJ_VER(data->version) < 5)
|
||||
PG_ENT_SHIFT = SYSMMU_PG_ENT_SHIFT;
|
||||
else
|
||||
PG_ENT_SHIFT = SYSMMU_V5_PG_ENT_SHIFT;
|
||||
}
|
||||
|
||||
pm_runtime_enable(dev);
|
||||
|
||||
return 0;
|
||||
|
|
@ -637,6 +726,8 @@ static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type)
|
|||
dma_addr_t handle;
|
||||
int i;
|
||||
|
||||
/* Check if correct PTE offsets are initialized */
|
||||
BUG_ON(PG_ENT_SHIFT < 0 || !dma_dev);
|
||||
|
||||
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
|
||||
if (!domain)
|
||||
|
|
@ -816,7 +907,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain,
|
|||
bool need_flush_flpd_cache = lv1ent_zero(sent);
|
||||
|
||||
pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC);
|
||||
BUG_ON((unsigned int)pent & (LV2TABLE_SIZE - 1));
|
||||
BUG_ON((phys_addr_t)pent & (LV2TABLE_SIZE - 1));
|
||||
if (!pent)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue