![Linus Torvalds](/assets/img/avatar_default.png)
Notable changes: - Removal of the NPU DMA code, used by the out-of-tree Nvidia driver, as well as some other functions only used by drivers that haven't (yet?) made it upstream. - A fix for a bug in our handling of hardware watchpoints (eg. perf record -e mem: ...) which could lead to register corruption and kernel crashes. - Enable HAVE_ARCH_HUGE_VMAP, which allows us to use large pages for vmalloc when using the Radix MMU. - A large but incremental rewrite of our exception handling code to use gas macros rather than multiple levels of nested CPP macros. And the usual small fixes, cleanups and improvements. Thanks to: Alastair D'Silva, Alexey Kardashevskiy, Andreas Schwab, Aneesh Kumar K.V, Anju T Sudhakar, Anton Blanchard, Arnd Bergmann, Athira Rajeev, Cédric Le Goater, Christian Lamparter, Christophe Leroy, Christophe Lombard, Christoph Hellwig, Daniel Axtens, Denis Efremov, Enrico Weigelt, Frederic Barrat, Gautham R. Shenoy, Geert Uytterhoeven, Geliang Tang, Gen Zhang, Greg Kroah-Hartman, Greg Kurz, Gustavo Romero, Krzysztof Kozlowski, Madhavan Srinivasan, Masahiro Yamada, Mathieu Malaterre, Michael Neuling, Nathan Lynch, Naveen N. Rao, Nicholas Piggin, Nishad Kamdar, Oliver O'Halloran, Qian Cai, Ravi Bangoria, Sachin Sant, Sam Bobroff, Satheesh Rajendran, Segher Boessenkool, Shaokun Zhang, Shawn Anastasio, Stewart Smith, Suraj Jitindar Singh, Thiago Jung Bauermann, YueHaibing. -----BEGIN PGP SIGNATURE----- iQIcBAABAgAGBQJdKVoLAAoJEFHr6jzI4aWA0kIP/A6shIbbE7H5W2hFrqt/PPPK 3+VrvPKbOFF+W6hcE/RgSZmEnUo0svdNjHUd/eMfFS1vb/uRt2QDdrsHUNNwURQL M2mcLXFwYpnjSjb/XMgDbHpAQxjeGfTdYLonUIejN7Rk8KQUeLyKQ3SBn6kfMc46 DnUUcPcjuRGaETUmVuZZ4e40ZWbJp8PKDrSJOuUrTPXMaK5ciNbZk5mCWXGbYl6G BMQAyv4ld/417rNTjBEP/T2foMJtioAt4W6mtlgdkOTdIEZnFU67nNxDBthNSu2c 95+I+/sML4KOp1R4yhqLSLIDDbc3bg3c99hLGij0d948z3bkSZ8bwnPaUuy70C4v U8rvl/+N6C6H3DgSsPE/Gnkd8DnudqWY8nULc+8p3fXljGwww6/Qgt+6yCUn8BdW WgixkSjKgjDmzTw8trIUNEqORrTVle7cM2hIyIK2Q5T4kWzNQxrLZ/x/3wgoYjUa 1KwIzaRo5JKZ9D3pJnJ5U+knE2/90rJIyfcp0W6ygyJsWKi2GNmq1eN3sKOw0IxH Tg86RENIA/rEMErNOfP45sLteMuTR7of7peCG3yumIOZqsDVYAzerpvtSgip2cvK aG+9HcYlBFOOOF9Dabi8GXsTBLXLfwiyjjLSpA9eXPwW8KObgiNfTZa7ujjTPvis 4mk9oukFTFUpfhsMmI3T =3dBZ -----END PGP SIGNATURE----- Merge tag 'powerpc-5.3-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux Pull powerpc updates from Michael Ellerman: "Notable changes: - Removal of the NPU DMA code, used by the out-of-tree Nvidia driver, as well as some other functions only used by drivers that haven't (yet?) made it upstream. - A fix for a bug in our handling of hardware watchpoints (eg. perf record -e mem: ...) which could lead to register corruption and kernel crashes. - Enable HAVE_ARCH_HUGE_VMAP, which allows us to use large pages for vmalloc when using the Radix MMU. - A large but incremental rewrite of our exception handling code to use gas macros rather than multiple levels of nested CPP macros. And the usual small fixes, cleanups and improvements. Thanks to: Alastair D'Silva, Alexey Kardashevskiy, Andreas Schwab, Aneesh Kumar K.V, Anju T Sudhakar, Anton Blanchard, Arnd Bergmann, Athira Rajeev, Cédric Le Goater, Christian Lamparter, Christophe Leroy, Christophe Lombard, Christoph Hellwig, Daniel Axtens, Denis Efremov, Enrico Weigelt, Frederic Barrat, Gautham R. Shenoy, Geert Uytterhoeven, Geliang Tang, Gen Zhang, Greg Kroah-Hartman, Greg Kurz, Gustavo Romero, Krzysztof Kozlowski, Madhavan Srinivasan, Masahiro Yamada, Mathieu Malaterre, Michael Neuling, Nathan Lynch, Naveen N. Rao, Nicholas Piggin, Nishad Kamdar, Oliver O'Halloran, Qian Cai, Ravi Bangoria, Sachin Sant, Sam Bobroff, Satheesh Rajendran, Segher Boessenkool, Shaokun Zhang, Shawn Anastasio, Stewart Smith, Suraj Jitindar Singh, Thiago Jung Bauermann, YueHaibing" * tag 'powerpc-5.3-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux: (163 commits) powerpc/powernv/idle: Fix restore of SPRN_LDBAR for POWER9 stop state. powerpc/eeh: Handle hugepages in ioremap space ocxl: Update for AFU descriptor template version 1.1 powerpc/boot: pass CONFIG options in a simpler and more robust way powerpc/boot: add {get, put}_unaligned_be32 to xz_config.h powerpc/irq: Don't WARN continuously in arch_local_irq_restore() powerpc/module64: Use symbolic instructions names. powerpc/module32: Use symbolic instructions names. powerpc: Move PPC_HA() PPC_HI() and PPC_LO() to ppc-opcode.h powerpc/module64: Fix comment in R_PPC64_ENTRY handling powerpc/boot: Add lzo support for uImage powerpc/boot: Add lzma support for uImage powerpc/boot: don't force gzipped uImage powerpc/8xx: Add microcode patch to move SMC parameter RAM. powerpc/8xx: Use IO accessors in microcode programming. powerpc/8xx: replace #ifdefs by IS_ENABLED() in microcode.c powerpc/8xx: refactor programming of microcode CPM params. powerpc/8xx: refactor printing of microcode patch name. powerpc/8xx: Refactor microcode write powerpc/8xx: refactor writing of CPM microcode arrays ...
362 lines
8.7 KiB
C
362 lines
8.7 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
/*
|
|
* This file contains ioremap and related functions for 64-bit machines.
|
|
*
|
|
* Derived from arch/ppc64/mm/init.c
|
|
* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
|
|
*
|
|
* Modifications by Paul Mackerras (PowerMac) (paulus@samba.org)
|
|
* and Cort Dougan (PReP) (cort@cs.nmt.edu)
|
|
* Copyright (C) 1996 Paul Mackerras
|
|
*
|
|
* Derived from "arch/i386/mm/init.c"
|
|
* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
|
|
*
|
|
* Dave Engebretsen <engebret@us.ibm.com>
|
|
* Rework for PPC64 port.
|
|
*/
|
|
|
|
#include <linux/signal.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/string.h>
|
|
#include <linux/export.h>
|
|
#include <linux/types.h>
|
|
#include <linux/mman.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/swap.h>
|
|
#include <linux/stddef.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/hugetlb.h>
|
|
|
|
#include <asm/pgalloc.h>
|
|
#include <asm/page.h>
|
|
#include <asm/prom.h>
|
|
#include <asm/io.h>
|
|
#include <asm/mmu_context.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/mmu.h>
|
|
#include <asm/smp.h>
|
|
#include <asm/machdep.h>
|
|
#include <asm/tlb.h>
|
|
#include <asm/processor.h>
|
|
#include <asm/cputable.h>
|
|
#include <asm/sections.h>
|
|
#include <asm/firmware.h>
|
|
#include <asm/dma.h>
|
|
|
|
#include <mm/mmu_decl.h>
|
|
|
|
|
|
#ifdef CONFIG_PPC_BOOK3S_64
|
|
/*
|
|
* partition table and process table for ISA 3.0
|
|
*/
|
|
struct prtb_entry *process_tb;
|
|
struct patb_entry *partition_tb;
|
|
/*
|
|
* page table size
|
|
*/
|
|
unsigned long __pte_index_size;
|
|
EXPORT_SYMBOL(__pte_index_size);
|
|
unsigned long __pmd_index_size;
|
|
EXPORT_SYMBOL(__pmd_index_size);
|
|
unsigned long __pud_index_size;
|
|
EXPORT_SYMBOL(__pud_index_size);
|
|
unsigned long __pgd_index_size;
|
|
EXPORT_SYMBOL(__pgd_index_size);
|
|
unsigned long __pud_cache_index;
|
|
EXPORT_SYMBOL(__pud_cache_index);
|
|
unsigned long __pte_table_size;
|
|
EXPORT_SYMBOL(__pte_table_size);
|
|
unsigned long __pmd_table_size;
|
|
EXPORT_SYMBOL(__pmd_table_size);
|
|
unsigned long __pud_table_size;
|
|
EXPORT_SYMBOL(__pud_table_size);
|
|
unsigned long __pgd_table_size;
|
|
EXPORT_SYMBOL(__pgd_table_size);
|
|
unsigned long __pmd_val_bits;
|
|
EXPORT_SYMBOL(__pmd_val_bits);
|
|
unsigned long __pud_val_bits;
|
|
EXPORT_SYMBOL(__pud_val_bits);
|
|
unsigned long __pgd_val_bits;
|
|
EXPORT_SYMBOL(__pgd_val_bits);
|
|
unsigned long __kernel_virt_start;
|
|
EXPORT_SYMBOL(__kernel_virt_start);
|
|
unsigned long __vmalloc_start;
|
|
EXPORT_SYMBOL(__vmalloc_start);
|
|
unsigned long __vmalloc_end;
|
|
EXPORT_SYMBOL(__vmalloc_end);
|
|
unsigned long __kernel_io_start;
|
|
EXPORT_SYMBOL(__kernel_io_start);
|
|
unsigned long __kernel_io_end;
|
|
struct page *vmemmap;
|
|
EXPORT_SYMBOL(vmemmap);
|
|
unsigned long __pte_frag_nr;
|
|
EXPORT_SYMBOL(__pte_frag_nr);
|
|
unsigned long __pte_frag_size_shift;
|
|
EXPORT_SYMBOL(__pte_frag_size_shift);
|
|
unsigned long ioremap_bot;
|
|
#else /* !CONFIG_PPC_BOOK3S_64 */
|
|
unsigned long ioremap_bot = IOREMAP_BASE;
|
|
#endif
|
|
|
|
int __weak ioremap_range(unsigned long ea, phys_addr_t pa, unsigned long size, pgprot_t prot, int nid)
|
|
{
|
|
unsigned long i;
|
|
|
|
for (i = 0; i < size; i += PAGE_SIZE) {
|
|
int err = map_kernel_page(ea + i, pa + i, prot);
|
|
if (err) {
|
|
if (slab_is_available())
|
|
unmap_kernel_range(ea, size);
|
|
else
|
|
WARN_ON_ONCE(1); /* Should clean up */
|
|
return err;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* __ioremap_at - Low level function to establish the page tables
|
|
* for an IO mapping
|
|
*/
|
|
void __iomem *__ioremap_at(phys_addr_t pa, void *ea, unsigned long size, pgprot_t prot)
|
|
{
|
|
/* We don't support the 4K PFN hack with ioremap */
|
|
if (pgprot_val(prot) & H_PAGE_4K_PFN)
|
|
return NULL;
|
|
|
|
if ((ea + size) >= (void *)IOREMAP_END) {
|
|
pr_warn("Outside the supported range\n");
|
|
return NULL;
|
|
}
|
|
|
|
WARN_ON(pa & ~PAGE_MASK);
|
|
WARN_ON(((unsigned long)ea) & ~PAGE_MASK);
|
|
WARN_ON(size & ~PAGE_MASK);
|
|
|
|
if (ioremap_range((unsigned long)ea, pa, size, prot, NUMA_NO_NODE))
|
|
return NULL;
|
|
|
|
return (void __iomem *)ea;
|
|
}
|
|
|
|
/**
|
|
* __iounmap_from - Low level function to tear down the page tables
|
|
* for an IO mapping. This is used for mappings that
|
|
* are manipulated manually, like partial unmapping of
|
|
* PCI IOs or ISA space.
|
|
*/
|
|
void __iounmap_at(void *ea, unsigned long size)
|
|
{
|
|
WARN_ON(((unsigned long)ea) & ~PAGE_MASK);
|
|
WARN_ON(size & ~PAGE_MASK);
|
|
|
|
unmap_kernel_range((unsigned long)ea, size);
|
|
}
|
|
|
|
void __iomem * __ioremap_caller(phys_addr_t addr, unsigned long size,
|
|
pgprot_t prot, void *caller)
|
|
{
|
|
phys_addr_t paligned;
|
|
void __iomem *ret;
|
|
|
|
/*
|
|
* Choose an address to map it to.
|
|
* Once the imalloc system is running, we use it.
|
|
* Before that, we map using addresses going
|
|
* up from ioremap_bot. imalloc will use
|
|
* the addresses from ioremap_bot through
|
|
* IMALLOC_END
|
|
*
|
|
*/
|
|
paligned = addr & PAGE_MASK;
|
|
size = PAGE_ALIGN(addr + size) - paligned;
|
|
|
|
if ((size == 0) || (paligned == 0))
|
|
return NULL;
|
|
|
|
if (slab_is_available()) {
|
|
struct vm_struct *area;
|
|
|
|
area = __get_vm_area_caller(size, VM_IOREMAP,
|
|
ioremap_bot, IOREMAP_END,
|
|
caller);
|
|
if (area == NULL)
|
|
return NULL;
|
|
|
|
area->phys_addr = paligned;
|
|
ret = __ioremap_at(paligned, area->addr, size, prot);
|
|
} else {
|
|
ret = __ioremap_at(paligned, (void *)ioremap_bot, size, prot);
|
|
if (ret)
|
|
ioremap_bot += size;
|
|
}
|
|
|
|
if (ret)
|
|
ret += addr & ~PAGE_MASK;
|
|
return ret;
|
|
}
|
|
|
|
void __iomem * __ioremap(phys_addr_t addr, unsigned long size,
|
|
unsigned long flags)
|
|
{
|
|
return __ioremap_caller(addr, size, __pgprot(flags), __builtin_return_address(0));
|
|
}
|
|
|
|
void __iomem * ioremap(phys_addr_t addr, unsigned long size)
|
|
{
|
|
pgprot_t prot = pgprot_noncached(PAGE_KERNEL);
|
|
void *caller = __builtin_return_address(0);
|
|
|
|
if (ppc_md.ioremap)
|
|
return ppc_md.ioremap(addr, size, prot, caller);
|
|
return __ioremap_caller(addr, size, prot, caller);
|
|
}
|
|
|
|
void __iomem * ioremap_wc(phys_addr_t addr, unsigned long size)
|
|
{
|
|
pgprot_t prot = pgprot_noncached_wc(PAGE_KERNEL);
|
|
void *caller = __builtin_return_address(0);
|
|
|
|
if (ppc_md.ioremap)
|
|
return ppc_md.ioremap(addr, size, prot, caller);
|
|
return __ioremap_caller(addr, size, prot, caller);
|
|
}
|
|
|
|
void __iomem *ioremap_coherent(phys_addr_t addr, unsigned long size)
|
|
{
|
|
pgprot_t prot = pgprot_cached(PAGE_KERNEL);
|
|
void *caller = __builtin_return_address(0);
|
|
|
|
if (ppc_md.ioremap)
|
|
return ppc_md.ioremap(addr, size, prot, caller);
|
|
return __ioremap_caller(addr, size, prot, caller);
|
|
}
|
|
|
|
void __iomem * ioremap_prot(phys_addr_t addr, unsigned long size,
|
|
unsigned long flags)
|
|
{
|
|
pte_t pte = __pte(flags);
|
|
void *caller = __builtin_return_address(0);
|
|
|
|
/* writeable implies dirty for kernel addresses */
|
|
if (pte_write(pte))
|
|
pte = pte_mkdirty(pte);
|
|
|
|
/* we don't want to let _PAGE_EXEC leak out */
|
|
pte = pte_exprotect(pte);
|
|
/*
|
|
* Force kernel mapping.
|
|
*/
|
|
pte = pte_mkprivileged(pte);
|
|
|
|
if (ppc_md.ioremap)
|
|
return ppc_md.ioremap(addr, size, pte_pgprot(pte), caller);
|
|
return __ioremap_caller(addr, size, pte_pgprot(pte), caller);
|
|
}
|
|
|
|
|
|
/*
|
|
* Unmap an IO region and remove it from imalloc'd list.
|
|
* Access to IO memory should be serialized by driver.
|
|
*/
|
|
void __iounmap(volatile void __iomem *token)
|
|
{
|
|
void *addr;
|
|
|
|
if (!slab_is_available())
|
|
return;
|
|
|
|
addr = (void *) ((unsigned long __force)
|
|
PCI_FIX_ADDR(token) & PAGE_MASK);
|
|
if ((unsigned long)addr < ioremap_bot) {
|
|
printk(KERN_WARNING "Attempt to iounmap early bolted mapping"
|
|
" at 0x%p\n", addr);
|
|
return;
|
|
}
|
|
vunmap(addr);
|
|
}
|
|
|
|
void iounmap(volatile void __iomem *token)
|
|
{
|
|
if (ppc_md.iounmap)
|
|
ppc_md.iounmap(token);
|
|
else
|
|
__iounmap(token);
|
|
}
|
|
|
|
EXPORT_SYMBOL(ioremap);
|
|
EXPORT_SYMBOL(ioremap_wc);
|
|
EXPORT_SYMBOL(ioremap_prot);
|
|
EXPORT_SYMBOL(__ioremap);
|
|
EXPORT_SYMBOL(__ioremap_at);
|
|
EXPORT_SYMBOL(iounmap);
|
|
EXPORT_SYMBOL(__iounmap);
|
|
EXPORT_SYMBOL(__iounmap_at);
|
|
|
|
#ifndef __PAGETABLE_PUD_FOLDED
|
|
/* 4 level page table */
|
|
struct page *pgd_page(pgd_t pgd)
|
|
{
|
|
if (pgd_is_leaf(pgd)) {
|
|
VM_WARN_ON(!pgd_huge(pgd));
|
|
return pte_page(pgd_pte(pgd));
|
|
}
|
|
return virt_to_page(pgd_page_vaddr(pgd));
|
|
}
|
|
#endif
|
|
|
|
struct page *pud_page(pud_t pud)
|
|
{
|
|
if (pud_is_leaf(pud)) {
|
|
VM_WARN_ON(!pud_huge(pud));
|
|
return pte_page(pud_pte(pud));
|
|
}
|
|
return virt_to_page(pud_page_vaddr(pud));
|
|
}
|
|
|
|
/*
|
|
* For hugepage we have pfn in the pmd, we use PTE_RPN_SHIFT bits for flags
|
|
* For PTE page, we have a PTE_FRAG_SIZE (4K) aligned virtual address.
|
|
*/
|
|
struct page *pmd_page(pmd_t pmd)
|
|
{
|
|
if (pmd_is_leaf(pmd)) {
|
|
VM_WARN_ON(!(pmd_large(pmd) || pmd_huge(pmd)));
|
|
return pte_page(pmd_pte(pmd));
|
|
}
|
|
return virt_to_page(pmd_page_vaddr(pmd));
|
|
}
|
|
|
|
#ifdef CONFIG_STRICT_KERNEL_RWX
|
|
void mark_rodata_ro(void)
|
|
{
|
|
if (!mmu_has_feature(MMU_FTR_KERNEL_RO)) {
|
|
pr_warn("Warning: Unable to mark rodata read only on this CPU.\n");
|
|
return;
|
|
}
|
|
|
|
if (radix_enabled())
|
|
radix__mark_rodata_ro();
|
|
else
|
|
hash__mark_rodata_ro();
|
|
|
|
// mark_initmem_nx() should have already run by now
|
|
ptdump_check_wx();
|
|
}
|
|
|
|
void mark_initmem_nx(void)
|
|
{
|
|
if (radix_enabled())
|
|
radix__mark_initmem_nx();
|
|
else
|
|
hash__mark_initmem_nx();
|
|
}
|
|
#endif
|