![Punit Agrawal](/assets/img/avatar_default.png)
A poisoned or migrated hugepage is stored as a swap entry in the page tables. On architectures that support hugepages consisting of contiguous page table entries (such as on arm64) this leads to ambiguity in determining the page table entry to return in huge_pte_offset() when a poisoned entry is encountered. Let's remove the ambiguity by adding a size parameter to convey additional information about the requested address. Also fixup the definition/usage of huge_pte_offset() throughout the tree. Link: http://lkml.kernel.org/r/20170522133604.11392-4-punit.agrawal@arm.com Signed-off-by: Punit Agrawal <punit.agrawal@arm.com> Acked-by: Steve Capper <steve.capper@arm.com> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Will Deacon <will.deacon@arm.com> Cc: Tony Luck <tony.luck@intel.com> Cc: Fenghua Yu <fenghua.yu@intel.com> Cc: James Hogan <james.hogan@imgtec.com> (odd fixer:METAG ARCHITECTURE) Cc: Ralf Baechle <ralf@linux-mips.org> (supporter:MIPS) Cc: "James E.J. Bottomley" <jejb@parisc-linux.org> Cc: Helge Deller <deller@gmx.de> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Paul Mackerras <paulus@samba.org> Cc: Michael Ellerman <mpe@ellerman.id.au> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Heiko Carstens <heiko.carstens@de.ibm.com> Cc: Yoshinori Sato <ysato@users.sourceforge.jp> Cc: Rich Felker <dalias@libc.org> Cc: "David S. Miller" <davem@davemloft.net> Cc: Chris Metcalf <cmetcalf@mellanox.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Ingo Molnar <mingo@redhat.com> Cc: "H. Peter Anvin" <hpa@zytor.com> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Michal Hocko <mhocko@suse.com> Cc: Mike Kravetz <mike.kravetz@oracle.com> Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com> Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> Cc: Hillf Danton <hillf.zj@alibaba-inc.com> Cc: Mark Rutland <mark.rutland@arm.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
251 lines
5.5 KiB
C
251 lines
5.5 KiB
C
/*
|
|
* arch/metag/mm/hugetlbpage.c
|
|
*
|
|
* METAG HugeTLB page support.
|
|
*
|
|
* Cloned from SuperH
|
|
*
|
|
* Cloned from sparc64 by Paul Mundt.
|
|
*
|
|
* Copyright (C) 2002, 2003 David S. Miller (davem@redhat.com)
|
|
*/
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/hugetlb.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/sysctl.h>
|
|
|
|
#include <asm/mman.h>
|
|
#include <asm/pgalloc.h>
|
|
#include <asm/tlb.h>
|
|
#include <asm/tlbflush.h>
|
|
#include <asm/cacheflush.h>
|
|
|
|
/*
|
|
* If the arch doesn't supply something else, assume that hugepage
|
|
* size aligned regions are ok without further preparation.
|
|
*/
|
|
int prepare_hugepage_range(struct file *file, unsigned long addr,
|
|
unsigned long len)
|
|
{
|
|
struct mm_struct *mm = current->mm;
|
|
struct hstate *h = hstate_file(file);
|
|
struct vm_area_struct *vma;
|
|
|
|
if (len & ~huge_page_mask(h))
|
|
return -EINVAL;
|
|
if (addr & ~huge_page_mask(h))
|
|
return -EINVAL;
|
|
if (TASK_SIZE - len < addr)
|
|
return -EINVAL;
|
|
|
|
vma = find_vma(mm, ALIGN_HUGEPT(addr));
|
|
if (vma && !(vma->vm_flags & MAP_HUGETLB))
|
|
return -EINVAL;
|
|
|
|
vma = find_vma(mm, addr);
|
|
if (vma) {
|
|
if (addr + len > vma->vm_start)
|
|
return -EINVAL;
|
|
if (!(vma->vm_flags & MAP_HUGETLB) &&
|
|
(ALIGN_HUGEPT(addr + len) > vma->vm_start))
|
|
return -EINVAL;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
pte_t *huge_pte_alloc(struct mm_struct *mm,
|
|
unsigned long addr, unsigned long sz)
|
|
{
|
|
pgd_t *pgd;
|
|
pud_t *pud;
|
|
pmd_t *pmd;
|
|
pte_t *pte;
|
|
|
|
pgd = pgd_offset(mm, addr);
|
|
pud = pud_offset(pgd, addr);
|
|
pmd = pmd_offset(pud, addr);
|
|
pte = pte_alloc_map(mm, pmd, addr);
|
|
pgd->pgd &= ~_PAGE_SZ_MASK;
|
|
pgd->pgd |= _PAGE_SZHUGE;
|
|
|
|
return pte;
|
|
}
|
|
|
|
pte_t *huge_pte_offset(struct mm_struct *mm,
|
|
unsigned long addr, unsigned long sz)
|
|
{
|
|
pgd_t *pgd;
|
|
pud_t *pud;
|
|
pmd_t *pmd;
|
|
pte_t *pte = NULL;
|
|
|
|
pgd = pgd_offset(mm, addr);
|
|
pud = pud_offset(pgd, addr);
|
|
pmd = pmd_offset(pud, addr);
|
|
pte = pte_offset_kernel(pmd, addr);
|
|
|
|
return pte;
|
|
}
|
|
|
|
int pmd_huge(pmd_t pmd)
|
|
{
|
|
return pmd_page_shift(pmd) > PAGE_SHIFT;
|
|
}
|
|
|
|
int pud_huge(pud_t pud)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
|
pmd_t *pmd, int write)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
|
|
|
|
/*
|
|
* Look for an unmapped area starting after another hugetlb vma.
|
|
* There are guaranteed to be no huge pte's spare if all the huge pages are
|
|
* full size (4MB), so in that case compile out this search.
|
|
*/
|
|
#if HPAGE_SHIFT == HUGEPT_SHIFT
|
|
static inline unsigned long
|
|
hugetlb_get_unmapped_area_existing(unsigned long len)
|
|
{
|
|
return 0;
|
|
}
|
|
#else
|
|
static unsigned long
|
|
hugetlb_get_unmapped_area_existing(unsigned long len)
|
|
{
|
|
struct mm_struct *mm = current->mm;
|
|
struct vm_area_struct *vma;
|
|
unsigned long start_addr, addr;
|
|
int after_huge;
|
|
|
|
if (mm->context.part_huge) {
|
|
start_addr = mm->context.part_huge;
|
|
after_huge = 1;
|
|
} else {
|
|
start_addr = TASK_UNMAPPED_BASE;
|
|
after_huge = 0;
|
|
}
|
|
new_search:
|
|
addr = start_addr;
|
|
|
|
for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
|
|
if ((!vma && !after_huge) || TASK_SIZE - len < addr) {
|
|
/*
|
|
* Start a new search - just in case we missed
|
|
* some holes.
|
|
*/
|
|
if (start_addr != TASK_UNMAPPED_BASE) {
|
|
start_addr = TASK_UNMAPPED_BASE;
|
|
goto new_search;
|
|
}
|
|
return 0;
|
|
}
|
|
/* skip ahead if we've aligned right over some vmas */
|
|
if (vma && vma->vm_end <= addr)
|
|
continue;
|
|
/* space before the next vma? */
|
|
if (after_huge && (!vma || ALIGN_HUGEPT(addr + len)
|
|
<= vma->vm_start)) {
|
|
unsigned long end = addr + len;
|
|
if (end & HUGEPT_MASK)
|
|
mm->context.part_huge = end;
|
|
else if (addr == mm->context.part_huge)
|
|
mm->context.part_huge = 0;
|
|
return addr;
|
|
}
|
|
if (vma->vm_flags & MAP_HUGETLB) {
|
|
/* space after a huge vma in 2nd level page table? */
|
|
if (vma->vm_end & HUGEPT_MASK) {
|
|
after_huge = 1;
|
|
/* no need to align to the next PT block */
|
|
addr = vma->vm_end;
|
|
continue;
|
|
}
|
|
}
|
|
after_huge = 0;
|
|
addr = ALIGN_HUGEPT(vma->vm_end);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
/* Do a full search to find an area without any nearby normal pages. */
|
|
static unsigned long
|
|
hugetlb_get_unmapped_area_new_pmd(unsigned long len)
|
|
{
|
|
struct vm_unmapped_area_info info;
|
|
|
|
info.flags = 0;
|
|
info.length = len;
|
|
info.low_limit = TASK_UNMAPPED_BASE;
|
|
info.high_limit = TASK_SIZE;
|
|
info.align_mask = PAGE_MASK & HUGEPT_MASK;
|
|
info.align_offset = 0;
|
|
return vm_unmapped_area(&info);
|
|
}
|
|
|
|
unsigned long
|
|
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
|
|
unsigned long len, unsigned long pgoff, unsigned long flags)
|
|
{
|
|
struct hstate *h = hstate_file(file);
|
|
|
|
if (len & ~huge_page_mask(h))
|
|
return -EINVAL;
|
|
if (len > TASK_SIZE)
|
|
return -ENOMEM;
|
|
|
|
if (flags & MAP_FIXED) {
|
|
if (prepare_hugepage_range(file, addr, len))
|
|
return -EINVAL;
|
|
return addr;
|
|
}
|
|
|
|
if (addr) {
|
|
addr = ALIGN(addr, huge_page_size(h));
|
|
if (!prepare_hugepage_range(file, addr, len))
|
|
return addr;
|
|
}
|
|
|
|
/*
|
|
* Look for an existing hugetlb vma with space after it (this is to to
|
|
* minimise fragmentation caused by huge pages.
|
|
*/
|
|
addr = hugetlb_get_unmapped_area_existing(len);
|
|
if (addr)
|
|
return addr;
|
|
|
|
/*
|
|
* Find an unmapped naturally aligned set of 4MB blocks that we can use
|
|
* for huge pages.
|
|
*/
|
|
return hugetlb_get_unmapped_area_new_pmd(len);
|
|
}
|
|
|
|
#endif /*HAVE_ARCH_HUGETLB_UNMAPPED_AREA*/
|
|
|
|
/* necessary for boot time 4MB huge page allocation */
|
|
static __init int setup_hugepagesz(char *opt)
|
|
{
|
|
unsigned long ps = memparse(opt, &opt);
|
|
if (ps == (1 << HPAGE_SHIFT)) {
|
|
hugetlb_add_hstate(HPAGE_SHIFT - PAGE_SHIFT);
|
|
} else {
|
|
hugetlb_bad_size();
|
|
pr_err("hugepagesz: Unsupported page size %lu M\n",
|
|
ps >> 20);
|
|
return 0;
|
|
}
|
|
return 1;
|
|
}
|
|
__setup("hugepagesz=", setup_hugepagesz);
|