![Linus Torvalds](/assets/img/avatar_default.png)
This reverts commits5c9d2d5c26
,c7da82b894
, ande7fe7b5cae
. We'll probably need to revisit this, but basically we should not complicate the get_user_pages_fast() case, and checking the actual page table protection key bits will require more care anyway, since the protection keys depend on the exact state of the VM in question. Particularly when doing a "remote" page lookup (ie in somebody elses VM, not your own), you need to be much more careful than this was. Dave Hansen says: "So, the underlying bug here is that we now a get_user_pages_remote() and then go ahead and do the p*_access_permitted() checks against the current PKRU. This was introduced recently with the addition of the new p??_access_permitted() calls. We have checks in the VMA path for the "remote" gups and we avoid consulting PKRU for them. This got missed in the pkeys selftests because I did a ptrace read, but not a *write*. I also didn't explicitly test it against something where a COW needed to be done" It's also not entirely clear that it makes sense to check the protection key bits at this level at all. But one possible eventual solution is to make the get_user_pages_fast() case just abort if it sees protection key bits set, which makes us fall back to the regular get_user_pages() case, which then has a vma and can do the check there if we want to. We'll see. Somewhat related to this all: what we _do_ want to do some day is to check the PAGE_USER bit - it should obviously always be set for user pages, but it would be a good check to have back. Because we have no generic way to test for it, we lost it as part of moving over from the architecture-specific x86 GUP implementation to the generic one in commite585513b76
("x86/mm/gup: Switch GUP to the generic get_user_page_fast() implementation"). Cc: Peter Zijlstra <peterz@infradead.org> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Dave Hansen <dave.hansen@intel.com> Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: "Jérôme Glisse" <jglisse@redhat.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Al Viro <viro@zeniv.linux.org.uk> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
299 lines
6.4 KiB
C
299 lines
6.4 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Lockless get_user_pages_fast for sparc, cribbed from powerpc
|
|
*
|
|
* Copyright (C) 2008 Nick Piggin
|
|
* Copyright (C) 2008 Novell Inc.
|
|
*/
|
|
|
|
#include <linux/sched.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/vmstat.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/rwsem.h>
|
|
#include <asm/pgtable.h>
|
|
|
|
/*
|
|
* The performance critical leaf functions are made noinline otherwise gcc
|
|
* inlines everything into a single function which results in too much
|
|
* register pressure.
|
|
*/
|
|
static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
|
|
unsigned long end, int write, struct page **pages, int *nr)
|
|
{
|
|
unsigned long mask, result;
|
|
pte_t *ptep;
|
|
|
|
if (tlb_type == hypervisor) {
|
|
result = _PAGE_PRESENT_4V|_PAGE_P_4V;
|
|
if (write)
|
|
result |= _PAGE_WRITE_4V;
|
|
} else {
|
|
result = _PAGE_PRESENT_4U|_PAGE_P_4U;
|
|
if (write)
|
|
result |= _PAGE_WRITE_4U;
|
|
}
|
|
mask = result | _PAGE_SPECIAL;
|
|
|
|
ptep = pte_offset_kernel(&pmd, addr);
|
|
do {
|
|
struct page *page, *head;
|
|
pte_t pte = *ptep;
|
|
|
|
if ((pte_val(pte) & mask) != result)
|
|
return 0;
|
|
VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
|
|
|
|
/* The hugepage case is simplified on sparc64 because
|
|
* we encode the sub-page pfn offsets into the
|
|
* hugepage PTEs. We could optimize this in the future
|
|
* use page_cache_add_speculative() for the hugepage case.
|
|
*/
|
|
page = pte_page(pte);
|
|
head = compound_head(page);
|
|
if (!page_cache_get_speculative(head))
|
|
return 0;
|
|
if (unlikely(pte_val(pte) != pte_val(*ptep))) {
|
|
put_page(head);
|
|
return 0;
|
|
}
|
|
|
|
pages[*nr] = page;
|
|
(*nr)++;
|
|
} while (ptep++, addr += PAGE_SIZE, addr != end);
|
|
|
|
return 1;
|
|
}
|
|
|
|
static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
|
|
unsigned long end, int write, struct page **pages,
|
|
int *nr)
|
|
{
|
|
struct page *head, *page;
|
|
int refs;
|
|
|
|
if (!(pmd_val(pmd) & _PAGE_VALID))
|
|
return 0;
|
|
|
|
if (write && !pmd_write(pmd))
|
|
return 0;
|
|
|
|
refs = 0;
|
|
page = pmd_page(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
|
|
head = compound_head(page);
|
|
do {
|
|
VM_BUG_ON(compound_head(page) != head);
|
|
pages[*nr] = page;
|
|
(*nr)++;
|
|
page++;
|
|
refs++;
|
|
} while (addr += PAGE_SIZE, addr != end);
|
|
|
|
if (!page_cache_add_speculative(head, refs)) {
|
|
*nr -= refs;
|
|
return 0;
|
|
}
|
|
|
|
if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
|
|
*nr -= refs;
|
|
while (refs--)
|
|
put_page(head);
|
|
return 0;
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr,
|
|
unsigned long end, int write, struct page **pages,
|
|
int *nr)
|
|
{
|
|
struct page *head, *page;
|
|
int refs;
|
|
|
|
if (!(pud_val(pud) & _PAGE_VALID))
|
|
return 0;
|
|
|
|
if (write && !pud_write(pud))
|
|
return 0;
|
|
|
|
refs = 0;
|
|
page = pud_page(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
|
|
head = compound_head(page);
|
|
do {
|
|
VM_BUG_ON(compound_head(page) != head);
|
|
pages[*nr] = page;
|
|
(*nr)++;
|
|
page++;
|
|
refs++;
|
|
} while (addr += PAGE_SIZE, addr != end);
|
|
|
|
if (!page_cache_add_speculative(head, refs)) {
|
|
*nr -= refs;
|
|
return 0;
|
|
}
|
|
|
|
if (unlikely(pud_val(pud) != pud_val(*pudp))) {
|
|
*nr -= refs;
|
|
while (refs--)
|
|
put_page(head);
|
|
return 0;
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
|
|
int write, struct page **pages, int *nr)
|
|
{
|
|
unsigned long next;
|
|
pmd_t *pmdp;
|
|
|
|
pmdp = pmd_offset(&pud, addr);
|
|
do {
|
|
pmd_t pmd = *pmdp;
|
|
|
|
next = pmd_addr_end(addr, end);
|
|
if (pmd_none(pmd))
|
|
return 0;
|
|
if (unlikely(pmd_large(pmd))) {
|
|
if (!gup_huge_pmd(pmdp, pmd, addr, next,
|
|
write, pages, nr))
|
|
return 0;
|
|
} else if (!gup_pte_range(pmd, addr, next, write,
|
|
pages, nr))
|
|
return 0;
|
|
} while (pmdp++, addr = next, addr != end);
|
|
|
|
return 1;
|
|
}
|
|
|
|
static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
|
|
int write, struct page **pages, int *nr)
|
|
{
|
|
unsigned long next;
|
|
pud_t *pudp;
|
|
|
|
pudp = pud_offset(&pgd, addr);
|
|
do {
|
|
pud_t pud = *pudp;
|
|
|
|
next = pud_addr_end(addr, end);
|
|
if (pud_none(pud))
|
|
return 0;
|
|
if (unlikely(pud_large(pud))) {
|
|
if (!gup_huge_pud(pudp, pud, addr, next,
|
|
write, pages, nr))
|
|
return 0;
|
|
} else if (!gup_pmd_range(pud, addr, next, write, pages, nr))
|
|
return 0;
|
|
} while (pudp++, addr = next, addr != end);
|
|
|
|
return 1;
|
|
}
|
|
|
|
int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|
struct page **pages)
|
|
{
|
|
struct mm_struct *mm = current->mm;
|
|
unsigned long addr, len, end;
|
|
unsigned long next, flags;
|
|
pgd_t *pgdp;
|
|
int nr = 0;
|
|
|
|
start &= PAGE_MASK;
|
|
addr = start;
|
|
len = (unsigned long) nr_pages << PAGE_SHIFT;
|
|
end = start + len;
|
|
|
|
local_irq_save(flags);
|
|
pgdp = pgd_offset(mm, addr);
|
|
do {
|
|
pgd_t pgd = *pgdp;
|
|
|
|
next = pgd_addr_end(addr, end);
|
|
if (pgd_none(pgd))
|
|
break;
|
|
if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
|
|
break;
|
|
} while (pgdp++, addr = next, addr != end);
|
|
local_irq_restore(flags);
|
|
|
|
return nr;
|
|
}
|
|
|
|
int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|
struct page **pages)
|
|
{
|
|
struct mm_struct *mm = current->mm;
|
|
unsigned long addr, len, end;
|
|
unsigned long next;
|
|
pgd_t *pgdp;
|
|
int nr = 0;
|
|
|
|
start &= PAGE_MASK;
|
|
addr = start;
|
|
len = (unsigned long) nr_pages << PAGE_SHIFT;
|
|
end = start + len;
|
|
|
|
/*
|
|
* XXX: batch / limit 'nr', to avoid large irq off latency
|
|
* needs some instrumenting to determine the common sizes used by
|
|
* important workloads (eg. DB2), and whether limiting the batch size
|
|
* will decrease performance.
|
|
*
|
|
* It seems like we're in the clear for the moment. Direct-IO is
|
|
* the main guy that batches up lots of get_user_pages, and even
|
|
* they are limited to 64-at-a-time which is not so many.
|
|
*/
|
|
/*
|
|
* This doesn't prevent pagetable teardown, but does prevent
|
|
* the pagetables from being freed on sparc.
|
|
*
|
|
* So long as we atomically load page table pointers versus teardown,
|
|
* we can follow the address down to the the page and take a ref on it.
|
|
*/
|
|
local_irq_disable();
|
|
|
|
pgdp = pgd_offset(mm, addr);
|
|
do {
|
|
pgd_t pgd = *pgdp;
|
|
|
|
next = pgd_addr_end(addr, end);
|
|
if (pgd_none(pgd))
|
|
goto slow;
|
|
if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
|
|
goto slow;
|
|
} while (pgdp++, addr = next, addr != end);
|
|
|
|
local_irq_enable();
|
|
|
|
VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT);
|
|
return nr;
|
|
|
|
{
|
|
int ret;
|
|
|
|
slow:
|
|
local_irq_enable();
|
|
|
|
/* Try to get the remaining pages with get_user_pages */
|
|
start += nr << PAGE_SHIFT;
|
|
pages += nr;
|
|
|
|
ret = get_user_pages_unlocked(start,
|
|
(end - start) >> PAGE_SHIFT, pages,
|
|
write ? FOLL_WRITE : 0);
|
|
|
|
/* Have to be a bit careful with return values */
|
|
if (nr > 0) {
|
|
if (ret < 0)
|
|
ret = nr;
|
|
else
|
|
ret += nr;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
}
|