mm/gup: change GUP fast to use flags rather than a write 'bool'
To facilitate additional options to get_user_pages_fast() change the singular write parameter to be gup_flags. This patch does not change any functionality. New functionality will follow in subsequent patches. Some of the get_user_pages_fast() call sites were unchanged because they already passed FOLL_WRITE or 0 for the write parameter. NOTE: It was suggested to change the ordering of the get_user_pages_fast() arguments to ensure that callers were converted. This breaks the current GUP call site convention of having the returned pages be the final parameter. So the suggestion was rejected. Link: http://lkml.kernel.org/r/20190328084422.29911-4-ira.weiny@intel.com Link: http://lkml.kernel.org/r/20190317183438.2057-4-ira.weiny@intel.com Signed-off-by: Ira Weiny <ira.weiny@intel.com> Reviewed-by: Mike Marshall <hubcap@omnibond.com> Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Borislav Petkov <bp@alien8.de> Cc: Dan Williams <dan.j.williams@intel.com> Cc: "David S. Miller" <davem@davemloft.net> Cc: Heiko Carstens <heiko.carstens@de.ibm.com> Cc: Ingo Molnar <mingo@redhat.com> Cc: James Hogan <jhogan@kernel.org> Cc: Jason Gunthorpe <jgg@ziepe.ca> Cc: John Hubbard <jhubbard@nvidia.com> Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Michal Hocko <mhocko@kernel.org> Cc: Paul Mackerras <paulus@samba.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Ralf Baechle <ralf@linux-mips.org> Cc: Rich Felker <dalias@libc.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Yoshinori Sato <ysato@users.sourceforge.jp> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
b798bec474
commit
73b0140bf0
|
@ -235,7 +235,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
* get_user_pages_fast() - pin user pages in memory
|
||||
* @start: starting user address
|
||||
* @nr_pages: number of pages from start to pin
|
||||
* @write: whether pages will be written to
|
||||
* @gup_flags: flags modifying pin behaviour
|
||||
* @pages: array that receives pointers to the pages pinned.
|
||||
* Should be at least nr_pages long.
|
||||
*
|
||||
|
@ -247,8 +247,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
* requested. If nr_pages is 0 or negative, returns 0. If no pages
|
||||
* were pinned, returns -errno.
|
||||
*/
|
||||
int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
||||
struct page **pages)
|
||||
int get_user_pages_fast(unsigned long start, int nr_pages,
|
||||
unsigned int gup_flags, struct page **pages)
|
||||
{
|
||||
struct mm_struct *mm = current->mm;
|
||||
unsigned long addr, len, end;
|
||||
|
@ -273,7 +273,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
next = pgd_addr_end(addr, end);
|
||||
if (pgd_none(pgd))
|
||||
goto slow;
|
||||
if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
|
||||
if (!gup_pud_range(pgd, addr, next, gup_flags & FOLL_WRITE,
|
||||
pages, &nr))
|
||||
goto slow;
|
||||
} while (pgdp++, addr = next, addr != end);
|
||||
local_irq_enable();
|
||||
|
@ -289,7 +290,7 @@ slow_irqon:
|
|||
pages += nr;
|
||||
|
||||
ret = get_user_pages_unlocked(start, (end - start) >> PAGE_SHIFT,
|
||||
pages, write ? FOLL_WRITE : 0);
|
||||
pages, gup_flags);
|
||||
|
||||
/* Have to be a bit careful with return values */
|
||||
if (nr > 0) {
|
||||
|
|
|
@ -600,7 +600,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
|
|||
/* If writing != 0, then the HPTE must allow writing, if we get here */
|
||||
write_ok = writing;
|
||||
hva = gfn_to_hva_memslot(memslot, gfn);
|
||||
npages = get_user_pages_fast(hva, 1, writing, pages);
|
||||
npages = get_user_pages_fast(hva, 1, writing ? FOLL_WRITE : 0, pages);
|
||||
if (npages < 1) {
|
||||
/* Check if it's an I/O mapping */
|
||||
down_read(¤t->mm->mmap_sem);
|
||||
|
@ -1193,7 +1193,7 @@ void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,
|
|||
if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
|
||||
goto err;
|
||||
hva = gfn_to_hva_memslot(memslot, gfn);
|
||||
npages = get_user_pages_fast(hva, 1, 1, pages);
|
||||
npages = get_user_pages_fast(hva, 1, FOLL_WRITE, pages);
|
||||
if (npages < 1)
|
||||
goto err;
|
||||
page = pages[0];
|
||||
|
|
|
@ -783,7 +783,7 @@ int kvm_vcpu_ioctl_config_tlb(struct kvm_vcpu *vcpu,
|
|||
if (!pages)
|
||||
return -ENOMEM;
|
||||
|
||||
ret = get_user_pages_fast(cfg->array, num_pages, 1, pages);
|
||||
ret = get_user_pages_fast(cfg->array, num_pages, FOLL_WRITE, pages);
|
||||
if (ret < 0)
|
||||
goto free_pages;
|
||||
|
||||
|
|
|
@ -2376,7 +2376,7 @@ static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr)
|
|||
ret = -EFAULT;
|
||||
goto out;
|
||||
}
|
||||
ret = get_user_pages_fast(map->addr, 1, 1, &map->page);
|
||||
ret = get_user_pages_fast(map->addr, 1, FOLL_WRITE, &map->page);
|
||||
if (ret < 0)
|
||||
goto out;
|
||||
BUG_ON(ret != 1);
|
||||
|
|
|
@ -204,7 +204,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
* get_user_pages_fast() - pin user pages in memory
|
||||
* @start: starting user address
|
||||
* @nr_pages: number of pages from start to pin
|
||||
* @write: whether pages will be written to
|
||||
* @gup_flags: flags modifying pin behaviour
|
||||
* @pages: array that receives pointers to the pages pinned.
|
||||
* Should be at least nr_pages long.
|
||||
*
|
||||
|
@ -216,8 +216,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
* requested. If nr_pages is 0 or negative, returns 0. If no pages
|
||||
* were pinned, returns -errno.
|
||||
*/
|
||||
int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
||||
struct page **pages)
|
||||
int get_user_pages_fast(unsigned long start, int nr_pages,
|
||||
unsigned int gup_flags, struct page **pages)
|
||||
{
|
||||
struct mm_struct *mm = current->mm;
|
||||
unsigned long addr, len, end;
|
||||
|
@ -241,7 +241,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
next = pgd_addr_end(addr, end);
|
||||
if (pgd_none(pgd))
|
||||
goto slow;
|
||||
if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
|
||||
if (!gup_pud_range(pgd, addr, next, gup_flags & FOLL_WRITE,
|
||||
pages, &nr))
|
||||
goto slow;
|
||||
} while (pgdp++, addr = next, addr != end);
|
||||
local_irq_enable();
|
||||
|
@ -261,7 +262,7 @@ slow_irqon:
|
|||
|
||||
ret = get_user_pages_unlocked(start,
|
||||
(end - start) >> PAGE_SHIFT, pages,
|
||||
write ? FOLL_WRITE : 0);
|
||||
gup_flags);
|
||||
|
||||
/* Have to be a bit careful with return values */
|
||||
if (nr > 0) {
|
||||
|
|
|
@ -245,8 +245,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
return nr;
|
||||
}
|
||||
|
||||
int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
||||
struct page **pages)
|
||||
int get_user_pages_fast(unsigned long start, int nr_pages,
|
||||
unsigned int gup_flags, struct page **pages)
|
||||
{
|
||||
struct mm_struct *mm = current->mm;
|
||||
unsigned long addr, len, end;
|
||||
|
@ -303,7 +303,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
next = pgd_addr_end(addr, end);
|
||||
if (pgd_none(pgd))
|
||||
goto slow;
|
||||
if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
|
||||
if (!gup_pud_range(pgd, addr, next, gup_flags & FOLL_WRITE,
|
||||
pages, &nr))
|
||||
goto slow;
|
||||
} while (pgdp++, addr = next, addr != end);
|
||||
|
||||
|
@ -324,7 +325,7 @@ slow:
|
|||
|
||||
ret = get_user_pages_unlocked(start,
|
||||
(end - start) >> PAGE_SHIFT, pages,
|
||||
write ? FOLL_WRITE : 0);
|
||||
gup_flags);
|
||||
|
||||
/* Have to be a bit careful with return values */
|
||||
if (nr > 0) {
|
||||
|
|
|
@ -140,7 +140,7 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
|
|||
pt_element_t *table;
|
||||
struct page *page;
|
||||
|
||||
npages = get_user_pages_fast((unsigned long)ptep_user, 1, 1, &page);
|
||||
npages = get_user_pages_fast((unsigned long)ptep_user, 1, FOLL_WRITE, &page);
|
||||
/* Check if the user is doing something meaningless. */
|
||||
if (unlikely(npages != 1))
|
||||
return -EFAULT;
|
||||
|
|
|
@ -1805,7 +1805,7 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
|
|||
return NULL;
|
||||
|
||||
/* Pin the user virtual address. */
|
||||
npinned = get_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages);
|
||||
npinned = get_user_pages_fast(uaddr, npages, FOLL_WRITE, pages);
|
||||
if (npinned != npages) {
|
||||
pr_err("SEV: Failure locking %lu pages.\n", npages);
|
||||
goto err;
|
||||
|
|
|
@ -102,7 +102,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
|
|||
goto unlock_vm;
|
||||
}
|
||||
|
||||
pinned = get_user_pages_fast(region->user_addr, npages, 1,
|
||||
pinned = get_user_pages_fast(region->user_addr, npages, FOLL_WRITE,
|
||||
region->pages);
|
||||
if (pinned < 0) {
|
||||
ret = pinned;
|
||||
|
|
|
@ -243,7 +243,8 @@ via_lock_all_dma_pages(drm_via_sg_info_t *vsg, drm_via_dmablit_t *xfer)
|
|||
if (NULL == vsg->pages)
|
||||
return -ENOMEM;
|
||||
ret = get_user_pages_fast((unsigned long)xfer->mem_addr,
|
||||
vsg->num_pages, vsg->direction == DMA_FROM_DEVICE,
|
||||
vsg->num_pages,
|
||||
vsg->direction == DMA_FROM_DEVICE ? FOLL_WRITE : 0,
|
||||
vsg->pages);
|
||||
if (ret != vsg->num_pages) {
|
||||
if (ret < 0)
|
||||
|
|
|
@ -105,7 +105,8 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np
|
|||
{
|
||||
int ret;
|
||||
|
||||
ret = get_user_pages_fast(vaddr, npages, writable, pages);
|
||||
ret = get_user_pages_fast(vaddr, npages, writable ? FOLL_WRITE : 0,
|
||||
pages);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
|
|
|
@ -603,7 +603,7 @@ int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
|
|||
/* pin user pages in memory */
|
||||
rc = get_user_pages_fast(data & PAGE_MASK, /* page aligned addr */
|
||||
m->nr_pages,
|
||||
m->write, /* readable/writable */
|
||||
m->write ? FOLL_WRITE : 0, /* readable/writable */
|
||||
m->page_list); /* ptrs to pages */
|
||||
if (rc < 0)
|
||||
goto fail_get_user_pages;
|
||||
|
|
|
@ -242,7 +242,7 @@ static int vmci_host_setup_notify(struct vmci_ctx *context,
|
|||
/*
|
||||
* Lock physical page backing a given user VA.
|
||||
*/
|
||||
retval = get_user_pages_fast(uva, 1, 1, &context->notify_page);
|
||||
retval = get_user_pages_fast(uva, 1, FOLL_WRITE, &context->notify_page);
|
||||
if (retval != 1) {
|
||||
context->notify_page = NULL;
|
||||
return VMCI_ERROR_GENERIC;
|
||||
|
|
|
@ -659,7 +659,8 @@ static int qp_host_get_user_memory(u64 produce_uva,
|
|||
int err = VMCI_SUCCESS;
|
||||
|
||||
retval = get_user_pages_fast((uintptr_t) produce_uva,
|
||||
produce_q->kernel_if->num_pages, 1,
|
||||
produce_q->kernel_if->num_pages,
|
||||
FOLL_WRITE,
|
||||
produce_q->kernel_if->u.h.header_page);
|
||||
if (retval < (int)produce_q->kernel_if->num_pages) {
|
||||
pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
|
||||
|
@ -671,7 +672,8 @@ static int qp_host_get_user_memory(u64 produce_uva,
|
|||
}
|
||||
|
||||
retval = get_user_pages_fast((uintptr_t) consume_uva,
|
||||
consume_q->kernel_if->num_pages, 1,
|
||||
consume_q->kernel_if->num_pages,
|
||||
FOLL_WRITE,
|
||||
consume_q->kernel_if->u.h.header_page);
|
||||
if (retval < (int)consume_q->kernel_if->num_pages) {
|
||||
pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
|
||||
|
|
|
@ -274,7 +274,8 @@ static int pin_user_pages(unsigned long first_page,
|
|||
*iter_last_page_size = last_page_size;
|
||||
}
|
||||
|
||||
ret = get_user_pages_fast(first_page, requested_pages, !is_write,
|
||||
ret = get_user_pages_fast(first_page, requested_pages,
|
||||
!is_write ? FOLL_WRITE : 0,
|
||||
pages);
|
||||
if (ret <= 0)
|
||||
return -EFAULT;
|
||||
|
|
|
@ -868,7 +868,9 @@ rio_dma_transfer(struct file *filp, u32 transfer_mode,
|
|||
|
||||
pinned = get_user_pages_fast(
|
||||
(unsigned long)xfer->loc_addr & PAGE_MASK,
|
||||
nr_pages, dir == DMA_FROM_DEVICE, page_list);
|
||||
nr_pages,
|
||||
dir == DMA_FROM_DEVICE ? FOLL_WRITE : 0,
|
||||
page_list);
|
||||
|
||||
if (pinned != nr_pages) {
|
||||
if (pinned < 0) {
|
||||
|
|
|
@ -437,7 +437,7 @@ static int dax_lock_page(void *va, struct page **p)
|
|||
|
||||
dax_dbg("uva %p", va);
|
||||
|
||||
ret = get_user_pages_fast((unsigned long)va, 1, 1, p);
|
||||
ret = get_user_pages_fast((unsigned long)va, 1, FOLL_WRITE, p);
|
||||
if (ret == 1) {
|
||||
dax_dbg("locked page %p, for VA %p", *p, va);
|
||||
return 0;
|
||||
|
|
|
@ -4922,7 +4922,8 @@ static int sgl_map_user_pages(struct st_buffer *STbp,
|
|||
|
||||
/* Try to fault in all of the necessary pages */
|
||||
/* rw==READ means read from drive, write into memory area */
|
||||
res = get_user_pages_fast(uaddr, nr_pages, rw == READ, pages);
|
||||
res = get_user_pages_fast(uaddr, nr_pages, rw == READ ? FOLL_WRITE : 0,
|
||||
pages);
|
||||
|
||||
/* Errors and no page mapped should return here */
|
||||
if (res < nr_pages)
|
||||
|
|
|
@ -486,8 +486,8 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl,
|
|||
ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr +
|
||||
off + i * PAGE_SIZE;
|
||||
} else {
|
||||
ret = get_user_pages_fast(page_addr - offset, 1, 1,
|
||||
&page);
|
||||
ret = get_user_pages_fast(page_addr - offset, 1,
|
||||
FOLL_WRITE, &page);
|
||||
|
||||
if (ret <= 0) {
|
||||
dev_err(pg_tbl->device,
|
||||
|
|
|
@ -273,7 +273,7 @@ struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr,
|
|||
goto err;
|
||||
}
|
||||
|
||||
rc = get_user_pages_fast(start, num_pages, 1, shm->pages);
|
||||
rc = get_user_pages_fast(start, num_pages, FOLL_WRITE, shm->pages);
|
||||
if (rc > 0)
|
||||
shm->num_pages = rc;
|
||||
if (rc != num_pages) {
|
||||
|
|
|
@ -532,7 +532,8 @@ static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)
|
|||
enum dma_data_direction direction = iommu_tce_direction(tce);
|
||||
|
||||
if (get_user_pages_fast(tce & PAGE_MASK, 1,
|
||||
direction != DMA_TO_DEVICE, &page) != 1)
|
||||
direction != DMA_TO_DEVICE ? FOLL_WRITE : 0,
|
||||
&page) != 1)
|
||||
return -EFAULT;
|
||||
|
||||
*hpa = __pa((unsigned long) page_address(page));
|
||||
|
|
|
@ -1704,7 +1704,7 @@ static int set_bit_to_user(int nr, void __user *addr)
|
|||
int bit = nr + (log % PAGE_SIZE) * 8;
|
||||
int r;
|
||||
|
||||
r = get_user_pages_fast(log, 1, 1, &page);
|
||||
r = get_user_pages_fast(log, 1, FOLL_WRITE, &page);
|
||||
if (r < 0)
|
||||
return r;
|
||||
BUG_ON(r != 1);
|
||||
|
|
|
@ -686,7 +686,7 @@ static ssize_t pvr2fb_write(struct fb_info *info, const char *buf,
|
|||
if (!pages)
|
||||
return -ENOMEM;
|
||||
|
||||
ret = get_user_pages_fast((unsigned long)buf, nr_pages, true, pages);
|
||||
ret = get_user_pages_fast((unsigned long)buf, nr_pages, FOLL_WRITE, pages);
|
||||
if (ret < nr_pages) {
|
||||
nr_pages = ret;
|
||||
ret = -EINVAL;
|
||||
|
|
|
@ -244,7 +244,7 @@ static long ioctl_memcpy(struct fsl_hv_ioctl_memcpy __user *p)
|
|||
|
||||
/* Get the physical addresses of the source buffer */
|
||||
num_pinned = get_user_pages_fast(param.local_vaddr - lb_offset,
|
||||
num_pages, param.source != -1, pages);
|
||||
num_pages, param.source != -1 ? FOLL_WRITE : 0, pages);
|
||||
|
||||
if (num_pinned != num_pages) {
|
||||
/* get_user_pages() failed */
|
||||
|
|
|
@ -852,7 +852,7 @@ static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt,
|
|||
unsigned long xen_pfn;
|
||||
int ret;
|
||||
|
||||
ret = get_user_pages_fast(addr, 1, writeable, &page);
|
||||
ret = get_user_pages_fast(addr, 1, writeable ? FOLL_WRITE : 0, &page);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
|
|
|
@ -269,7 +269,7 @@ orangefs_bufmap_map(struct orangefs_bufmap *bufmap,
|
|||
|
||||
/* map the pages */
|
||||
ret = get_user_pages_fast((unsigned long)user_desc->ptr,
|
||||
bufmap->page_count, 1, bufmap->page_array);
|
||||
bufmap->page_count, FOLL_WRITE, bufmap->page_array);
|
||||
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
|
|
@ -1505,8 +1505,8 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
|
|||
long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
|
||||
struct page **pages, unsigned int gup_flags);
|
||||
|
||||
int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
||||
struct page **pages);
|
||||
int get_user_pages_fast(unsigned long start, int nr_pages,
|
||||
unsigned int gup_flags, struct page **pages);
|
||||
|
||||
/* Container for pinned pfns / pages */
|
||||
struct frame_vector {
|
||||
|
|
|
@ -543,7 +543,7 @@ again:
|
|||
if (unlikely(should_fail_futex(fshared)))
|
||||
return -EFAULT;
|
||||
|
||||
err = get_user_pages_fast(address, 1, 1, &page);
|
||||
err = get_user_pages_fast(address, 1, FOLL_WRITE, &page);
|
||||
/*
|
||||
* If write access is not required (eg. FUTEX_WAIT), try
|
||||
* and get read-only access.
|
||||
|
|
|
@ -1293,7 +1293,9 @@ ssize_t iov_iter_get_pages(struct iov_iter *i,
|
|||
len = maxpages * PAGE_SIZE;
|
||||
addr &= ~(PAGE_SIZE - 1);
|
||||
n = DIV_ROUND_UP(len, PAGE_SIZE);
|
||||
res = get_user_pages_fast(addr, n, iov_iter_rw(i) != WRITE, pages);
|
||||
res = get_user_pages_fast(addr, n,
|
||||
iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0,
|
||||
pages);
|
||||
if (unlikely(res < 0))
|
||||
return res;
|
||||
return (res == n ? len : res * PAGE_SIZE) - *start;
|
||||
|
@ -1374,7 +1376,8 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
|
|||
p = get_pages_array(n);
|
||||
if (!p)
|
||||
return -ENOMEM;
|
||||
res = get_user_pages_fast(addr, n, iov_iter_rw(i) != WRITE, p);
|
||||
res = get_user_pages_fast(addr, n,
|
||||
iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, p);
|
||||
if (unlikely(res < 0)) {
|
||||
kvfree(p);
|
||||
return res;
|
||||
|
|
10
mm/gup.c
10
mm/gup.c
|
@ -2062,7 +2062,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
* get_user_pages_fast() - pin user pages in memory
|
||||
* @start: starting user address
|
||||
* @nr_pages: number of pages from start to pin
|
||||
* @write: whether pages will be written to
|
||||
* @gup_flags: flags modifying pin behaviour
|
||||
* @pages: array that receives pointers to the pages pinned.
|
||||
* Should be at least nr_pages long.
|
||||
*
|
||||
|
@ -2074,8 +2074,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
* requested. If nr_pages is 0 or negative, returns 0. If no pages
|
||||
* were pinned, returns -errno.
|
||||
*/
|
||||
int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
||||
struct page **pages)
|
||||
int get_user_pages_fast(unsigned long start, int nr_pages,
|
||||
unsigned int gup_flags, struct page **pages)
|
||||
{
|
||||
unsigned long addr, len, end;
|
||||
int nr = 0, ret = 0;
|
||||
|
@ -2093,7 +2093,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
|
||||
if (gup_fast_permitted(start, nr_pages)) {
|
||||
local_irq_disable();
|
||||
gup_pgd_range(addr, end, write ? FOLL_WRITE : 0, pages, &nr);
|
||||
gup_pgd_range(addr, end, gup_flags, pages, &nr);
|
||||
local_irq_enable();
|
||||
ret = nr;
|
||||
}
|
||||
|
@ -2104,7 +2104,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
pages += nr;
|
||||
|
||||
ret = get_user_pages_unlocked(start, nr_pages - nr, pages,
|
||||
write ? FOLL_WRITE : 0);
|
||||
gup_flags);
|
||||
|
||||
/* Have to be a bit careful with return values */
|
||||
if (nr > 0) {
|
||||
|
|
|
@ -318,7 +318,7 @@ EXPORT_SYMBOL_GPL(__get_user_pages_fast);
|
|||
* get_user_pages_fast() - pin user pages in memory
|
||||
* @start: starting user address
|
||||
* @nr_pages: number of pages from start to pin
|
||||
* @write: whether pages will be written to
|
||||
* @gup_flags: flags modifying pin behaviour
|
||||
* @pages: array that receives pointers to the pages pinned.
|
||||
* Should be at least nr_pages long.
|
||||
*
|
||||
|
@ -339,10 +339,10 @@ EXPORT_SYMBOL_GPL(__get_user_pages_fast);
|
|||
* were pinned, returns -errno.
|
||||
*/
|
||||
int __weak get_user_pages_fast(unsigned long start,
|
||||
int nr_pages, int write, struct page **pages)
|
||||
int nr_pages, unsigned int gup_flags,
|
||||
struct page **pages)
|
||||
{
|
||||
return get_user_pages_unlocked(start, nr_pages, pages,
|
||||
write ? FOLL_WRITE : 0);
|
||||
return get_user_pages_unlocked(start, nr_pages, pages, gup_flags);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(get_user_pages_fast);
|
||||
|
||||
|
|
|
@ -27,7 +27,7 @@ struct page **ceph_get_direct_page_vector(const void __user *data,
|
|||
while (got < num_pages) {
|
||||
rc = get_user_pages_fast(
|
||||
(unsigned long)data + ((unsigned long)got * PAGE_SIZE),
|
||||
num_pages - got, write_page, pages + got);
|
||||
num_pages - got, write_page ? FOLL_WRITE : 0, pages + got);
|
||||
if (rc < 0)
|
||||
break;
|
||||
BUG_ON(rc == 0);
|
||||
|
|
|
@ -193,7 +193,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, char __user *optval,
|
|||
ret = -ENOMEM;
|
||||
goto out;
|
||||
}
|
||||
ret = get_user_pages_fast(start, nr_pages, 1, pages);
|
||||
ret = get_user_pages_fast(start, nr_pages, FOLL_WRITE, pages);
|
||||
if (ret != nr_pages) {
|
||||
if (ret > 0)
|
||||
nr_pages = ret;
|
||||
|
|
|
@ -158,7 +158,8 @@ static int rds_pin_pages(unsigned long user_addr, unsigned int nr_pages,
|
|||
{
|
||||
int ret;
|
||||
|
||||
ret = get_user_pages_fast(user_addr, nr_pages, write, pages);
|
||||
ret = get_user_pages_fast(user_addr, nr_pages, write ? FOLL_WRITE : 0,
|
||||
pages);
|
||||
|
||||
if (ret >= 0 && ret < nr_pages) {
|
||||
while (ret--)
|
||||
|
|
Loading…
Reference in a new issue