vfio/type1: Remove locked page accounting workqueue
commit 0cfef2b741
upstream.
If the mmap_sem is contented then the vfio type1 IOMMU backend will
defer locked page accounting updates to a workqueue task. This has a
few problems and depending on which side the user tries to play, they
might be over-penalized for unmaps that haven't yet been accounted or
race the workqueue to enter more mappings than they're allowed. The
original intent of this workqueue mechanism seems to be focused on
reducing latency through the ioctl, but we cannot do so at the cost
of correctness. Remove this workqueue mechanism and update the
callers to allow for failure. We can also now recheck the limit under
write lock to make sure we don't exceed it.
vfio_pin_pages_remote() also now necessarily includes an unwind path
which we can jump to directly if the consecutive page pinning finds
that we're exceeding the user's memory limits. This avoids the
current lazy approach which does accounting and mapping up to the
fault, only to return an error on the next iteration to unwind the
entire vfio_dma.
Reviewed-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
parent
1773131ec4
commit
9f43f70dcc
|
@ -130,57 +130,36 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)
|
||||||
rb_erase(&old->node, &iommu->dma_list);
|
rb_erase(&old->node, &iommu->dma_list);
|
||||||
}
|
}
|
||||||
|
|
||||||
struct vwork {
|
static int vfio_lock_acct(long npage, bool *lock_cap)
|
||||||
struct mm_struct *mm;
|
|
||||||
long npage;
|
|
||||||
struct work_struct work;
|
|
||||||
};
|
|
||||||
|
|
||||||
/* delayed decrement/increment for locked_vm */
|
|
||||||
static void vfio_lock_acct_bg(struct work_struct *work)
|
|
||||||
{
|
{
|
||||||
struct vwork *vwork = container_of(work, struct vwork, work);
|
int ret;
|
||||||
struct mm_struct *mm;
|
|
||||||
|
|
||||||
mm = vwork->mm;
|
if (!npage)
|
||||||
down_write(&mm->mmap_sem);
|
return 0;
|
||||||
mm->locked_vm += vwork->npage;
|
|
||||||
up_write(&mm->mmap_sem);
|
|
||||||
mmput(mm);
|
|
||||||
kfree(vwork);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void vfio_lock_acct(long npage)
|
if (!current->mm)
|
||||||
{
|
return -ESRCH; /* process exited */
|
||||||
struct vwork *vwork;
|
|
||||||
struct mm_struct *mm;
|
|
||||||
|
|
||||||
if (!current->mm || !npage)
|
ret = down_write_killable(¤t->mm->mmap_sem);
|
||||||
return; /* process exited or nothing to do */
|
if (!ret) {
|
||||||
|
if (npage > 0) {
|
||||||
|
if (lock_cap ? !*lock_cap : !capable(CAP_IPC_LOCK)) {
|
||||||
|
unsigned long limit;
|
||||||
|
|
||||||
|
limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
||||||
|
|
||||||
|
if (current->mm->locked_vm + npage > limit)
|
||||||
|
ret = -ENOMEM;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!ret)
|
||||||
|
current->mm->locked_vm += npage;
|
||||||
|
|
||||||
if (down_write_trylock(¤t->mm->mmap_sem)) {
|
|
||||||
current->mm->locked_vm += npage;
|
|
||||||
up_write(¤t->mm->mmap_sem);
|
up_write(¤t->mm->mmap_sem);
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
return ret;
|
||||||
* Couldn't get mmap_sem lock, so must setup to update
|
|
||||||
* mm->locked_vm later. If locked_vm were atomic, we
|
|
||||||
* wouldn't need this silliness
|
|
||||||
*/
|
|
||||||
vwork = kmalloc(sizeof(struct vwork), GFP_KERNEL);
|
|
||||||
if (!vwork)
|
|
||||||
return;
|
|
||||||
mm = get_task_mm(current);
|
|
||||||
if (!mm) {
|
|
||||||
kfree(vwork);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
INIT_WORK(&vwork->work, vfio_lock_acct_bg);
|
|
||||||
vwork->mm = mm;
|
|
||||||
vwork->npage = npage;
|
|
||||||
schedule_work(&vwork->work);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -262,9 +241,9 @@ static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
|
||||||
static long vfio_pin_pages(unsigned long vaddr, long npage,
|
static long vfio_pin_pages(unsigned long vaddr, long npage,
|
||||||
int prot, unsigned long *pfn_base)
|
int prot, unsigned long *pfn_base)
|
||||||
{
|
{
|
||||||
unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
unsigned long pfn = 0, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
||||||
bool lock_cap = capable(CAP_IPC_LOCK);
|
bool lock_cap = capable(CAP_IPC_LOCK);
|
||||||
long ret, i;
|
long ret, i = 1;
|
||||||
bool rsvd;
|
bool rsvd;
|
||||||
|
|
||||||
if (!current->mm)
|
if (!current->mm)
|
||||||
|
@ -283,16 +262,11 @@ static long vfio_pin_pages(unsigned long vaddr, long npage,
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (unlikely(disable_hugepages)) {
|
if (unlikely(disable_hugepages))
|
||||||
if (!rsvd)
|
goto out;
|
||||||
vfio_lock_acct(1);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Lock all the consecutive pages from pfn_base */
|
/* Lock all the consecutive pages from pfn_base */
|
||||||
for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
|
for (vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
|
||||||
unsigned long pfn = 0;
|
|
||||||
|
|
||||||
ret = vaddr_get_pfn(vaddr, prot, &pfn);
|
ret = vaddr_get_pfn(vaddr, prot, &pfn);
|
||||||
if (ret)
|
if (ret)
|
||||||
break;
|
break;
|
||||||
|
@ -308,12 +282,24 @@ static long vfio_pin_pages(unsigned long vaddr, long npage,
|
||||||
put_pfn(pfn, prot);
|
put_pfn(pfn, prot);
|
||||||
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
|
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
|
||||||
__func__, limit << PAGE_SHIFT);
|
__func__, limit << PAGE_SHIFT);
|
||||||
break;
|
ret = -ENOMEM;
|
||||||
|
goto unpin_out;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
out:
|
||||||
if (!rsvd)
|
if (!rsvd)
|
||||||
vfio_lock_acct(i);
|
ret = vfio_lock_acct(i, &lock_cap);
|
||||||
|
|
||||||
|
unpin_out:
|
||||||
|
if (ret) {
|
||||||
|
if (!rsvd) {
|
||||||
|
for (pfn = *pfn_base ; i ; pfn++, i--)
|
||||||
|
put_pfn(pfn, prot);
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
return i;
|
return i;
|
||||||
}
|
}
|
||||||
|
@ -328,7 +314,7 @@ static long vfio_unpin_pages(unsigned long pfn, long npage,
|
||||||
unlocked += put_pfn(pfn++, prot);
|
unlocked += put_pfn(pfn++, prot);
|
||||||
|
|
||||||
if (do_accounting)
|
if (do_accounting)
|
||||||
vfio_lock_acct(-unlocked);
|
vfio_lock_acct(-unlocked, NULL);
|
||||||
|
|
||||||
return unlocked;
|
return unlocked;
|
||||||
}
|
}
|
||||||
|
@ -390,7 +376,7 @@ static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
|
||||||
cond_resched();
|
cond_resched();
|
||||||
}
|
}
|
||||||
|
|
||||||
vfio_lock_acct(-unlocked);
|
vfio_lock_acct(-unlocked, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
|
static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
|
||||||
|
|
Loading…
Reference in a new issue