thp: fix zap_huge_pmd() for DAX
The original DAX code assumed that pgtable_t was a pointer, which isn't true on all architectures. Restructure the code to not rely on that assumption. [willy@linux.intel.com: further fixes integrated into this patch] Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Signed-off-by: Matthew Wilcox <willy@linux.intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
5b701b846a
commit
da14676900
|
@ -1456,50 +1456,41 @@ out:
|
|||
int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
|
||||
pmd_t *pmd, unsigned long addr)
|
||||
{
|
||||
pmd_t orig_pmd;
|
||||
spinlock_t *ptl;
|
||||
int ret = 0;
|
||||
|
||||
if (__pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
|
||||
pgtable_t pgtable;
|
||||
pmd_t orig_pmd;
|
||||
/*
|
||||
* For architectures like ppc64 we look at deposited pgtable
|
||||
* when calling pmdp_huge_get_and_clear. So do the
|
||||
* pgtable_trans_huge_withdraw after finishing pmdp related
|
||||
* operations.
|
||||
*/
|
||||
orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
|
||||
tlb->fullmm);
|
||||
tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
|
||||
if (vma_is_dax(vma)) {
|
||||
if (is_huge_zero_pmd(orig_pmd)) {
|
||||
pgtable = NULL;
|
||||
} else {
|
||||
spin_unlock(ptl);
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
|
||||
}
|
||||
if (is_huge_zero_pmd(orig_pmd)) {
|
||||
atomic_long_dec(&tlb->mm->nr_ptes);
|
||||
spin_unlock(ptl);
|
||||
if (__pmd_trans_huge_lock(pmd, vma, &ptl) != 1)
|
||||
return 0;
|
||||
/*
|
||||
* For architectures like ppc64 we look at deposited pgtable
|
||||
* when calling pmdp_huge_get_and_clear. So do the
|
||||
* pgtable_trans_huge_withdraw after finishing pmdp related
|
||||
* operations.
|
||||
*/
|
||||
orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
|
||||
tlb->fullmm);
|
||||
tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
|
||||
if (vma_is_dax(vma)) {
|
||||
spin_unlock(ptl);
|
||||
if (is_huge_zero_pmd(orig_pmd))
|
||||
put_huge_zero_page();
|
||||
} else {
|
||||
struct page *page = pmd_page(orig_pmd);
|
||||
page_remove_rmap(page);
|
||||
VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
|
||||
add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
|
||||
VM_BUG_ON_PAGE(!PageHead(page), page);
|
||||
atomic_long_dec(&tlb->mm->nr_ptes);
|
||||
spin_unlock(ptl);
|
||||
tlb_remove_page(tlb, page);
|
||||
}
|
||||
if (pgtable)
|
||||
pte_free(tlb->mm, pgtable);
|
||||
ret = 1;
|
||||
} else if (is_huge_zero_pmd(orig_pmd)) {
|
||||
pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
|
||||
atomic_long_dec(&tlb->mm->nr_ptes);
|
||||
spin_unlock(ptl);
|
||||
put_huge_zero_page();
|
||||
} else {
|
||||
struct page *page = pmd_page(orig_pmd);
|
||||
page_remove_rmap(page);
|
||||
VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
|
||||
add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
|
||||
VM_BUG_ON_PAGE(!PageHead(page), page);
|
||||
pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
|
||||
atomic_long_dec(&tlb->mm->nr_ptes);
|
||||
spin_unlock(ptl);
|
||||
tlb_remove_page(tlb, page);
|
||||
}
|
||||
return ret;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma,
|
||||
|
|
Loading…
Reference in a new issue