27e1f82731
khugepaged needs exclusive mmap_sem to access page table. When it fails to lock mmap_sem, the page will fault in as pte-mapped THP. As the page is already a THP, khugepaged will not handle this pmd again. This patch enables the khugepaged to retry collapse the page table. struct mm_slot (in khugepaged.c) is extended with an array, containing addresses of pte-mapped THPs. We use array here for simplicity. We can easily replace it with more advanced data structures when needed. In khugepaged_scan_mm_slot(), if the mm contains pte-mapped THP, we try to collapse the page table. Since collapse may happen at an later time, some pages may already fault in. collapse_pte_mapped_thp() is added to properly handle these pages. collapse_pte_mapped_thp() also double checks whether all ptes in this pmd are mapping to the same THP. This is necessary because some subpage of the THP may be replaced, for example by uprobe. In such cases, it is not possible to collapse the pmd. [kirill.shutemov@linux.intel.com: add comments for retract_page_tables()] Link: http://lkml.kernel.org/r/20190816145443.6ard3iilytc6jlgv@box Link: http://lkml.kernel.org/r/20190815164525.1848545-6-songliubraving@fb.com Signed-off-by: Song Liu <songliubraving@fb.com> Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Suggested-by: Johannes Weiner <hannes@cmpxchg.org> Reviewed-by: Oleg Nesterov <oleg@redhat.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
91 lines
2.5 KiB
C
91 lines
2.5 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_KHUGEPAGED_H
|
|
#define _LINUX_KHUGEPAGED_H
|
|
|
|
#include <linux/sched/coredump.h> /* MMF_VM_HUGEPAGE */
|
|
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
extern struct attribute_group khugepaged_attr_group;
|
|
|
|
extern int khugepaged_init(void);
|
|
extern void khugepaged_destroy(void);
|
|
extern int start_stop_khugepaged(void);
|
|
extern int __khugepaged_enter(struct mm_struct *mm);
|
|
extern void __khugepaged_exit(struct mm_struct *mm);
|
|
extern int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
|
|
unsigned long vm_flags);
|
|
#ifdef CONFIG_SHMEM
|
|
extern void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr);
|
|
#else
|
|
static inline void collapse_pte_mapped_thp(struct mm_struct *mm,
|
|
unsigned long addr)
|
|
{
|
|
}
|
|
#endif
|
|
|
|
#define khugepaged_enabled() \
|
|
(transparent_hugepage_flags & \
|
|
((1<<TRANSPARENT_HUGEPAGE_FLAG) | \
|
|
(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)))
|
|
#define khugepaged_always() \
|
|
(transparent_hugepage_flags & \
|
|
(1<<TRANSPARENT_HUGEPAGE_FLAG))
|
|
#define khugepaged_req_madv() \
|
|
(transparent_hugepage_flags & \
|
|
(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG))
|
|
#define khugepaged_defrag() \
|
|
(transparent_hugepage_flags & \
|
|
(1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG))
|
|
|
|
static inline int khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm)
|
|
{
|
|
if (test_bit(MMF_VM_HUGEPAGE, &oldmm->flags))
|
|
return __khugepaged_enter(mm);
|
|
return 0;
|
|
}
|
|
|
|
static inline void khugepaged_exit(struct mm_struct *mm)
|
|
{
|
|
if (test_bit(MMF_VM_HUGEPAGE, &mm->flags))
|
|
__khugepaged_exit(mm);
|
|
}
|
|
|
|
static inline int khugepaged_enter(struct vm_area_struct *vma,
|
|
unsigned long vm_flags)
|
|
{
|
|
if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags))
|
|
if ((khugepaged_always() ||
|
|
(khugepaged_req_madv() && (vm_flags & VM_HUGEPAGE))) &&
|
|
!(vm_flags & VM_NOHUGEPAGE) &&
|
|
!test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
|
|
if (__khugepaged_enter(vma->vm_mm))
|
|
return -ENOMEM;
|
|
return 0;
|
|
}
|
|
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
|
|
static inline int khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm)
|
|
{
|
|
return 0;
|
|
}
|
|
static inline void khugepaged_exit(struct mm_struct *mm)
|
|
{
|
|
}
|
|
static inline int khugepaged_enter(struct vm_area_struct *vma,
|
|
unsigned long vm_flags)
|
|
{
|
|
return 0;
|
|
}
|
|
static inline int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
|
|
unsigned long vm_flags)
|
|
{
|
|
return 0;
|
|
}
|
|
static inline void collapse_pte_mapped_thp(struct mm_struct *mm,
|
|
unsigned long addr)
|
|
{
|
|
}
|
|
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
|
|
|
#endif /* _LINUX_KHUGEPAGED_H */
|