![Andrea Arcangeli](/assets/img/avatar_default.png)
This is purely required because exit_aio() may block and exit_mmap() may
never start, if the oom_reap_task cannot start running on a mm with
mm_users == 0.
At the same time if the OOM reaper doesn't wait at all for the memory of
the current OOM candidate to be freed by exit_mmap->unmap_vmas, it would
generate a spurious OOM kill.
If it wasn't because of the exit_aio or similar blocking functions in
the last mmput, it would be enough to change the oom_reap_task() in the
case it finds mm_users == 0, to wait for a timeout or to wait for
__mmput to set MMF_OOM_SKIP itself, but it's not just exit_mmap the
problem here so the concurrency of exit_mmap and oom_reap_task is
apparently warranted.
It's a non standard runtime, exit_mmap() runs without mmap_sem, and
oom_reap_task runs with the mmap_sem for reading as usual (kind of
MADV_DONTNEED).
The race between the two is solved with a combination of
tsk_is_oom_victim() (serialized by task_lock) and MMF_OOM_SKIP
(serialized by a dummy down_write/up_write cycle on the same lines of
the ksm_exit method).
If the oom_reap_task() may be running concurrently during exit_mmap,
exit_mmap will wait it to finish in down_write (before taking down mm
structures that would make the oom_reap_task fail with use after free).
If exit_mmap comes first, oom_reap_task() will skip the mm if
MMF_OOM_SKIP is already set and in turn all memory is already freed and
furthermore the mm data structures may already have been taken down by
free_pgtables.
[aarcange@redhat.com: incremental one liner]
Link: http://lkml.kernel.org/r/20170726164319.GC29716@redhat.com
[rientjes@google.com: remove unused mmput_async]
Link: http://lkml.kernel.org/r/alpine.DEB.2.10.1708141733130.50317@chino.kir.corp.google.com
[aarcange@redhat.com: microoptimization]
Link: http://lkml.kernel.org/r/20170817171240.GB5066@redhat.com
Link: http://lkml.kernel.org/r/20170726162912.GA29716@redhat.com
Fixes: 26db62f179
("oom: keep mm of the killed task available")
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: David Rientjes <rientjes@google.com>
Reported-by: David Rientjes <rientjes@google.com>
Tested-by: David Rientjes <rientjes@google.com>
Reviewed-by: Michal Hocko <mhocko@suse.com>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
209 lines
5.8 KiB
C
209 lines
5.8 KiB
C
#ifndef _LINUX_SCHED_MM_H
|
|
#define _LINUX_SCHED_MM_H
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/mm_types.h>
|
|
#include <linux/gfp.h>
|
|
|
|
/*
|
|
* Routines for handling mm_structs
|
|
*/
|
|
extern struct mm_struct * mm_alloc(void);
|
|
|
|
/**
|
|
* mmgrab() - Pin a &struct mm_struct.
|
|
* @mm: The &struct mm_struct to pin.
|
|
*
|
|
* Make sure that @mm will not get freed even after the owning task
|
|
* exits. This doesn't guarantee that the associated address space
|
|
* will still exist later on and mmget_not_zero() has to be used before
|
|
* accessing it.
|
|
*
|
|
* This is a preferred way to to pin @mm for a longer/unbounded amount
|
|
* of time.
|
|
*
|
|
* Use mmdrop() to release the reference acquired by mmgrab().
|
|
*
|
|
* See also <Documentation/vm/active_mm.txt> for an in-depth explanation
|
|
* of &mm_struct.mm_count vs &mm_struct.mm_users.
|
|
*/
|
|
static inline void mmgrab(struct mm_struct *mm)
|
|
{
|
|
atomic_inc(&mm->mm_count);
|
|
}
|
|
|
|
/* mmdrop drops the mm and the page tables */
|
|
extern void __mmdrop(struct mm_struct *);
|
|
static inline void mmdrop(struct mm_struct *mm)
|
|
{
|
|
if (unlikely(atomic_dec_and_test(&mm->mm_count)))
|
|
__mmdrop(mm);
|
|
}
|
|
|
|
static inline void mmdrop_async_fn(struct work_struct *work)
|
|
{
|
|
struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work);
|
|
__mmdrop(mm);
|
|
}
|
|
|
|
static inline void mmdrop_async(struct mm_struct *mm)
|
|
{
|
|
if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
|
|
INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
|
|
schedule_work(&mm->async_put_work);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* mmget() - Pin the address space associated with a &struct mm_struct.
|
|
* @mm: The address space to pin.
|
|
*
|
|
* Make sure that the address space of the given &struct mm_struct doesn't
|
|
* go away. This does not protect against parts of the address space being
|
|
* modified or freed, however.
|
|
*
|
|
* Never use this function to pin this address space for an
|
|
* unbounded/indefinite amount of time.
|
|
*
|
|
* Use mmput() to release the reference acquired by mmget().
|
|
*
|
|
* See also <Documentation/vm/active_mm.txt> for an in-depth explanation
|
|
* of &mm_struct.mm_count vs &mm_struct.mm_users.
|
|
*/
|
|
static inline void mmget(struct mm_struct *mm)
|
|
{
|
|
atomic_inc(&mm->mm_users);
|
|
}
|
|
|
|
static inline bool mmget_not_zero(struct mm_struct *mm)
|
|
{
|
|
return atomic_inc_not_zero(&mm->mm_users);
|
|
}
|
|
|
|
/* mmput gets rid of the mappings and all user-space */
|
|
extern void mmput(struct mm_struct *);
|
|
|
|
/* Grab a reference to a task's mm, if it is not already going away */
|
|
extern struct mm_struct *get_task_mm(struct task_struct *task);
|
|
/*
|
|
* Grab a reference to a task's mm, if it is not already going away
|
|
* and ptrace_may_access with the mode parameter passed to it
|
|
* succeeds.
|
|
*/
|
|
extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode);
|
|
/* Remove the current tasks stale references to the old mm_struct */
|
|
extern void mm_release(struct task_struct *, struct mm_struct *);
|
|
|
|
#ifdef CONFIG_MEMCG
|
|
extern void mm_update_next_owner(struct mm_struct *mm);
|
|
#else
|
|
static inline void mm_update_next_owner(struct mm_struct *mm)
|
|
{
|
|
}
|
|
#endif /* CONFIG_MEMCG */
|
|
|
|
#ifdef CONFIG_MMU
|
|
extern void arch_pick_mmap_layout(struct mm_struct *mm);
|
|
extern unsigned long
|
|
arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
|
|
unsigned long, unsigned long);
|
|
extern unsigned long
|
|
arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
|
|
unsigned long len, unsigned long pgoff,
|
|
unsigned long flags);
|
|
#else
|
|
static inline void arch_pick_mmap_layout(struct mm_struct *mm) {}
|
|
#endif
|
|
|
|
static inline bool in_vfork(struct task_struct *tsk)
|
|
{
|
|
bool ret;
|
|
|
|
/*
|
|
* need RCU to access ->real_parent if CLONE_VM was used along with
|
|
* CLONE_PARENT.
|
|
*
|
|
* We check real_parent->mm == tsk->mm because CLONE_VFORK does not
|
|
* imply CLONE_VM
|
|
*
|
|
* CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus
|
|
* ->real_parent is not necessarily the task doing vfork(), so in
|
|
* theory we can't rely on task_lock() if we want to dereference it.
|
|
*
|
|
* And in this case we can't trust the real_parent->mm == tsk->mm
|
|
* check, it can be false negative. But we do not care, if init or
|
|
* another oom-unkillable task does this it should blame itself.
|
|
*/
|
|
rcu_read_lock();
|
|
ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm;
|
|
rcu_read_unlock();
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Applies per-task gfp context to the given allocation flags.
|
|
* PF_MEMALLOC_NOIO implies GFP_NOIO
|
|
* PF_MEMALLOC_NOFS implies GFP_NOFS
|
|
*/
|
|
static inline gfp_t current_gfp_context(gfp_t flags)
|
|
{
|
|
/*
|
|
* NOIO implies both NOIO and NOFS and it is a weaker context
|
|
* so always make sure it makes precendence
|
|
*/
|
|
if (unlikely(current->flags & PF_MEMALLOC_NOIO))
|
|
flags &= ~(__GFP_IO | __GFP_FS);
|
|
else if (unlikely(current->flags & PF_MEMALLOC_NOFS))
|
|
flags &= ~__GFP_FS;
|
|
return flags;
|
|
}
|
|
|
|
#ifdef CONFIG_LOCKDEP
|
|
extern void fs_reclaim_acquire(gfp_t gfp_mask);
|
|
extern void fs_reclaim_release(gfp_t gfp_mask);
|
|
#else
|
|
static inline void fs_reclaim_acquire(gfp_t gfp_mask) { }
|
|
static inline void fs_reclaim_release(gfp_t gfp_mask) { }
|
|
#endif
|
|
|
|
static inline unsigned int memalloc_noio_save(void)
|
|
{
|
|
unsigned int flags = current->flags & PF_MEMALLOC_NOIO;
|
|
current->flags |= PF_MEMALLOC_NOIO;
|
|
return flags;
|
|
}
|
|
|
|
static inline void memalloc_noio_restore(unsigned int flags)
|
|
{
|
|
current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags;
|
|
}
|
|
|
|
static inline unsigned int memalloc_nofs_save(void)
|
|
{
|
|
unsigned int flags = current->flags & PF_MEMALLOC_NOFS;
|
|
current->flags |= PF_MEMALLOC_NOFS;
|
|
return flags;
|
|
}
|
|
|
|
static inline void memalloc_nofs_restore(unsigned int flags)
|
|
{
|
|
current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags;
|
|
}
|
|
|
|
static inline unsigned int memalloc_noreclaim_save(void)
|
|
{
|
|
unsigned int flags = current->flags & PF_MEMALLOC;
|
|
current->flags |= PF_MEMALLOC;
|
|
return flags;
|
|
}
|
|
|
|
static inline void memalloc_noreclaim_restore(unsigned int flags)
|
|
{
|
|
current->flags = (current->flags & ~PF_MEMALLOC) | flags;
|
|
}
|
|
|
|
#endif /* _LINUX_SCHED_MM_H */
|