![Michal Hocko](/assets/img/avatar_default.png)
throttle_vm_writeout() was introduced back in 2005 to fix OOMs caused by excessive pageout activity during the reclaim. Too many pages could be put under writeback therefore LRUs would be full of unreclaimable pages until the IO completes and in turn the OOM killer could be invoked. There have been some important changes introduced since then in the reclaim path though. Writers are throttled by balance_dirty_pages when initiating the buffered IO and later during the memory pressure, the direct reclaim is throttled by wait_iff_congested if the node is considered congested by dirty pages on LRUs and the underlying bdi is congested by the queued IO. The kswapd is throttled as well if it encounters pages marked for immediate reclaim or under writeback which signals that that there are too many pages under writeback already. Finally should_reclaim_retry does congestion_wait if the reclaim cannot make any progress and there are too many dirty/writeback pages. Another important aspect is that we do not issue any IO from the direct reclaim context anymore. In a heavy parallel load this could queue a lot of IO which would be very scattered and thus unefficient which would just make the problem worse. This three mechanisms should throttle and keep the amount of IO in a steady state even under heavy IO and memory pressure so yet another throttling point doesn't really seem helpful. Quite contrary, Mikulas Patocka has reported that swap backed by dm-crypt doesn't work properly because the swapout IO cannot make sufficient progress as the writeout path depends on dm_crypt worker which has to allocate memory to perform the encryption. In order to guarantee a forward progress it relies on the mempool allocator. mempool_alloc(), however, prefers to use the underlying (usually page) allocator before it grabs objects from the pool. Such an allocation can dive into the memory reclaim and consequently to throttle_vm_writeout. If there are too many dirty or pages under writeback it will get throttled even though it is in fact a flusher to clear pending pages. kworker/u4:0 D ffff88003df7f438 10488 6 2 0x00000000 Workqueue: kcryptd kcryptd_crypt [dm_crypt] Call Trace: schedule+0x3c/0x90 schedule_timeout+0x1d8/0x360 io_schedule_timeout+0xa4/0x110 congestion_wait+0x86/0x1f0 throttle_vm_writeout+0x44/0xd0 shrink_zone_memcg+0x613/0x720 shrink_zone+0xe0/0x300 do_try_to_free_pages+0x1ad/0x450 try_to_free_pages+0xef/0x300 __alloc_pages_nodemask+0x879/0x1210 alloc_pages_current+0xa1/0x1f0 new_slab+0x2d7/0x6a0 ___slab_alloc+0x3fb/0x5c0 __slab_alloc+0x51/0x90 kmem_cache_alloc+0x27b/0x310 mempool_alloc_slab+0x1d/0x30 mempool_alloc+0x91/0x230 bio_alloc_bioset+0xbd/0x260 kcryptd_crypt+0x114/0x3b0 [dm_crypt] Let's just drop throttle_vm_writeout altogether. It is not very much helpful anymore. I have tried to test a potential writeback IO runaway similar to the one described in the original patch which has introduced that [1]. Small virtual machine (512MB RAM, 4 CPUs, 2G of swap space and disk image on a rather slow NFS in a sync mode on the host) with 8 parallel writers each writing 1G worth of data. As soon as the pagecache fills up and the direct reclaim hits then I start anon memory consumer in a loop (allocating 300M and exiting after populating it) in the background to make the memory pressure even stronger as well as to disrupt the steady state for the IO. The direct reclaim is throttled because of the congestion as well as kswapd hitting congestion_wait due to nr_immediate but throttle_vm_writeout doesn't ever trigger the sleep throughout the test. Dirty+writeback are close to nr_dirty_threshold with some fluctuations caused by the anon consumer. [1] https://www2.kernel.org/pub/linux/kernel/people/akpm/patches/2.6/2.6.9-rc1/2.6.9-rc1-mm3/broken-out/vm-pageout-throttling.patch Link: http://lkml.kernel.org/r/1471171473-21418-1-git-send-email-mhocko@kernel.org Signed-off-by: Michal Hocko <mhocko@suse.com> Reported-by: Mikulas Patocka <mpatocka@redhat.com> Cc: Marcelo Tosatti <mtosatti@redhat.com> Cc: NeilBrown <neilb@suse.com> Cc: Ondrej Kozina <okozina@redhat.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
390 lines
12 KiB
C
390 lines
12 KiB
C
/*
|
|
* include/linux/writeback.h
|
|
*/
|
|
#ifndef WRITEBACK_H
|
|
#define WRITEBACK_H
|
|
|
|
#include <linux/sched.h>
|
|
#include <linux/workqueue.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/flex_proportions.h>
|
|
#include <linux/backing-dev-defs.h>
|
|
|
|
DECLARE_PER_CPU(int, dirty_throttle_leaks);
|
|
|
|
/*
|
|
* The 1/4 region under the global dirty thresh is for smooth dirty throttling:
|
|
*
|
|
* (thresh - thresh/DIRTY_FULL_SCOPE, thresh)
|
|
*
|
|
* Further beyond, all dirtier tasks will enter a loop waiting (possibly long
|
|
* time) for the dirty pages to drop, unless written enough pages.
|
|
*
|
|
* The global dirty threshold is normally equal to the global dirty limit,
|
|
* except when the system suddenly allocates a lot of anonymous memory and
|
|
* knocks down the global dirty threshold quickly, in which case the global
|
|
* dirty limit will follow down slowly to prevent livelocking all dirtier tasks.
|
|
*/
|
|
#define DIRTY_SCOPE 8
|
|
#define DIRTY_FULL_SCOPE (DIRTY_SCOPE / 2)
|
|
|
|
struct backing_dev_info;
|
|
|
|
/*
|
|
* fs/fs-writeback.c
|
|
*/
|
|
enum writeback_sync_modes {
|
|
WB_SYNC_NONE, /* Don't wait on anything */
|
|
WB_SYNC_ALL, /* Wait on every mapping */
|
|
};
|
|
|
|
/*
|
|
* why some writeback work was initiated
|
|
*/
|
|
enum wb_reason {
|
|
WB_REASON_BACKGROUND,
|
|
WB_REASON_TRY_TO_FREE_PAGES,
|
|
WB_REASON_SYNC,
|
|
WB_REASON_PERIODIC,
|
|
WB_REASON_LAPTOP_TIMER,
|
|
WB_REASON_FREE_MORE_MEM,
|
|
WB_REASON_FS_FREE_SPACE,
|
|
/*
|
|
* There is no bdi forker thread any more and works are done
|
|
* by emergency worker, however, this is TPs userland visible
|
|
* and we'll be exposing exactly the same information,
|
|
* so it has a mismatch name.
|
|
*/
|
|
WB_REASON_FORKER_THREAD,
|
|
|
|
WB_REASON_MAX,
|
|
};
|
|
|
|
/*
|
|
* A control structure which tells the writeback code what to do. These are
|
|
* always on the stack, and hence need no locking. They are always initialised
|
|
* in a manner such that unspecified fields are set to zero.
|
|
*/
|
|
struct writeback_control {
|
|
long nr_to_write; /* Write this many pages, and decrement
|
|
this for each page written */
|
|
long pages_skipped; /* Pages which were not written */
|
|
|
|
/*
|
|
* For a_ops->writepages(): if start or end are non-zero then this is
|
|
* a hint that the filesystem need only write out the pages inside that
|
|
* byterange. The byte at `end' is included in the writeout request.
|
|
*/
|
|
loff_t range_start;
|
|
loff_t range_end;
|
|
|
|
enum writeback_sync_modes sync_mode;
|
|
|
|
unsigned for_kupdate:1; /* A kupdate writeback */
|
|
unsigned for_background:1; /* A background writeback */
|
|
unsigned tagged_writepages:1; /* tag-and-write to avoid livelock */
|
|
unsigned for_reclaim:1; /* Invoked from the page allocator */
|
|
unsigned range_cyclic:1; /* range_start is cyclic */
|
|
unsigned for_sync:1; /* sync(2) WB_SYNC_ALL writeback */
|
|
#ifdef CONFIG_CGROUP_WRITEBACK
|
|
struct bdi_writeback *wb; /* wb this writeback is issued under */
|
|
struct inode *inode; /* inode being written out */
|
|
|
|
/* foreign inode detection, see wbc_detach_inode() */
|
|
int wb_id; /* current wb id */
|
|
int wb_lcand_id; /* last foreign candidate wb id */
|
|
int wb_tcand_id; /* this foreign candidate wb id */
|
|
size_t wb_bytes; /* bytes written by current wb */
|
|
size_t wb_lcand_bytes; /* bytes written by last candidate */
|
|
size_t wb_tcand_bytes; /* bytes written by this candidate */
|
|
#endif
|
|
};
|
|
|
|
/*
|
|
* A wb_domain represents a domain that wb's (bdi_writeback's) belong to
|
|
* and are measured against each other in. There always is one global
|
|
* domain, global_wb_domain, that every wb in the system is a member of.
|
|
* This allows measuring the relative bandwidth of each wb to distribute
|
|
* dirtyable memory accordingly.
|
|
*/
|
|
struct wb_domain {
|
|
spinlock_t lock;
|
|
|
|
/*
|
|
* Scale the writeback cache size proportional to the relative
|
|
* writeout speed.
|
|
*
|
|
* We do this by keeping a floating proportion between BDIs, based
|
|
* on page writeback completions [end_page_writeback()]. Those
|
|
* devices that write out pages fastest will get the larger share,
|
|
* while the slower will get a smaller share.
|
|
*
|
|
* We use page writeout completions because we are interested in
|
|
* getting rid of dirty pages. Having them written out is the
|
|
* primary goal.
|
|
*
|
|
* We introduce a concept of time, a period over which we measure
|
|
* these events, because demand can/will vary over time. The length
|
|
* of this period itself is measured in page writeback completions.
|
|
*/
|
|
struct fprop_global completions;
|
|
struct timer_list period_timer; /* timer for aging of completions */
|
|
unsigned long period_time;
|
|
|
|
/*
|
|
* The dirtyable memory and dirty threshold could be suddenly
|
|
* knocked down by a large amount (eg. on the startup of KVM in a
|
|
* swapless system). This may throw the system into deep dirty
|
|
* exceeded state and throttle heavy/light dirtiers alike. To
|
|
* retain good responsiveness, maintain global_dirty_limit for
|
|
* tracking slowly down to the knocked down dirty threshold.
|
|
*
|
|
* Both fields are protected by ->lock.
|
|
*/
|
|
unsigned long dirty_limit_tstamp;
|
|
unsigned long dirty_limit;
|
|
};
|
|
|
|
/**
|
|
* wb_domain_size_changed - memory available to a wb_domain has changed
|
|
* @dom: wb_domain of interest
|
|
*
|
|
* This function should be called when the amount of memory available to
|
|
* @dom has changed. It resets @dom's dirty limit parameters to prevent
|
|
* the past values which don't match the current configuration from skewing
|
|
* dirty throttling. Without this, when memory size of a wb_domain is
|
|
* greatly reduced, the dirty throttling logic may allow too many pages to
|
|
* be dirtied leading to consecutive unnecessary OOMs and may get stuck in
|
|
* that situation.
|
|
*/
|
|
static inline void wb_domain_size_changed(struct wb_domain *dom)
|
|
{
|
|
spin_lock(&dom->lock);
|
|
dom->dirty_limit_tstamp = jiffies;
|
|
dom->dirty_limit = 0;
|
|
spin_unlock(&dom->lock);
|
|
}
|
|
|
|
/*
|
|
* fs/fs-writeback.c
|
|
*/
|
|
struct bdi_writeback;
|
|
void writeback_inodes_sb(struct super_block *, enum wb_reason reason);
|
|
void writeback_inodes_sb_nr(struct super_block *, unsigned long nr,
|
|
enum wb_reason reason);
|
|
bool try_to_writeback_inodes_sb(struct super_block *, enum wb_reason reason);
|
|
bool try_to_writeback_inodes_sb_nr(struct super_block *, unsigned long nr,
|
|
enum wb_reason reason);
|
|
void sync_inodes_sb(struct super_block *);
|
|
void wakeup_flusher_threads(long nr_pages, enum wb_reason reason);
|
|
void inode_wait_for_writeback(struct inode *inode);
|
|
|
|
/* writeback.h requires fs.h; it, too, is not included from here. */
|
|
static inline void wait_on_inode(struct inode *inode)
|
|
{
|
|
might_sleep();
|
|
wait_on_bit(&inode->i_state, __I_NEW, TASK_UNINTERRUPTIBLE);
|
|
}
|
|
|
|
#ifdef CONFIG_CGROUP_WRITEBACK
|
|
|
|
#include <linux/cgroup.h>
|
|
#include <linux/bio.h>
|
|
|
|
void __inode_attach_wb(struct inode *inode, struct page *page);
|
|
void wbc_attach_and_unlock_inode(struct writeback_control *wbc,
|
|
struct inode *inode)
|
|
__releases(&inode->i_lock);
|
|
void wbc_detach_inode(struct writeback_control *wbc);
|
|
void wbc_account_io(struct writeback_control *wbc, struct page *page,
|
|
size_t bytes);
|
|
void cgroup_writeback_umount(void);
|
|
|
|
/**
|
|
* inode_attach_wb - associate an inode with its wb
|
|
* @inode: inode of interest
|
|
* @page: page being dirtied (may be NULL)
|
|
*
|
|
* If @inode doesn't have its wb, associate it with the wb matching the
|
|
* memcg of @page or, if @page is NULL, %current. May be called w/ or w/o
|
|
* @inode->i_lock.
|
|
*/
|
|
static inline void inode_attach_wb(struct inode *inode, struct page *page)
|
|
{
|
|
if (!inode->i_wb)
|
|
__inode_attach_wb(inode, page);
|
|
}
|
|
|
|
/**
|
|
* inode_detach_wb - disassociate an inode from its wb
|
|
* @inode: inode of interest
|
|
*
|
|
* @inode is being freed. Detach from its wb.
|
|
*/
|
|
static inline void inode_detach_wb(struct inode *inode)
|
|
{
|
|
if (inode->i_wb) {
|
|
wb_put(inode->i_wb);
|
|
inode->i_wb = NULL;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* wbc_attach_fdatawrite_inode - associate wbc and inode for fdatawrite
|
|
* @wbc: writeback_control of interest
|
|
* @inode: target inode
|
|
*
|
|
* This function is to be used by __filemap_fdatawrite_range(), which is an
|
|
* alternative entry point into writeback code, and first ensures @inode is
|
|
* associated with a bdi_writeback and attaches it to @wbc.
|
|
*/
|
|
static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc,
|
|
struct inode *inode)
|
|
{
|
|
spin_lock(&inode->i_lock);
|
|
inode_attach_wb(inode, NULL);
|
|
wbc_attach_and_unlock_inode(wbc, inode);
|
|
}
|
|
|
|
/**
|
|
* wbc_init_bio - writeback specific initializtion of bio
|
|
* @wbc: writeback_control for the writeback in progress
|
|
* @bio: bio to be initialized
|
|
*
|
|
* @bio is a part of the writeback in progress controlled by @wbc. Perform
|
|
* writeback specific initialization. This is used to apply the cgroup
|
|
* writeback context.
|
|
*/
|
|
static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio)
|
|
{
|
|
/*
|
|
* pageout() path doesn't attach @wbc to the inode being written
|
|
* out. This is intentional as we don't want the function to block
|
|
* behind a slow cgroup. Ultimately, we want pageout() to kick off
|
|
* regular writeback instead of writing things out itself.
|
|
*/
|
|
if (wbc->wb)
|
|
bio_associate_blkcg(bio, wbc->wb->blkcg_css);
|
|
}
|
|
|
|
#else /* CONFIG_CGROUP_WRITEBACK */
|
|
|
|
static inline void inode_attach_wb(struct inode *inode, struct page *page)
|
|
{
|
|
}
|
|
|
|
static inline void inode_detach_wb(struct inode *inode)
|
|
{
|
|
}
|
|
|
|
static inline void wbc_attach_and_unlock_inode(struct writeback_control *wbc,
|
|
struct inode *inode)
|
|
__releases(&inode->i_lock)
|
|
{
|
|
spin_unlock(&inode->i_lock);
|
|
}
|
|
|
|
static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc,
|
|
struct inode *inode)
|
|
{
|
|
}
|
|
|
|
static inline void wbc_detach_inode(struct writeback_control *wbc)
|
|
{
|
|
}
|
|
|
|
static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio)
|
|
{
|
|
}
|
|
|
|
static inline void wbc_account_io(struct writeback_control *wbc,
|
|
struct page *page, size_t bytes)
|
|
{
|
|
}
|
|
|
|
static inline void cgroup_writeback_umount(void)
|
|
{
|
|
}
|
|
|
|
#endif /* CONFIG_CGROUP_WRITEBACK */
|
|
|
|
/*
|
|
* mm/page-writeback.c
|
|
*/
|
|
#ifdef CONFIG_BLOCK
|
|
void laptop_io_completion(struct backing_dev_info *info);
|
|
void laptop_sync_completion(void);
|
|
void laptop_mode_sync(struct work_struct *work);
|
|
void laptop_mode_timer_fn(unsigned long data);
|
|
#else
|
|
static inline void laptop_sync_completion(void) { }
|
|
#endif
|
|
bool node_dirty_ok(struct pglist_data *pgdat);
|
|
int wb_domain_init(struct wb_domain *dom, gfp_t gfp);
|
|
#ifdef CONFIG_CGROUP_WRITEBACK
|
|
void wb_domain_exit(struct wb_domain *dom);
|
|
#endif
|
|
|
|
extern struct wb_domain global_wb_domain;
|
|
|
|
/* These are exported to sysctl. */
|
|
extern int dirty_background_ratio;
|
|
extern unsigned long dirty_background_bytes;
|
|
extern int vm_dirty_ratio;
|
|
extern unsigned long vm_dirty_bytes;
|
|
extern unsigned int dirty_writeback_interval;
|
|
extern unsigned int dirty_expire_interval;
|
|
extern unsigned int dirtytime_expire_interval;
|
|
extern int vm_highmem_is_dirtyable;
|
|
extern int block_dump;
|
|
extern int laptop_mode;
|
|
|
|
extern int dirty_background_ratio_handler(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *lenp,
|
|
loff_t *ppos);
|
|
extern int dirty_background_bytes_handler(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *lenp,
|
|
loff_t *ppos);
|
|
extern int dirty_ratio_handler(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *lenp,
|
|
loff_t *ppos);
|
|
extern int dirty_bytes_handler(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *lenp,
|
|
loff_t *ppos);
|
|
int dirtytime_interval_handler(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *lenp, loff_t *ppos);
|
|
|
|
struct ctl_table;
|
|
int dirty_writeback_centisecs_handler(struct ctl_table *, int,
|
|
void __user *, size_t *, loff_t *);
|
|
|
|
void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty);
|
|
unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh);
|
|
|
|
void wb_update_bandwidth(struct bdi_writeback *wb, unsigned long start_time);
|
|
void page_writeback_init(void);
|
|
void balance_dirty_pages_ratelimited(struct address_space *mapping);
|
|
bool wb_over_bg_thresh(struct bdi_writeback *wb);
|
|
|
|
typedef int (*writepage_t)(struct page *page, struct writeback_control *wbc,
|
|
void *data);
|
|
|
|
int generic_writepages(struct address_space *mapping,
|
|
struct writeback_control *wbc);
|
|
void tag_pages_for_writeback(struct address_space *mapping,
|
|
pgoff_t start, pgoff_t end);
|
|
int write_cache_pages(struct address_space *mapping,
|
|
struct writeback_control *wbc, writepage_t writepage,
|
|
void *data);
|
|
int do_writepages(struct address_space *mapping, struct writeback_control *wbc);
|
|
void writeback_set_ratelimit(void);
|
|
void tag_pages_for_writeback(struct address_space *mapping,
|
|
pgoff_t start, pgoff_t end);
|
|
|
|
void account_page_redirty(struct page *page);
|
|
|
|
void sb_mark_inode_writeback(struct inode *inode);
|
|
void sb_clear_inode_writeback(struct inode *inode);
|
|
|
|
#endif /* WRITEBACK_H */
|