97d47a65be
Since commit 53853e2d2b ("mm, compaction: defer each zone individually
instead of preferred zone"), compaction is deferred for each zone where
sync direct compaction fails, and reset where it succeeds. However, it
was observed that for DMA zone compaction often appeared to succeed
while subsequent allocation attempt would not, due to different outcome
of watermark check.
In order to properly defer compaction in this zone, the candidate zone
has to be passed back to __alloc_pages_direct_compact() and compaction
deferred in the zone after the allocation attempt fails.
The large source of mismatch between watermark check in compaction and
allocation was the lack of alloc_flags and classzone_idx values in
compaction, which has been fixed in the previous patch. So with this
problem fixed, we can simplify the code by removing the candidate_zone
parameter and deferring in __alloc_pages_direct_compact().
After this patch, the compaction activity during stress-highalloc
benchmark is still somewhat increased, but it's negligible compared to the
increase that occurred without the better watermark checking. This
suggests that it is still possible to apparently succeed in compaction but
fail to allocate, possibly due to parallel allocation activity.
[akpm@linux-foundation.org: fix build]
Suggested-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
154 lines
4.6 KiB
C
154 lines
4.6 KiB
C
#ifndef _LINUX_COMPACTION_H
|
|
#define _LINUX_COMPACTION_H
|
|
|
|
/* Return values for compact_zone() and try_to_compact_pages() */
|
|
/* compaction didn't start as it was deferred due to past failures */
|
|
#define COMPACT_DEFERRED 0
|
|
/* compaction didn't start as it was not possible or direct reclaim was more suitable */
|
|
#define COMPACT_SKIPPED 1
|
|
/* compaction should continue to another pageblock */
|
|
#define COMPACT_CONTINUE 2
|
|
/* direct compaction partially compacted a zone and there are suitable pages */
|
|
#define COMPACT_PARTIAL 3
|
|
/* The full zone was compacted */
|
|
#define COMPACT_COMPLETE 4
|
|
|
|
/* Used to signal whether compaction detected need_sched() or lock contention */
|
|
/* No contention detected */
|
|
#define COMPACT_CONTENDED_NONE 0
|
|
/* Either need_sched() was true or fatal signal pending */
|
|
#define COMPACT_CONTENDED_SCHED 1
|
|
/* Zone lock or lru_lock was contended in async compaction */
|
|
#define COMPACT_CONTENDED_LOCK 2
|
|
|
|
#ifdef CONFIG_COMPACTION
|
|
extern int sysctl_compact_memory;
|
|
extern int sysctl_compaction_handler(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *length, loff_t *ppos);
|
|
extern int sysctl_extfrag_threshold;
|
|
extern int sysctl_extfrag_handler(struct ctl_table *table, int write,
|
|
void __user *buffer, size_t *length, loff_t *ppos);
|
|
|
|
extern int fragmentation_index(struct zone *zone, unsigned int order);
|
|
extern unsigned long try_to_compact_pages(struct zonelist *zonelist,
|
|
int order, gfp_t gfp_mask, nodemask_t *mask,
|
|
enum migrate_mode mode, int *contended,
|
|
int alloc_flags, int classzone_idx);
|
|
extern void compact_pgdat(pg_data_t *pgdat, int order);
|
|
extern void reset_isolation_suitable(pg_data_t *pgdat);
|
|
extern unsigned long compaction_suitable(struct zone *zone, int order,
|
|
int alloc_flags, int classzone_idx);
|
|
|
|
/* Do not skip compaction more than 64 times */
|
|
#define COMPACT_MAX_DEFER_SHIFT 6
|
|
|
|
/*
|
|
* Compaction is deferred when compaction fails to result in a page
|
|
* allocation success. 1 << compact_defer_limit compactions are skipped up
|
|
* to a limit of 1 << COMPACT_MAX_DEFER_SHIFT
|
|
*/
|
|
static inline void defer_compaction(struct zone *zone, int order)
|
|
{
|
|
zone->compact_considered = 0;
|
|
zone->compact_defer_shift++;
|
|
|
|
if (order < zone->compact_order_failed)
|
|
zone->compact_order_failed = order;
|
|
|
|
if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
|
|
zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
|
|
}
|
|
|
|
/* Returns true if compaction should be skipped this time */
|
|
static inline bool compaction_deferred(struct zone *zone, int order)
|
|
{
|
|
unsigned long defer_limit = 1UL << zone->compact_defer_shift;
|
|
|
|
if (order < zone->compact_order_failed)
|
|
return false;
|
|
|
|
/* Avoid possible overflow */
|
|
if (++zone->compact_considered > defer_limit)
|
|
zone->compact_considered = defer_limit;
|
|
|
|
return zone->compact_considered < defer_limit;
|
|
}
|
|
|
|
/*
|
|
* Update defer tracking counters after successful compaction of given order,
|
|
* which means an allocation either succeeded (alloc_success == true) or is
|
|
* expected to succeed.
|
|
*/
|
|
static inline void compaction_defer_reset(struct zone *zone, int order,
|
|
bool alloc_success)
|
|
{
|
|
if (alloc_success) {
|
|
zone->compact_considered = 0;
|
|
zone->compact_defer_shift = 0;
|
|
}
|
|
if (order >= zone->compact_order_failed)
|
|
zone->compact_order_failed = order + 1;
|
|
}
|
|
|
|
/* Returns true if restarting compaction after many failures */
|
|
static inline bool compaction_restarting(struct zone *zone, int order)
|
|
{
|
|
if (order < zone->compact_order_failed)
|
|
return false;
|
|
|
|
return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
|
|
zone->compact_considered >= 1UL << zone->compact_defer_shift;
|
|
}
|
|
|
|
#else
|
|
static inline unsigned long try_to_compact_pages(struct zonelist *zonelist,
|
|
int order, gfp_t gfp_mask, nodemask_t *nodemask,
|
|
enum migrate_mode mode, int *contended,
|
|
int alloc_flags, int classzone_idx)
|
|
{
|
|
return COMPACT_CONTINUE;
|
|
}
|
|
|
|
static inline void compact_pgdat(pg_data_t *pgdat, int order)
|
|
{
|
|
}
|
|
|
|
static inline void reset_isolation_suitable(pg_data_t *pgdat)
|
|
{
|
|
}
|
|
|
|
static inline unsigned long compaction_suitable(struct zone *zone, int order,
|
|
int alloc_flags, int classzone_idx)
|
|
{
|
|
return COMPACT_SKIPPED;
|
|
}
|
|
|
|
static inline void defer_compaction(struct zone *zone, int order)
|
|
{
|
|
}
|
|
|
|
static inline bool compaction_deferred(struct zone *zone, int order)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
#endif /* CONFIG_COMPACTION */
|
|
|
|
#if defined(CONFIG_COMPACTION) && defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
|
|
extern int compaction_register_node(struct node *node);
|
|
extern void compaction_unregister_node(struct node *node);
|
|
|
|
#else
|
|
|
|
static inline int compaction_register_node(struct node *node)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void compaction_unregister_node(struct node *node)
|
|
{
|
|
}
|
|
#endif /* CONFIG_COMPACTION && CONFIG_SYSFS && CONFIG_NUMA */
|
|
|
|
#endif /* _LINUX_COMPACTION_H */
|