2006-06-30 02:55:32 -06:00
|
|
|
#ifndef _LINUX_VMSTAT_H
|
|
|
|
#define _LINUX_VMSTAT_H
|
|
|
|
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/percpu.h>
|
2007-02-10 02:43:03 -07:00
|
|
|
#include <linux/mm.h>
|
2006-06-30 02:55:33 -06:00
|
|
|
#include <linux/mmzone.h>
|
2011-05-26 17:25:24 -06:00
|
|
|
#include <linux/vm_event_item.h>
|
2011-07-26 17:09:06 -06:00
|
|
|
#include <linux/atomic.h>
|
2006-06-30 02:55:32 -06:00
|
|
|
|
2008-07-23 22:27:03 -06:00
|
|
|
extern int sysctl_stat_interval;
|
|
|
|
|
2007-02-10 02:44:41 -07:00
|
|
|
#ifdef CONFIG_VM_EVENT_COUNTERS
|
|
|
|
/*
|
|
|
|
* Light weight per cpu counter implementation.
|
|
|
|
*
|
|
|
|
* Counters should only be incremented and no critical kernel component
|
|
|
|
* should rely on the counter values.
|
|
|
|
*
|
|
|
|
* Counters are handled completely inline. On many platforms the code
|
|
|
|
* generated will simply be the increment of a global address.
|
|
|
|
*/
|
|
|
|
|
2006-06-30 02:55:45 -06:00
|
|
|
struct vm_event_state {
|
|
|
|
unsigned long event[NR_VM_EVENT_ITEMS];
|
2006-06-30 02:55:32 -06:00
|
|
|
};
|
|
|
|
|
2006-06-30 02:55:45 -06:00
|
|
|
DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
|
|
|
|
|
|
|
|
static inline void __count_vm_event(enum vm_event_item item)
|
|
|
|
{
|
2009-10-29 07:34:15 -06:00
|
|
|
__this_cpu_inc(vm_event_states.event[item]);
|
2006-06-30 02:55:45 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void count_vm_event(enum vm_event_item item)
|
|
|
|
{
|
2009-10-29 07:34:15 -06:00
|
|
|
this_cpu_inc(vm_event_states.event[item]);
|
2006-06-30 02:55:45 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void __count_vm_events(enum vm_event_item item, long delta)
|
|
|
|
{
|
2009-10-29 07:34:15 -06:00
|
|
|
__this_cpu_add(vm_event_states.event[item], delta);
|
2006-06-30 02:55:45 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void count_vm_events(enum vm_event_item item, long delta)
|
|
|
|
{
|
2009-10-29 07:34:15 -06:00
|
|
|
this_cpu_add(vm_event_states.event[item], delta);
|
2006-06-30 02:55:45 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
extern void all_vm_events(unsigned long *);
|
2006-12-22 02:08:01 -07:00
|
|
|
#ifdef CONFIG_HOTPLUG
|
2006-06-30 02:55:45 -06:00
|
|
|
extern void vm_events_fold_cpu(int cpu);
|
2006-12-22 02:08:01 -07:00
|
|
|
#else
|
|
|
|
static inline void vm_events_fold_cpu(int cpu)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
#endif
|
2006-06-30 02:55:45 -06:00
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
/* Disable counters */
|
2007-02-10 02:44:41 -07:00
|
|
|
static inline void count_vm_event(enum vm_event_item item)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static inline void count_vm_events(enum vm_event_item item, long delta)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static inline void __count_vm_event(enum vm_event_item item)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static inline void __count_vm_events(enum vm_event_item item, long delta)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static inline void all_vm_events(unsigned long *ret)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
static inline void vm_events_fold_cpu(int cpu)
|
|
|
|
{
|
|
|
|
}
|
2006-06-30 02:55:45 -06:00
|
|
|
|
|
|
|
#endif /* CONFIG_VM_EVENT_COUNTERS */
|
|
|
|
|
2012-11-02 08:52:48 -06:00
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
|
|
#define count_vm_numa_event(x) count_vm_event(x)
|
|
|
|
#define count_vm_numa_events(x, y) count_vm_events(x, y)
|
|
|
|
#else
|
|
|
|
#define count_vm_numa_event(x) do {} while (0)
|
2013-02-22 17:34:29 -07:00
|
|
|
#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
|
2012-11-02 08:52:48 -06:00
|
|
|
#endif /* CONFIG_NUMA_BALANCING */
|
|
|
|
|
2006-06-30 02:55:45 -06:00
|
|
|
#define __count_zone_vm_events(item, zone, delta) \
|
2007-02-10 02:43:10 -07:00
|
|
|
__count_vm_events(item##_NORMAL - ZONE_NORMAL + \
|
|
|
|
zone_idx(zone), delta)
|
2006-06-30 02:55:32 -06:00
|
|
|
|
2006-06-30 02:55:33 -06:00
|
|
|
/*
|
|
|
|
* Zone based page accounting with per cpu differentials.
|
|
|
|
*/
|
|
|
|
extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
|
|
|
|
|
|
|
|
static inline void zone_page_state_add(long x, struct zone *zone,
|
|
|
|
enum zone_stat_item item)
|
|
|
|
{
|
|
|
|
atomic_long_add(x, &zone->vm_stat[item]);
|
|
|
|
atomic_long_add(x, &vm_stat[item]);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned long global_page_state(enum zone_stat_item item)
|
|
|
|
{
|
|
|
|
long x = atomic_long_read(&vm_stat[item]);
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
if (x < 0)
|
|
|
|
x = 0;
|
|
|
|
#endif
|
|
|
|
return x;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned long zone_page_state(struct zone *zone,
|
|
|
|
enum zone_stat_item item)
|
|
|
|
{
|
|
|
|
long x = atomic_long_read(&zone->vm_stat[item]);
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
if (x < 0)
|
|
|
|
x = 0;
|
|
|
|
#endif
|
|
|
|
return x;
|
|
|
|
}
|
|
|
|
|
2010-09-09 17:38:17 -06:00
|
|
|
/*
|
|
|
|
* More accurate version that also considers the currently pending
|
|
|
|
* deltas. For that we need to loop over all cpus to find the current
|
|
|
|
* deltas. There is no synchronization so the result cannot be
|
|
|
|
* exactly accurate either.
|
|
|
|
*/
|
|
|
|
static inline unsigned long zone_page_state_snapshot(struct zone *zone,
|
|
|
|
enum zone_stat_item item)
|
|
|
|
{
|
|
|
|
long x = atomic_long_read(&zone->vm_stat[item]);
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
int cpu;
|
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];
|
|
|
|
|
|
|
|
if (x < 0)
|
|
|
|
x = 0;
|
|
|
|
#endif
|
|
|
|
return x;
|
|
|
|
}
|
|
|
|
|
2009-09-21 18:01:42 -06:00
|
|
|
extern unsigned long global_reclaimable_pages(void);
|
|
|
|
extern unsigned long zone_reclaimable_pages(struct zone *zone);
|
2008-10-18 21:26:32 -06:00
|
|
|
|
2006-06-30 02:55:33 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
/*
|
|
|
|
* Determine the per node value of a stat item. This function
|
|
|
|
* is called frequently in a NUMA machine, so try to be as
|
|
|
|
* frugal as possible.
|
|
|
|
*/
|
|
|
|
static inline unsigned long node_page_state(int node,
|
|
|
|
enum zone_stat_item item)
|
|
|
|
{
|
|
|
|
struct zone *zones = NODE_DATA(node)->node_zones;
|
|
|
|
|
|
|
|
return
|
2007-02-10 02:43:10 -07:00
|
|
|
#ifdef CONFIG_ZONE_DMA
|
|
|
|
zone_page_state(&zones[ZONE_DMA], item) +
|
|
|
|
#endif
|
2006-09-26 00:31:13 -06:00
|
|
|
#ifdef CONFIG_ZONE_DMA32
|
2006-06-30 02:55:33 -06:00
|
|
|
zone_page_state(&zones[ZONE_DMA32], item) +
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_HIGHMEM
|
|
|
|
zone_page_state(&zones[ZONE_HIGHMEM], item) +
|
|
|
|
#endif
|
2007-07-17 05:03:12 -06:00
|
|
|
zone_page_state(&zones[ZONE_NORMAL], item) +
|
|
|
|
zone_page_state(&zones[ZONE_MOVABLE], item);
|
2006-06-30 02:55:33 -06:00
|
|
|
}
|
2006-06-30 02:55:44 -06:00
|
|
|
|
2011-03-22 17:33:12 -06:00
|
|
|
extern void zone_statistics(struct zone *, struct zone *, gfp_t gfp);
|
2006-06-30 02:55:44 -06:00
|
|
|
|
2006-06-30 02:55:33 -06:00
|
|
|
#else
|
2006-06-30 02:55:44 -06:00
|
|
|
|
2006-06-30 02:55:33 -06:00
|
|
|
#define node_page_state(node, item) global_page_state(item)
|
2011-03-22 17:33:12 -06:00
|
|
|
#define zone_statistics(_zl, _z, gfp) do { } while (0)
|
2006-06-30 02:55:44 -06:00
|
|
|
|
|
|
|
#endif /* CONFIG_NUMA */
|
2006-06-30 02:55:33 -06:00
|
|
|
|
|
|
|
#define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
|
|
|
|
#define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
|
|
|
|
|
2006-06-30 02:55:44 -06:00
|
|
|
extern void inc_zone_state(struct zone *, enum zone_stat_item);
|
|
|
|
|
2006-06-30 02:55:33 -06:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int);
|
|
|
|
void __inc_zone_page_state(struct page *, enum zone_stat_item);
|
|
|
|
void __dec_zone_page_state(struct page *, enum zone_stat_item);
|
2006-06-30 02:55:32 -06:00
|
|
|
|
2006-06-30 02:55:33 -06:00
|
|
|
void mod_zone_page_state(struct zone *, enum zone_stat_item, int);
|
|
|
|
void inc_zone_page_state(struct page *, enum zone_stat_item);
|
|
|
|
void dec_zone_page_state(struct page *, enum zone_stat_item);
|
|
|
|
|
|
|
|
extern void inc_zone_state(struct zone *, enum zone_stat_item);
|
2007-02-10 02:43:01 -07:00
|
|
|
extern void __inc_zone_state(struct zone *, enum zone_stat_item);
|
|
|
|
extern void dec_zone_state(struct zone *, enum zone_stat_item);
|
|
|
|
extern void __dec_zone_state(struct zone *, enum zone_stat_item);
|
2006-06-30 02:55:33 -06:00
|
|
|
|
|
|
|
void refresh_cpu_vm_stats(int);
|
2011-05-24 18:11:33 -06:00
|
|
|
void refresh_zone_stat_thresholds(void);
|
2011-01-13 16:45:43 -07:00
|
|
|
|
2012-10-08 17:33:39 -06:00
|
|
|
void drain_zonestat(struct zone *zone, struct per_cpu_pageset *);
|
|
|
|
|
2011-01-13 16:45:43 -07:00
|
|
|
int calculate_pressure_threshold(struct zone *zone);
|
|
|
|
int calculate_normal_threshold(struct zone *zone);
|
|
|
|
void set_pgdat_percpu_threshold(pg_data_t *pgdat,
|
|
|
|
int (*calculate_pressure)(struct zone *));
|
2006-06-30 02:55:33 -06:00
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We do not maintain differentials in a single processor configuration.
|
|
|
|
* The functions directly modify the zone and global counters.
|
|
|
|
*/
|
|
|
|
static inline void __mod_zone_page_state(struct zone *zone,
|
|
|
|
enum zone_stat_item item, int delta)
|
|
|
|
{
|
|
|
|
zone_page_state_add(delta, zone, item);
|
|
|
|
}
|
|
|
|
|
2006-07-10 05:44:30 -06:00
|
|
|
static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
|
|
|
|
{
|
|
|
|
atomic_long_inc(&zone->vm_stat[item]);
|
|
|
|
atomic_long_inc(&vm_stat[item]);
|
|
|
|
}
|
|
|
|
|
2006-06-30 02:55:33 -06:00
|
|
|
static inline void __inc_zone_page_state(struct page *page,
|
|
|
|
enum zone_stat_item item)
|
|
|
|
{
|
2006-07-10 05:44:30 -06:00
|
|
|
__inc_zone_state(page_zone(page), item);
|
2006-06-30 02:55:33 -06:00
|
|
|
}
|
|
|
|
|
2007-02-10 02:43:01 -07:00
|
|
|
static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
|
|
|
|
{
|
|
|
|
atomic_long_dec(&zone->vm_stat[item]);
|
|
|
|
atomic_long_dec(&vm_stat[item]);
|
|
|
|
}
|
|
|
|
|
2006-06-30 02:55:33 -06:00
|
|
|
static inline void __dec_zone_page_state(struct page *page,
|
|
|
|
enum zone_stat_item item)
|
|
|
|
{
|
2008-02-25 08:45:03 -07:00
|
|
|
__dec_zone_state(page_zone(page), item);
|
2006-06-30 02:55:33 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We only use atomic operations to update counters. So there is no need to
|
|
|
|
* disable interrupts.
|
|
|
|
*/
|
|
|
|
#define inc_zone_page_state __inc_zone_page_state
|
|
|
|
#define dec_zone_page_state __dec_zone_page_state
|
|
|
|
#define mod_zone_page_state __mod_zone_page_state
|
|
|
|
|
2011-01-13 16:45:43 -07:00
|
|
|
#define set_pgdat_percpu_threshold(pgdat, callback) { }
|
mm: page allocator: adjust the per-cpu counter threshold when memory is low
Commit aa45484 ("calculate a better estimate of NR_FREE_PAGES when memory
is low") noted that watermarks were based on the vmstat NR_FREE_PAGES. To
avoid synchronization overhead, these counters are maintained on a per-cpu
basis and drained both periodically and when a threshold is above a
threshold. On large CPU systems, the difference between the estimate and
real value of NR_FREE_PAGES can be very high. The system can get into a
case where pages are allocated far below the min watermark potentially
causing livelock issues. The commit solved the problem by taking a better
reading of NR_FREE_PAGES when memory was low.
Unfortately, as reported by Shaohua Li this accurate reading can consume a
large amount of CPU time on systems with many sockets due to cache line
bouncing. This patch takes a different approach. For large machines
where counter drift might be unsafe and while kswapd is awake, the per-cpu
thresholds for the target pgdat are reduced to limit the level of drift to
what should be a safe level. This incurs a performance penalty in heavy
memory pressure by a factor that depends on the workload and the machine
but the machine should function correctly without accidentally exhausting
all memory on a node. There is an additional cost when kswapd wakes and
sleeps but the event is not expected to be frequent - in Shaohua's test
case, there was one recorded sleep and wake event at least.
To ensure that kswapd wakes up, a safe version of zone_watermark_ok() is
introduced that takes a more accurate reading of NR_FREE_PAGES when called
from wakeup_kswapd, when deciding whether it is really safe to go back to
sleep in sleeping_prematurely() and when deciding if a zone is really
balanced or not in balance_pgdat(). We are still using an expensive
function but limiting how often it is called.
When the test case is reproduced, the time spent in the watermark
functions is reduced. The following report is on the percentage of time
spent cumulatively spent in the functions zone_nr_free_pages(),
zone_watermark_ok(), __zone_watermark_ok(), zone_watermark_ok_safe(),
zone_page_state_snapshot(), zone_page_state().
vanilla 11.6615%
disable-threshold 0.2584%
David said:
: We had to pull aa454840 "mm: page allocator: calculate a better estimate
: of NR_FREE_PAGES when memory is low and kswapd is awake" from 2.6.36
: internally because tests showed that it would cause the machine to stall
: as the result of heavy kswapd activity. I merged it back with this fix as
: it is pending in the -mm tree and it solves the issue we were seeing, so I
: definitely think this should be pushed to -stable (and I would seriously
: consider it for 2.6.37 inclusion even at this late date).
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Reported-by: Shaohua Li <shaohua.li@intel.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Tested-by: Nicolas Bareil <nico@chdir.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: <stable@kernel.org> [2.6.37.1, 2.6.36.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 16:45:41 -07:00
|
|
|
|
2006-06-30 02:55:33 -06:00
|
|
|
static inline void refresh_cpu_vm_stats(int cpu) { }
|
2011-05-24 18:11:33 -06:00
|
|
|
static inline void refresh_zone_stat_thresholds(void) { }
|
|
|
|
|
2012-10-08 17:33:39 -06:00
|
|
|
static inline void drain_zonestat(struct zone *zone,
|
|
|
|
struct per_cpu_pageset *pset) { }
|
2011-05-24 18:11:28 -06:00
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
2012-10-08 17:32:02 -06:00
|
|
|
static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
|
|
|
|
int migratetype)
|
|
|
|
{
|
|
|
|
__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
|
|
|
|
if (is_migrate_cma(migratetype))
|
|
|
|
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
|
|
|
|
}
|
|
|
|
|
2011-05-24 18:11:28 -06:00
|
|
|
extern const char * const vmstat_text[];
|
2006-06-30 02:55:33 -06:00
|
|
|
|
|
|
|
#endif /* _LINUX_VMSTAT_H */
|