2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* linux/mm/page_alloc.c
|
|
|
|
*
|
|
|
|
* Manages the free list, the system allocates free pages here.
|
|
|
|
* Note that kmalloc() lives in slab.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
|
|
|
|
* Swap reorganised 29.12.95, Stephen Tweedie
|
|
|
|
* Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
|
|
|
|
* Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
|
|
|
|
* Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
|
|
|
|
* Zone balancing, Kanoj Sarcar, SGI, Jan 2000
|
|
|
|
* Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
|
|
|
|
* (lots of bits borrowed from Ingo Molnar & Andrew Morton)
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/stddef.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/swap.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/pagemap.h>
|
2008-03-04 15:28:32 -07:00
|
|
|
#include <linux/jiffies.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/bootmem.h>
|
2010-08-25 14:39:16 -06:00
|
|
|
#include <linux/memblock.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/compiler.h>
|
2005-09-13 02:25:16 -06:00
|
|
|
#include <linux/kernel.h>
|
2008-11-25 08:55:53 -07:00
|
|
|
#include <linux/kmemcheck.h>
|
2015-02-13 15:39:28 -07:00
|
|
|
#include <linux/kasan.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/suspend.h>
|
|
|
|
#include <linux/pagevec.h>
|
|
|
|
#include <linux/blkdev.h>
|
|
|
|
#include <linux/slab.h>
|
2011-05-24 18:12:16 -06:00
|
|
|
#include <linux/ratelimit.h>
|
2007-10-17 00:25:53 -06:00
|
|
|
#include <linux/oom.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/topology.h>
|
|
|
|
#include <linux/sysctl.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/cpuset.h>
|
2005-10-29 19:16:53 -06:00
|
|
|
#include <linux/memory_hotplug.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/nodemask.h>
|
|
|
|
#include <linux/vmalloc.h>
|
2011-05-24 18:11:33 -06:00
|
|
|
#include <linux/vmstat.h>
|
2006-01-06 01:11:17 -07:00
|
|
|
#include <linux/mempolicy.h>
|
2016-01-15 17:56:22 -07:00
|
|
|
#include <linux/memremap.h>
|
2006-06-23 03:03:11 -06:00
|
|
|
#include <linux/stop_machine.h>
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
#include <linux/sort.h>
|
|
|
|
#include <linux/pfn.h>
|
2006-10-20 00:28:16 -06:00
|
|
|
#include <linux/backing-dev.h>
|
2006-12-08 03:39:45 -07:00
|
|
|
#include <linux/fault-inject.h>
|
2007-10-16 02:26:11 -06:00
|
|
|
#include <linux/page-isolation.h>
|
mm/page_ext: resurrect struct page extending code for debugging
When we debug something, we'd like to insert some information to every
page. For this purpose, we sometimes modify struct page itself. But,
this has drawbacks. First, it requires re-compile. This makes us
hesitate to use the powerful debug feature so development process is
slowed down. And, second, sometimes it is impossible to rebuild the
kernel due to third party module dependency. At third, system behaviour
would be largely different after re-compile, because it changes size of
struct page greatly and this structure is accessed by every part of
kernel. Keeping this as it is would be better to reproduce errornous
situation.
This feature is intended to overcome above mentioned problems. This
feature allocates memory for extended data per page in certain place
rather than the struct page itself. This memory can be accessed by the
accessor functions provided by this code. During the boot process, it
checks whether allocation of huge chunk of memory is needed or not. If
not, it avoids allocating memory at all. With this advantage, we can
include this feature into the kernel in default and can avoid rebuild and
solve related problems.
Until now, memcg uses this technique. But, now, memcg decides to embed
their variable to struct page itself and it's code to extend struct page
has been removed. I'd like to use this code to develop debug feature, so
this patch resurrect it.
To help these things to work well, this patch introduces two callbacks for
clients. One is the need callback which is mandatory if user wants to
avoid useless memory allocation at boot-time. The other is optional, init
callback, which is used to do proper initialization after memory is
allocated. Detailed explanation about purpose of these functions is in
code comment. Please refer it.
Others are completely same with previous extension code in memcg.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:46 -07:00
|
|
|
#include <linux/page_ext.h>
|
2008-04-30 01:55:01 -06:00
|
|
|
#include <linux/debugobjects.h>
|
2009-06-11 06:23:19 -06:00
|
|
|
#include <linux/kmemleak.h>
|
2010-05-24 15:32:30 -06:00
|
|
|
#include <linux/compaction.h>
|
2009-09-21 18:02:44 -06:00
|
|
|
#include <trace/events/kmem.h>
|
2011-05-20 13:50:29 -06:00
|
|
|
#include <linux/prefetch.h>
|
2013-09-11 15:22:36 -06:00
|
|
|
#include <linux/mm_inline.h>
|
2011-12-29 05:09:50 -07:00
|
|
|
#include <linux/migrate.h>
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
#include <linux/page_ext.h>
|
2013-04-29 16:07:48 -06:00
|
|
|
#include <linux/hugetlb.h>
|
2013-02-07 08:47:07 -07:00
|
|
|
#include <linux/sched/rt.h>
|
mm/page_owner: keep track of page owners
This is the page owner tracking code which is introduced so far ago. It
is resident on Andrew's tree, though, nobody tried to upstream so it
remain as is. Our company uses this feature actively to debug memory leak
or to find a memory hogger so I decide to upstream this feature.
This functionality help us to know who allocates the page. When
allocating a page, we store some information about allocation in extra
memory. Later, if we need to know status of all pages, we can get and
analyze it from this stored information.
In previous version of this feature, extra memory is statically defined in
struct page, but, in this version, extra memory is allocated outside of
struct page. It enables us to turn on/off this feature at boottime
without considerable memory waste.
Although we already have tracepoint for tracing page allocation/free,
using it to analyze page owner is rather complex. We need to enlarge the
trace buffer for preventing overlapping until userspace program launched.
And, launched program continually dump out the trace buffer for later
analysis and it would change system behaviour with more possibility rather
than just keeping it in memory, so bad for debug.
Moreover, we can use page_owner feature further for various purposes. For
example, we can use it for fragmentation statistics implemented in this
patch. And, I also plan to implement some CMA failure debugging feature
using this interface.
I'd like to give the credit for all developers contributed this feature,
but, it's not easy because I don't know exact history. Sorry about that.
Below is people who has "Signed-off-by" in the patches in Andrew's tree.
Contributor:
Alexander Nyberg <alexn@dsv.su.se>
Mel Gorman <mgorman@suse.de>
Dave Hansen <dave@linux.vnet.ibm.com>
Minchan Kim <minchan@kernel.org>
Michal Nazarewicz <mina86@mina86.com>
Andrew Morton <akpm@linux-foundation.org>
Jungsoo Son <jungsoo.son@lge.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:56:01 -07:00
|
|
|
#include <linux/page_owner.h>
|
2015-06-30 15:57:27 -06:00
|
|
|
#include <linux/kthread.h>
|
mm: charge/uncharge kmemcg from generic page allocator paths
Currently, to charge a non-slab allocation to kmemcg one has to use
alloc_kmem_pages helper with __GFP_ACCOUNT flag. A page allocated with
this helper should finally be freed using free_kmem_pages, otherwise it
won't be uncharged.
This API suits its current users fine, but it turns out to be impossible
to use along with page reference counting, i.e. when an allocation is
supposed to be freed with put_page, as it is the case with pipe or unix
socket buffers.
To overcome this limitation, this patch moves charging/uncharging to
generic page allocator paths, i.e. to __alloc_pages_nodemask and
free_pages_prepare, and zaps alloc/free_kmem_pages helpers. This way,
one can use any of the available page allocation functions to get the
allocated page charged to kmemcg - it's enough to pass __GFP_ACCOUNT,
just like in case of kmalloc and friends. A charged page will be
automatically uncharged on free.
To make it possible, we need to mark pages charged to kmemcg somehow.
To avoid introducing a new page flag, we make use of page->_mapcount for
marking such pages. Since pages charged to kmemcg are not supposed to
be mapped to userspace, it should work just fine. There are other
(ab)users of page->_mapcount - buddy and balloon pages - but we don't
conflict with them.
In case kmemcg is compiled out or not used at runtime, this patch
introduces no overhead to generic page allocator paths. If kmemcg is
used, it will be plus one gfp flags check on alloc and plus one
page->_mapcount check on free, which shouldn't hurt performance, because
the data accessed are hot.
Link: http://lkml.kernel.org/r/a9736d856f895bcb465d9f257b54efe32eda6f99.1464079538.git.vdavydov@virtuozzo.com
Signed-off-by: Vladimir Davydov <vdavydov@virtuozzo.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-26 16:24:24 -06:00
|
|
|
#include <linux/memcontrol.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-07-03 16:03:41 -06:00
|
|
|
#include <asm/sections.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <asm/tlbflush.h>
|
2006-05-15 10:43:59 -06:00
|
|
|
#include <asm/div64.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include "internal.h"
|
|
|
|
|
2013-07-03 16:01:29 -06:00
|
|
|
/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
|
|
|
|
static DEFINE_MUTEX(pcp_batch_high_lock);
|
2014-06-23 14:22:04 -06:00
|
|
|
#define MIN_PERCPU_PAGELIST_FRACTION (8)
|
2013-07-03 16:01:29 -06:00
|
|
|
|
2010-05-26 15:44:56 -06:00
|
|
|
#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
|
|
|
|
DEFINE_PER_CPU(int, numa_node);
|
|
|
|
EXPORT_PER_CPU_SYMBOL(numa_node);
|
|
|
|
#endif
|
|
|
|
|
2010-05-26 15:45:00 -06:00
|
|
|
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
|
|
|
|
/*
|
|
|
|
* N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
|
|
|
|
* It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
|
|
|
|
* Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
|
|
|
|
* defined in <linux/topology.h>.
|
|
|
|
*/
|
|
|
|
DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
|
|
|
|
EXPORT_PER_CPU_SYMBOL(_numa_mem_);
|
topology: add support for node_to_mem_node() to determine the fallback node
Anton noticed (http://www.spinics.net/lists/linux-mm/msg67489.html) that
on ppc LPARs with memoryless nodes, a large amount of memory was consumed
by slabs and was marked unreclaimable. He tracked it down to slab
deactivations in the SLUB core when we allocate remotely, leading to poor
efficiency always when memoryless nodes are present.
After much discussion, Joonsoo provided a few patches that help
significantly. They don't resolve the problem altogether:
- memory hotplug still needs testing, that is when a memoryless node
becomes memory-ful, we want to dtrt
- there are other reasons for going off-node than memoryless nodes,
e.g., fully exhausted local nodes
Neither case is resolved with this series, but I don't think that should
block their acceptance, as they can be explored/resolved with follow-on
patches.
The series consists of:
[1/3] topology: add support for node_to_mem_node() to determine the
fallback node
[2/3] slub: fallback to node_to_mem_node() node if allocating on
memoryless node
- Joonsoo's patches to cache the nearest node with memory for each
NUMA node
[3/3] Partial revert of 81c98869faa5 (""kthread: ensure locality of
task_struct allocations")
- At Tejun's request, keep the knowledge of memoryless node fallback
to the allocator core.
This patch (of 3):
We need to determine the fallback node in slub allocator if the allocation
target node is memoryless node. Without it, the SLUB wrongly select the
node which has no memory and can't use a partial slab, because of node
mismatch. Introduced function, node_to_mem_node(X), will return a node Y
with memory that has the nearest distance. If X is memoryless node, it
will return nearest distance node, but, if X is normal node, it will
return itself.
We will use this function in following patch to determine the fallback
node.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Nishanth Aravamudan <nacc@linux.vnet.ibm.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Han Pingtian <hanpt@linux.vnet.ibm.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Anton Blanchard <anton@samba.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Wanpeng Li <liwanp@linux.vnet.ibm.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-09 16:26:13 -06:00
|
|
|
int _node_numa_mem_[MAX_NUMNODES];
|
2010-05-26 15:45:00 -06:00
|
|
|
#endif
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
Memoryless nodes: Generic management of nodemasks for various purposes
Why do we need to support memoryless nodes?
KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> wrote:
> For fujitsu, problem is called "empty" node.
>
> When ACPI's SRAT table includes "possible nodes", ia64 bootstrap(acpi_numa_init)
> creates nodes, which includes no memory, no cpu.
>
> I tried to remove empty-node in past, but that was denied.
> It was because we can hot-add cpu to the empty node.
> (node-hotplug triggered by cpu is not implemented now. and it will be ugly.)
>
>
> For HP, (Lee can comment on this later), they have memory-less-node.
> As far as I hear, HP's machine can have following configration.
>
> (example)
> Node0: CPU0 memory AAA MB
> Node1: CPU1 memory AAA MB
> Node2: CPU2 memory AAA MB
> Node3: CPU3 memory AAA MB
> Node4: Memory XXX GB
>
> AAA is very small value (below 16MB) and will be omitted by ia64 bootstrap.
> After boot, only Node 4 has valid memory (but have no cpu.)
>
> Maybe this is memory-interleave by firmware config.
Christoph Lameter <clameter@sgi.com> wrote:
> Future SGI platforms (actually also current one can have but nothing like
> that is deployed to my knowledge) have nodes with only cpus. Current SGI
> platforms have nodes with just I/O that we so far cannot manage in the
> core. So the arch code maps them to the nearest memory node.
Lee Schermerhorn <Lee.Schermerhorn@hp.com> wrote:
> For the HP platforms, we can configure each cell with from 0% to 100%
> "cell local memory". When we configure with <100% CLM, the "missing
> percentages" are interleaved by hardware on a cache-line granularity to
> improve bandwidth at the expense of latency for numa-challenged
> applications [and OSes, but not our problem ;-)]. When we boot Linux on
> such a config, all of the real nodes have no memory--it all resides in a
> single interleaved pseudo-node.
>
> When we boot Linux on a 100% CLM configuration [== NUMA], we still have
> the interleaved pseudo-node. It contains a few hundred MB stolen from
> the real nodes to contain the DMA zone. [Interleaved memory resides at
> phys addr 0]. The memoryless-nodes patches, along with the zoneorder
> patches, support this config as well.
>
> Also, when we boot a NUMA config with the "mem=" command line,
> specifying less memory than actually exists, Linux takes the excluded
> memory "off the top" rather than distributing it across the nodes. This
> can result in memoryless nodes, as well.
>
This patch:
Preparation for memoryless node patches.
Provide a generic way to keep nodemasks describing various characteristics of
NUMA nodes.
Remove the node_online_map and the node_possible map and realize the same
functionality using two nodes stats: N_POSSIBLE and N_ONLINE.
[Lee.Schermerhorn@hp.com: Initialize N_*_MEMORY and N_CPU masks for non-NUMA config]
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Tested-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Acked-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Acked-by: Bob Picco <bob.picco@hp.com>
Cc: Nishanth Aravamudan <nacc@us.ibm.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@skynet.ie>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: "Serge E. Hallyn" <serge@hallyn.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:27 -06:00
|
|
|
* Array of node states.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
Memoryless nodes: Generic management of nodemasks for various purposes
Why do we need to support memoryless nodes?
KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> wrote:
> For fujitsu, problem is called "empty" node.
>
> When ACPI's SRAT table includes "possible nodes", ia64 bootstrap(acpi_numa_init)
> creates nodes, which includes no memory, no cpu.
>
> I tried to remove empty-node in past, but that was denied.
> It was because we can hot-add cpu to the empty node.
> (node-hotplug triggered by cpu is not implemented now. and it will be ugly.)
>
>
> For HP, (Lee can comment on this later), they have memory-less-node.
> As far as I hear, HP's machine can have following configration.
>
> (example)
> Node0: CPU0 memory AAA MB
> Node1: CPU1 memory AAA MB
> Node2: CPU2 memory AAA MB
> Node3: CPU3 memory AAA MB
> Node4: Memory XXX GB
>
> AAA is very small value (below 16MB) and will be omitted by ia64 bootstrap.
> After boot, only Node 4 has valid memory (but have no cpu.)
>
> Maybe this is memory-interleave by firmware config.
Christoph Lameter <clameter@sgi.com> wrote:
> Future SGI platforms (actually also current one can have but nothing like
> that is deployed to my knowledge) have nodes with only cpus. Current SGI
> platforms have nodes with just I/O that we so far cannot manage in the
> core. So the arch code maps them to the nearest memory node.
Lee Schermerhorn <Lee.Schermerhorn@hp.com> wrote:
> For the HP platforms, we can configure each cell with from 0% to 100%
> "cell local memory". When we configure with <100% CLM, the "missing
> percentages" are interleaved by hardware on a cache-line granularity to
> improve bandwidth at the expense of latency for numa-challenged
> applications [and OSes, but not our problem ;-)]. When we boot Linux on
> such a config, all of the real nodes have no memory--it all resides in a
> single interleaved pseudo-node.
>
> When we boot Linux on a 100% CLM configuration [== NUMA], we still have
> the interleaved pseudo-node. It contains a few hundred MB stolen from
> the real nodes to contain the DMA zone. [Interleaved memory resides at
> phys addr 0]. The memoryless-nodes patches, along with the zoneorder
> patches, support this config as well.
>
> Also, when we boot a NUMA config with the "mem=" command line,
> specifying less memory than actually exists, Linux takes the excluded
> memory "off the top" rather than distributing it across the nodes. This
> can result in memoryless nodes, as well.
>
This patch:
Preparation for memoryless node patches.
Provide a generic way to keep nodemasks describing various characteristics of
NUMA nodes.
Remove the node_online_map and the node_possible map and realize the same
functionality using two nodes stats: N_POSSIBLE and N_ONLINE.
[Lee.Schermerhorn@hp.com: Initialize N_*_MEMORY and N_CPU masks for non-NUMA config]
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Tested-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Acked-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Acked-by: Bob Picco <bob.picco@hp.com>
Cc: Nishanth Aravamudan <nacc@us.ibm.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@skynet.ie>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: "Serge E. Hallyn" <serge@hallyn.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:27 -06:00
|
|
|
nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
|
|
|
|
[N_POSSIBLE] = NODE_MASK_ALL,
|
|
|
|
[N_ONLINE] = { { [0] = 1UL } },
|
|
|
|
#ifndef CONFIG_NUMA
|
|
|
|
[N_NORMAL_MEMORY] = { { [0] = 1UL } },
|
|
|
|
#ifdef CONFIG_HIGHMEM
|
|
|
|
[N_HIGH_MEMORY] = { { [0] = 1UL } },
|
2012-12-12 14:52:00 -07:00
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_MOVABLE_NODE
|
|
|
|
[N_MEMORY] = { { [0] = 1UL } },
|
Memoryless nodes: Generic management of nodemasks for various purposes
Why do we need to support memoryless nodes?
KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> wrote:
> For fujitsu, problem is called "empty" node.
>
> When ACPI's SRAT table includes "possible nodes", ia64 bootstrap(acpi_numa_init)
> creates nodes, which includes no memory, no cpu.
>
> I tried to remove empty-node in past, but that was denied.
> It was because we can hot-add cpu to the empty node.
> (node-hotplug triggered by cpu is not implemented now. and it will be ugly.)
>
>
> For HP, (Lee can comment on this later), they have memory-less-node.
> As far as I hear, HP's machine can have following configration.
>
> (example)
> Node0: CPU0 memory AAA MB
> Node1: CPU1 memory AAA MB
> Node2: CPU2 memory AAA MB
> Node3: CPU3 memory AAA MB
> Node4: Memory XXX GB
>
> AAA is very small value (below 16MB) and will be omitted by ia64 bootstrap.
> After boot, only Node 4 has valid memory (but have no cpu.)
>
> Maybe this is memory-interleave by firmware config.
Christoph Lameter <clameter@sgi.com> wrote:
> Future SGI platforms (actually also current one can have but nothing like
> that is deployed to my knowledge) have nodes with only cpus. Current SGI
> platforms have nodes with just I/O that we so far cannot manage in the
> core. So the arch code maps them to the nearest memory node.
Lee Schermerhorn <Lee.Schermerhorn@hp.com> wrote:
> For the HP platforms, we can configure each cell with from 0% to 100%
> "cell local memory". When we configure with <100% CLM, the "missing
> percentages" are interleaved by hardware on a cache-line granularity to
> improve bandwidth at the expense of latency for numa-challenged
> applications [and OSes, but not our problem ;-)]. When we boot Linux on
> such a config, all of the real nodes have no memory--it all resides in a
> single interleaved pseudo-node.
>
> When we boot Linux on a 100% CLM configuration [== NUMA], we still have
> the interleaved pseudo-node. It contains a few hundred MB stolen from
> the real nodes to contain the DMA zone. [Interleaved memory resides at
> phys addr 0]. The memoryless-nodes patches, along with the zoneorder
> patches, support this config as well.
>
> Also, when we boot a NUMA config with the "mem=" command line,
> specifying less memory than actually exists, Linux takes the excluded
> memory "off the top" rather than distributing it across the nodes. This
> can result in memoryless nodes, as well.
>
This patch:
Preparation for memoryless node patches.
Provide a generic way to keep nodemasks describing various characteristics of
NUMA nodes.
Remove the node_online_map and the node_possible map and realize the same
functionality using two nodes stats: N_POSSIBLE and N_ONLINE.
[Lee.Schermerhorn@hp.com: Initialize N_*_MEMORY and N_CPU masks for non-NUMA config]
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Tested-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Acked-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Acked-by: Bob Picco <bob.picco@hp.com>
Cc: Nishanth Aravamudan <nacc@us.ibm.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@skynet.ie>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: "Serge E. Hallyn" <serge@hallyn.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:27 -06:00
|
|
|
#endif
|
|
|
|
[N_CPU] = { { [0] = 1UL } },
|
|
|
|
#endif /* NUMA */
|
|
|
|
};
|
|
|
|
EXPORT_SYMBOL(node_states);
|
|
|
|
|
2013-07-03 16:03:14 -06:00
|
|
|
/* Protect totalram_pages and zone->managed_pages */
|
|
|
|
static DEFINE_SPINLOCK(managed_page_count_lock);
|
|
|
|
|
2005-09-06 16:17:45 -06:00
|
|
|
unsigned long totalram_pages __read_mostly;
|
2006-04-10 23:52:59 -06:00
|
|
|
unsigned long totalreserve_pages __read_mostly;
|
mm: cma: split cma-reserved in dmesg log
When the system boots up, in the dmesg logs we can see the memory
statistics along with total reserved as below. Memory: 458840k/458840k
available, 65448k reserved, 0K highmem
When CMA is enabled, still the total reserved memory remains the same.
However, the CMA memory is not considered as reserved. But, when we see
/proc/meminfo, the CMA memory is part of free memory. This creates
confusion. This patch corrects the problem by properly subtracting the
CMA reserved memory from the total reserved memory in dmesg logs.
Below is the dmesg snapshot from an arm based device with 512MB RAM and
12MB single CMA region.
Before this change:
Memory: 458840k/458840k available, 65448k reserved, 0K highmem
After this change:
Memory: 458840k/458840k available, 53160k reserved, 12288k cma-reserved, 0K highmem
Signed-off-by: Pintu Kumar <pintu.k@samsung.com>
Signed-off-by: Vishnu Pratap Singh <vishnu.ps@samsung.com>
Acked-by: Michal Nazarewicz <mina86@mina86.com>
Cc: Rafael Aquini <aquini@redhat.com>
Cc: Jerome Marchand <jmarchan@redhat.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-18 17:17:15 -07:00
|
|
|
unsigned long totalcma_pages __read_mostly;
|
2012-01-10 16:07:42 -07:00
|
|
|
|
2012-05-11 02:00:07 -06:00
|
|
|
int percpu_pagelist_fraction;
|
2009-06-17 21:24:12 -06:00
|
|
|
gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2015-09-08 16:01:25 -06:00
|
|
|
/*
|
|
|
|
* A cached value of the page's pageblock's migratetype, used when the page is
|
|
|
|
* put on a pcplist. Used to avoid the pageblock migratetype lookup when
|
|
|
|
* freeing from pcplists in most cases, at the cost of possibly becoming stale.
|
|
|
|
* Also the migratetype set in the page does not necessarily match the pcplist
|
|
|
|
* index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
|
|
|
|
* other index - this ensures that it will be put on the correct CMA freelist.
|
|
|
|
*/
|
|
|
|
static inline int get_pcppage_migratetype(struct page *page)
|
|
|
|
{
|
|
|
|
return page->index;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void set_pcppage_migratetype(struct page *page, int migratetype)
|
|
|
|
{
|
|
|
|
page->index = migratetype;
|
|
|
|
}
|
|
|
|
|
2010-03-05 14:42:13 -07:00
|
|
|
#ifdef CONFIG_PM_SLEEP
|
|
|
|
/*
|
|
|
|
* The following functions are used by the suspend/hibernate code to temporarily
|
|
|
|
* change gfp_allowed_mask in order to avoid using I/O during memory allocations
|
|
|
|
* while devices are suspended. To avoid races with the suspend/hibernate code,
|
|
|
|
* they should always be called with pm_mutex held (gfp_allowed_mask also should
|
|
|
|
* only be modified with pm_mutex held, unless the suspend/hibernate code is
|
|
|
|
* guaranteed not to run in parallel with that modification).
|
|
|
|
*/
|
2010-12-03 14:57:45 -07:00
|
|
|
|
|
|
|
static gfp_t saved_gfp_mask;
|
|
|
|
|
|
|
|
void pm_restore_gfp_mask(void)
|
2010-03-05 14:42:13 -07:00
|
|
|
{
|
|
|
|
WARN_ON(!mutex_is_locked(&pm_mutex));
|
2010-12-03 14:57:45 -07:00
|
|
|
if (saved_gfp_mask) {
|
|
|
|
gfp_allowed_mask = saved_gfp_mask;
|
|
|
|
saved_gfp_mask = 0;
|
|
|
|
}
|
2010-03-05 14:42:13 -07:00
|
|
|
}
|
|
|
|
|
2010-12-03 14:57:45 -07:00
|
|
|
void pm_restrict_gfp_mask(void)
|
2010-03-05 14:42:13 -07:00
|
|
|
{
|
|
|
|
WARN_ON(!mutex_is_locked(&pm_mutex));
|
2010-12-03 14:57:45 -07:00
|
|
|
WARN_ON(saved_gfp_mask);
|
|
|
|
saved_gfp_mask = gfp_allowed_mask;
|
2015-11-06 17:28:21 -07:00
|
|
|
gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
|
2010-03-05 14:42:13 -07:00
|
|
|
}
|
2012-01-10 16:07:15 -07:00
|
|
|
|
|
|
|
bool pm_suspended_storage(void)
|
|
|
|
{
|
2015-11-06 17:28:21 -07:00
|
|
|
if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
|
2012-01-10 16:07:15 -07:00
|
|
|
return false;
|
|
|
|
return true;
|
|
|
|
}
|
2010-03-05 14:42:13 -07:00
|
|
|
#endif /* CONFIG_PM_SLEEP */
|
|
|
|
|
2007-10-16 02:26:01 -06:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
|
2015-11-06 17:29:57 -07:00
|
|
|
unsigned int pageblock_order __read_mostly;
|
2007-10-16 02:26:01 -06:00
|
|
|
#endif
|
|
|
|
|
2006-02-14 14:52:59 -07:00
|
|
|
static void __free_pages_ok(struct page *page, unsigned int order);
|
2006-01-06 01:11:08 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* results with 256, 32 in the lowmem_reserve sysctl:
|
|
|
|
* 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
|
|
|
|
* 1G machine -> (16M dma, 784M normal, 224M high)
|
|
|
|
* NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
|
|
|
|
* HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
|
2015-02-12 16:00:22 -07:00
|
|
|
* HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
|
2005-11-05 09:25:53 -07:00
|
|
|
*
|
|
|
|
* TBD: should special case ZONE_DMA32 machines here - in those we normally
|
|
|
|
* don't need any ZONE_NORMAL reservation
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2006-09-26 00:31:13 -06:00
|
|
|
int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
|
2007-02-10 02:43:10 -07:00
|
|
|
#ifdef CONFIG_ZONE_DMA
|
2006-09-26 00:31:13 -06:00
|
|
|
256,
|
2007-02-10 02:43:10 -07:00
|
|
|
#endif
|
2006-09-26 00:31:13 -06:00
|
|
|
#ifdef CONFIG_ZONE_DMA32
|
2006-09-26 00:31:13 -06:00
|
|
|
256,
|
2006-09-26 00:31:13 -06:00
|
|
|
#endif
|
2006-09-26 00:31:14 -06:00
|
|
|
#ifdef CONFIG_HIGHMEM
|
2007-07-17 05:03:12 -06:00
|
|
|
32,
|
2006-09-26 00:31:14 -06:00
|
|
|
#endif
|
2007-07-17 05:03:12 -06:00
|
|
|
32,
|
2006-09-26 00:31:13 -06:00
|
|
|
};
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
EXPORT_SYMBOL(totalram_pages);
|
|
|
|
|
2006-12-06 21:40:36 -07:00
|
|
|
static char * const zone_names[MAX_NR_ZONES] = {
|
2007-02-10 02:43:10 -07:00
|
|
|
#ifdef CONFIG_ZONE_DMA
|
2006-09-26 00:31:13 -06:00
|
|
|
"DMA",
|
2007-02-10 02:43:10 -07:00
|
|
|
#endif
|
2006-09-26 00:31:13 -06:00
|
|
|
#ifdef CONFIG_ZONE_DMA32
|
2006-09-26 00:31:13 -06:00
|
|
|
"DMA32",
|
2006-09-26 00:31:13 -06:00
|
|
|
#endif
|
2006-09-26 00:31:13 -06:00
|
|
|
"Normal",
|
2006-09-26 00:31:14 -06:00
|
|
|
#ifdef CONFIG_HIGHMEM
|
2007-07-17 05:03:12 -06:00
|
|
|
"HighMem",
|
2006-09-26 00:31:14 -06:00
|
|
|
#endif
|
2007-07-17 05:03:12 -06:00
|
|
|
"Movable",
|
2015-08-09 13:29:06 -06:00
|
|
|
#ifdef CONFIG_ZONE_DEVICE
|
|
|
|
"Device",
|
|
|
|
#endif
|
2006-09-26 00:31:13 -06:00
|
|
|
};
|
|
|
|
|
mm, page_owner: print migratetype of page and pageblock, symbolic flags
The information in /sys/kernel/debug/page_owner includes the migratetype
of the pageblock the page belongs to. This is also checked against the
page's migratetype (as declared by gfp_flags during its allocation), and
the page is reported as Fallback if its migratetype differs from the
pageblock's one. t This is somewhat misleading because in fact fallback
allocation is not the only reason why these two can differ. It also
doesn't direcly provide the page's migratetype, although it's possible
to derive that from the gfp_flags.
It's arguably better to print both page and pageblock's migratetype and
leave the interpretation to the consumer than to suggest fallback
allocation as the only possible reason. While at it, we can print the
migratetypes as string the same way as /proc/pagetypeinfo does, as some
of the numeric values depend on kernel configuration. For that, this
patch moves the migratetype_names array from #ifdef CONFIG_PROC_FS part
of mm/vmstat.c to mm/page_alloc.c and exports it.
With the new format strings for flags, we can now also provide symbolic
page and gfp flags in the /sys/kernel/debug/page_owner file. This
replaces the positional printing of page flags as single letters, which
might have looked nicer, but was limited to a subset of flags, and
required the user to remember the letters.
Example page_owner entry after the patch:
Page allocated via order 0, mask 0x24213ca(GFP_HIGHUSER_MOVABLE|__GFP_COLD|__GFP_NOWARN|__GFP_NORETRY)
PFN 520 type Movable Block 1 type Movable Flags 0xfffff8001006c(referenced|uptodate|lru|active|mappedtodisk)
[<ffffffff811682c4>] __alloc_pages_nodemask+0x134/0x230
[<ffffffff811b4058>] alloc_pages_current+0x88/0x120
[<ffffffff8115e386>] __page_cache_alloc+0xe6/0x120
[<ffffffff8116ba6c>] __do_page_cache_readahead+0xdc/0x240
[<ffffffff8116bd05>] ondemand_readahead+0x135/0x260
[<ffffffff8116bfb1>] page_cache_sync_readahead+0x31/0x50
[<ffffffff81160523>] generic_file_read_iter+0x453/0x760
[<ffffffff811e0d57>] __vfs_read+0xa7/0xd0
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Sasha Levin <sasha.levin@oracle.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Mel Gorman <mgorman@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-15 15:56:08 -06:00
|
|
|
char * const migratetype_names[MIGRATE_TYPES] = {
|
|
|
|
"Unmovable",
|
|
|
|
"Movable",
|
|
|
|
"Reclaimable",
|
|
|
|
"HighAtomic",
|
|
|
|
#ifdef CONFIG_CMA
|
|
|
|
"CMA",
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_MEMORY_ISOLATION
|
|
|
|
"Isolate",
|
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
2015-11-06 17:29:50 -07:00
|
|
|
compound_page_dtor * const compound_page_dtors[] = {
|
|
|
|
NULL,
|
|
|
|
free_compound_page,
|
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
free_huge_page,
|
|
|
|
#endif
|
2016-01-15 17:54:17 -07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
free_transhuge_page,
|
|
|
|
#endif
|
2015-11-06 17:29:50 -07:00
|
|
|
};
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
int min_free_kbytes = 1024;
|
2014-01-23 16:53:28 -07:00
|
|
|
int user_min_free_kbytes = -1;
|
2016-03-17 15:19:14 -06:00
|
|
|
int watermark_scale_factor = 10;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-09-21 18:03:07 -06:00
|
|
|
static unsigned long __meminitdata nr_kernel_pages;
|
|
|
|
static unsigned long __meminitdata nr_all_pages;
|
2007-05-08 01:23:07 -06:00
|
|
|
static unsigned long __meminitdata dma_reserve;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2011-12-08 11:22:09 -07:00
|
|
|
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
|
|
|
|
static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
|
|
|
|
static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
|
|
|
|
static unsigned long __initdata required_kernelcore;
|
|
|
|
static unsigned long __initdata required_movablecore;
|
|
|
|
static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
|
2016-03-15 15:55:22 -06:00
|
|
|
static bool mirrored_kernelcore;
|
2011-12-08 11:22:09 -07:00
|
|
|
|
|
|
|
/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
|
|
|
|
int movable_zone;
|
|
|
|
EXPORT_SYMBOL(movable_zone);
|
|
|
|
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
2007-05-23 14:57:55 -06:00
|
|
|
#if MAX_NUMNODES > 1
|
|
|
|
int nr_node_ids __read_mostly = MAX_NUMNODES;
|
2009-06-16 16:32:15 -06:00
|
|
|
int nr_online_nodes __read_mostly = 1;
|
2007-05-23 14:57:55 -06:00
|
|
|
EXPORT_SYMBOL(nr_node_ids);
|
2009-06-16 16:32:15 -06:00
|
|
|
EXPORT_SYMBOL(nr_online_nodes);
|
2007-05-23 14:57:55 -06:00
|
|
|
#endif
|
|
|
|
|
2007-10-16 02:25:54 -06:00
|
|
|
int page_group_by_mobility_disabled __read_mostly;
|
|
|
|
|
2015-06-30 15:57:02 -06:00
|
|
|
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
|
|
|
|
static inline void reset_deferred_meminit(pg_data_t *pgdat)
|
|
|
|
{
|
|
|
|
pgdat->first_deferred_pfn = ULONG_MAX;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Returns true if the struct page for the pfn is uninitialised */
|
2015-06-30 15:57:27 -06:00
|
|
|
static inline bool __meminit early_page_uninitialised(unsigned long pfn)
|
2015-06-30 15:57:02 -06:00
|
|
|
{
|
2016-07-14 13:07:23 -06:00
|
|
|
int nid = early_pfn_to_nid(pfn);
|
|
|
|
|
|
|
|
if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
|
2015-06-30 15:57:02 -06:00
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Returns false when the remaining initialisation should be deferred until
|
|
|
|
* later in the boot cycle when it can be parallelised.
|
|
|
|
*/
|
|
|
|
static inline bool update_defer_init(pg_data_t *pgdat,
|
|
|
|
unsigned long pfn, unsigned long zone_end,
|
|
|
|
unsigned long *nr_initialised)
|
|
|
|
{
|
mm: meminit: initialise more memory for inode/dentry hash tables in early boot
Upstream has supported page parallel initialisation for X86 and the boot
time is improved greately. Some tests have been done for Power.
Here is the result I have done with different memory size.
* 4GB memory:
boot time is as the following:
with patch vs without patch: 10.4s vs 24.5s
boot time is improved 57%
* 200GB memory:
boot time looks the same with and without patches.
boot time is about 38s
* 32TB memory:
boot time looks the same with and without patches
boot time is about 160s.
The boot time is much shorter than X86 with 24TB memory.
From community discussion, it costs about 694s for X86 24T system.
Parallel initialisation improves the performance by deferring memory
initilisation to kswap with N kthreads, it should improve the performance
therotically.
In testing on X86, performance is improved greatly with huge memory. But
on Power platform, it is improved greatly with less than 100GB memory.
For huge memory, it is not improved greatly. But it saves the time with
several threads at least, as the following information shows(32TB system
log):
[ 22.648169] node 9 initialised, 16607461 pages in 280ms
[ 22.783772] node 3 initialised, 23937243 pages in 410ms
[ 22.858877] node 6 initialised, 29179347 pages in 490ms
[ 22.863252] node 2 initialised, 29179347 pages in 490ms
[ 22.907545] node 0 initialised, 32049614 pages in 540ms
[ 22.920891] node 15 initialised, 32212280 pages in 550ms
[ 22.923236] node 4 initialised, 32306127 pages in 550ms
[ 22.923384] node 12 initialised, 32314319 pages in 550ms
[ 22.924754] node 8 initialised, 32314319 pages in 550ms
[ 22.940780] node 13 initialised, 33353677 pages in 570ms
[ 22.940796] node 11 initialised, 33353677 pages in 570ms
[ 22.941700] node 5 initialised, 33353677 pages in 570ms
[ 22.941721] node 10 initialised, 33353677 pages in 570ms
[ 22.941876] node 7 initialised, 33353677 pages in 570ms
[ 22.944946] node 14 initialised, 33353677 pages in 570ms
[ 22.946063] node 1 initialised, 33345485 pages in 580ms
It saves the time about 550*16 ms at least, although it can be ignore to
compare the boot time about 160 seconds. What's more, the boot time is
much shorter on Power even without patches than x86 for huge memory
machine.
So this patchset is still necessary to be enabled for Power.
This patch (of 2):
This patch is based on Mel Gorman's old patch in the mailing list,
https://lkml.org/lkml/2015/5/5/280 which is discussed but it is fixed with
a completion to wait for all memory initialised in page_alloc_init_late().
It is to fix the OOM problem on X86 with 24TB memory which allocates
memory in late initialisation. But for Power platform with 32TB memory,
it causes a call trace in vfs_caches_init->inode_init() and inode hash
table needs more memory. So this patch allocates 1GB for 0.25TB/node for
large system as it is mentioned in https://lkml.org/lkml/2015/5/1/627
This call trace is found on Power with 32TB memory, 1024CPUs, 16nodes.
Currently, it only allocates 2GB*16=32GB for early initialisation. But
Dentry cache hash table needes 16GB and Inode cache hash table needs 16GB.
So the system have no enough memory for it. The log from dmesg as the
following:
Dentry cache hash table entries: 2147483648 (order: 18,17179869184 bytes)
vmalloc: allocation failure, allocated 16021913600 of 17179934720 bytes
swapper/0: page allocation failure: order:0,mode:0x2080020
CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.4.0-0-ppc64
Call Trace:
.dump_stack+0xb4/0xb664 (unreliable)
.warn_alloc_failed+0x114/0x160
.__vmalloc_area_node+0x1a4/0x2b0
.__vmalloc_node_range+0xe4/0x110
.__vmalloc_node+0x40/0x50
.alloc_large_system_hash+0x134/0x2a4
.inode_init+0xa4/0xf0
.vfs_caches_init+0x80/0x144
.start_kernel+0x40c/0x4e0
start_here_common+0x20/0x4a4
Signed-off-by: Li Zhang <zhlcindy@linux.vnet.ibm.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-17 15:20:16 -06:00
|
|
|
unsigned long max_initialise;
|
|
|
|
|
2015-06-30 15:57:02 -06:00
|
|
|
/* Always populate low zones for address-contrained allocations */
|
|
|
|
if (zone_end < pgdat_end_pfn(pgdat))
|
|
|
|
return true;
|
mm: meminit: initialise more memory for inode/dentry hash tables in early boot
Upstream has supported page parallel initialisation for X86 and the boot
time is improved greately. Some tests have been done for Power.
Here is the result I have done with different memory size.
* 4GB memory:
boot time is as the following:
with patch vs without patch: 10.4s vs 24.5s
boot time is improved 57%
* 200GB memory:
boot time looks the same with and without patches.
boot time is about 38s
* 32TB memory:
boot time looks the same with and without patches
boot time is about 160s.
The boot time is much shorter than X86 with 24TB memory.
From community discussion, it costs about 694s for X86 24T system.
Parallel initialisation improves the performance by deferring memory
initilisation to kswap with N kthreads, it should improve the performance
therotically.
In testing on X86, performance is improved greatly with huge memory. But
on Power platform, it is improved greatly with less than 100GB memory.
For huge memory, it is not improved greatly. But it saves the time with
several threads at least, as the following information shows(32TB system
log):
[ 22.648169] node 9 initialised, 16607461 pages in 280ms
[ 22.783772] node 3 initialised, 23937243 pages in 410ms
[ 22.858877] node 6 initialised, 29179347 pages in 490ms
[ 22.863252] node 2 initialised, 29179347 pages in 490ms
[ 22.907545] node 0 initialised, 32049614 pages in 540ms
[ 22.920891] node 15 initialised, 32212280 pages in 550ms
[ 22.923236] node 4 initialised, 32306127 pages in 550ms
[ 22.923384] node 12 initialised, 32314319 pages in 550ms
[ 22.924754] node 8 initialised, 32314319 pages in 550ms
[ 22.940780] node 13 initialised, 33353677 pages in 570ms
[ 22.940796] node 11 initialised, 33353677 pages in 570ms
[ 22.941700] node 5 initialised, 33353677 pages in 570ms
[ 22.941721] node 10 initialised, 33353677 pages in 570ms
[ 22.941876] node 7 initialised, 33353677 pages in 570ms
[ 22.944946] node 14 initialised, 33353677 pages in 570ms
[ 22.946063] node 1 initialised, 33345485 pages in 580ms
It saves the time about 550*16 ms at least, although it can be ignore to
compare the boot time about 160 seconds. What's more, the boot time is
much shorter on Power even without patches than x86 for huge memory
machine.
So this patchset is still necessary to be enabled for Power.
This patch (of 2):
This patch is based on Mel Gorman's old patch in the mailing list,
https://lkml.org/lkml/2015/5/5/280 which is discussed but it is fixed with
a completion to wait for all memory initialised in page_alloc_init_late().
It is to fix the OOM problem on X86 with 24TB memory which allocates
memory in late initialisation. But for Power platform with 32TB memory,
it causes a call trace in vfs_caches_init->inode_init() and inode hash
table needs more memory. So this patch allocates 1GB for 0.25TB/node for
large system as it is mentioned in https://lkml.org/lkml/2015/5/1/627
This call trace is found on Power with 32TB memory, 1024CPUs, 16nodes.
Currently, it only allocates 2GB*16=32GB for early initialisation. But
Dentry cache hash table needes 16GB and Inode cache hash table needs 16GB.
So the system have no enough memory for it. The log from dmesg as the
following:
Dentry cache hash table entries: 2147483648 (order: 18,17179869184 bytes)
vmalloc: allocation failure, allocated 16021913600 of 17179934720 bytes
swapper/0: page allocation failure: order:0,mode:0x2080020
CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.4.0-0-ppc64
Call Trace:
.dump_stack+0xb4/0xb664 (unreliable)
.warn_alloc_failed+0x114/0x160
.__vmalloc_area_node+0x1a4/0x2b0
.__vmalloc_node_range+0xe4/0x110
.__vmalloc_node+0x40/0x50
.alloc_large_system_hash+0x134/0x2a4
.inode_init+0xa4/0xf0
.vfs_caches_init+0x80/0x144
.start_kernel+0x40c/0x4e0
start_here_common+0x20/0x4a4
Signed-off-by: Li Zhang <zhlcindy@linux.vnet.ibm.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-17 15:20:16 -06:00
|
|
|
/*
|
|
|
|
* Initialise at least 2G of a node but also take into account that
|
|
|
|
* two large system hashes that can take up 1GB for 0.25TB/node.
|
|
|
|
*/
|
|
|
|
max_initialise = max(2UL << (30 - PAGE_SHIFT),
|
|
|
|
(pgdat->node_spanned_pages >> 8));
|
2015-06-30 15:57:02 -06:00
|
|
|
|
|
|
|
(*nr_initialised)++;
|
mm: meminit: initialise more memory for inode/dentry hash tables in early boot
Upstream has supported page parallel initialisation for X86 and the boot
time is improved greately. Some tests have been done for Power.
Here is the result I have done with different memory size.
* 4GB memory:
boot time is as the following:
with patch vs without patch: 10.4s vs 24.5s
boot time is improved 57%
* 200GB memory:
boot time looks the same with and without patches.
boot time is about 38s
* 32TB memory:
boot time looks the same with and without patches
boot time is about 160s.
The boot time is much shorter than X86 with 24TB memory.
From community discussion, it costs about 694s for X86 24T system.
Parallel initialisation improves the performance by deferring memory
initilisation to kswap with N kthreads, it should improve the performance
therotically.
In testing on X86, performance is improved greatly with huge memory. But
on Power platform, it is improved greatly with less than 100GB memory.
For huge memory, it is not improved greatly. But it saves the time with
several threads at least, as the following information shows(32TB system
log):
[ 22.648169] node 9 initialised, 16607461 pages in 280ms
[ 22.783772] node 3 initialised, 23937243 pages in 410ms
[ 22.858877] node 6 initialised, 29179347 pages in 490ms
[ 22.863252] node 2 initialised, 29179347 pages in 490ms
[ 22.907545] node 0 initialised, 32049614 pages in 540ms
[ 22.920891] node 15 initialised, 32212280 pages in 550ms
[ 22.923236] node 4 initialised, 32306127 pages in 550ms
[ 22.923384] node 12 initialised, 32314319 pages in 550ms
[ 22.924754] node 8 initialised, 32314319 pages in 550ms
[ 22.940780] node 13 initialised, 33353677 pages in 570ms
[ 22.940796] node 11 initialised, 33353677 pages in 570ms
[ 22.941700] node 5 initialised, 33353677 pages in 570ms
[ 22.941721] node 10 initialised, 33353677 pages in 570ms
[ 22.941876] node 7 initialised, 33353677 pages in 570ms
[ 22.944946] node 14 initialised, 33353677 pages in 570ms
[ 22.946063] node 1 initialised, 33345485 pages in 580ms
It saves the time about 550*16 ms at least, although it can be ignore to
compare the boot time about 160 seconds. What's more, the boot time is
much shorter on Power even without patches than x86 for huge memory
machine.
So this patchset is still necessary to be enabled for Power.
This patch (of 2):
This patch is based on Mel Gorman's old patch in the mailing list,
https://lkml.org/lkml/2015/5/5/280 which is discussed but it is fixed with
a completion to wait for all memory initialised in page_alloc_init_late().
It is to fix the OOM problem on X86 with 24TB memory which allocates
memory in late initialisation. But for Power platform with 32TB memory,
it causes a call trace in vfs_caches_init->inode_init() and inode hash
table needs more memory. So this patch allocates 1GB for 0.25TB/node for
large system as it is mentioned in https://lkml.org/lkml/2015/5/1/627
This call trace is found on Power with 32TB memory, 1024CPUs, 16nodes.
Currently, it only allocates 2GB*16=32GB for early initialisation. But
Dentry cache hash table needes 16GB and Inode cache hash table needs 16GB.
So the system have no enough memory for it. The log from dmesg as the
following:
Dentry cache hash table entries: 2147483648 (order: 18,17179869184 bytes)
vmalloc: allocation failure, allocated 16021913600 of 17179934720 bytes
swapper/0: page allocation failure: order:0,mode:0x2080020
CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.4.0-0-ppc64
Call Trace:
.dump_stack+0xb4/0xb664 (unreliable)
.warn_alloc_failed+0x114/0x160
.__vmalloc_area_node+0x1a4/0x2b0
.__vmalloc_node_range+0xe4/0x110
.__vmalloc_node+0x40/0x50
.alloc_large_system_hash+0x134/0x2a4
.inode_init+0xa4/0xf0
.vfs_caches_init+0x80/0x144
.start_kernel+0x40c/0x4e0
start_here_common+0x20/0x4a4
Signed-off-by: Li Zhang <zhlcindy@linux.vnet.ibm.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-17 15:20:16 -06:00
|
|
|
if ((*nr_initialised > max_initialise) &&
|
2015-06-30 15:57:02 -06:00
|
|
|
(pfn & (PAGES_PER_SECTION - 1)) == 0) {
|
|
|
|
pgdat->first_deferred_pfn = pfn;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline void reset_deferred_meminit(pg_data_t *pgdat)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool early_page_uninitialised(unsigned long pfn)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool update_defer_init(pg_data_t *pgdat,
|
|
|
|
unsigned long pfn, unsigned long zone_end,
|
|
|
|
unsigned long *nr_initialised)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2016-05-19 18:14:27 -06:00
|
|
|
/* Return a pointer to the bitmap storing bits affecting a block of pages */
|
|
|
|
static inline unsigned long *get_pageblock_bitmap(struct page *page,
|
|
|
|
unsigned long pfn)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_SPARSEMEM
|
|
|
|
return __pfn_to_section(pfn)->pageblock_flags;
|
|
|
|
#else
|
|
|
|
return page_zone(page)->pageblock_flags;
|
|
|
|
#endif /* CONFIG_SPARSEMEM */
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_SPARSEMEM
|
|
|
|
pfn &= (PAGES_PER_SECTION-1);
|
|
|
|
return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
|
|
|
|
#else
|
|
|
|
pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
|
|
|
|
return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
|
|
|
|
#endif /* CONFIG_SPARSEMEM */
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
|
|
|
|
* @page: The page within the block of interest
|
|
|
|
* @pfn: The target page frame number
|
|
|
|
* @end_bitidx: The last bit of interest to retrieve
|
|
|
|
* @mask: mask of bits that the caller is interested in
|
|
|
|
*
|
|
|
|
* Return: pageblock_bits flags
|
|
|
|
*/
|
|
|
|
static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
|
|
|
|
unsigned long pfn,
|
|
|
|
unsigned long end_bitidx,
|
|
|
|
unsigned long mask)
|
|
|
|
{
|
|
|
|
unsigned long *bitmap;
|
|
|
|
unsigned long bitidx, word_bitidx;
|
|
|
|
unsigned long word;
|
|
|
|
|
|
|
|
bitmap = get_pageblock_bitmap(page, pfn);
|
|
|
|
bitidx = pfn_to_bitidx(page, pfn);
|
|
|
|
word_bitidx = bitidx / BITS_PER_LONG;
|
|
|
|
bitidx &= (BITS_PER_LONG-1);
|
|
|
|
|
|
|
|
word = bitmap[word_bitidx];
|
|
|
|
bitidx += end_bitidx;
|
|
|
|
return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
|
|
|
|
unsigned long end_bitidx,
|
|
|
|
unsigned long mask)
|
|
|
|
{
|
|
|
|
return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
|
|
|
|
{
|
|
|
|
return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
|
|
|
|
* @page: The page within the block of interest
|
|
|
|
* @flags: The flags to set
|
|
|
|
* @pfn: The target page frame number
|
|
|
|
* @end_bitidx: The last bit of interest
|
|
|
|
* @mask: mask of bits that the caller is interested in
|
|
|
|
*/
|
|
|
|
void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
|
|
|
|
unsigned long pfn,
|
|
|
|
unsigned long end_bitidx,
|
|
|
|
unsigned long mask)
|
|
|
|
{
|
|
|
|
unsigned long *bitmap;
|
|
|
|
unsigned long bitidx, word_bitidx;
|
|
|
|
unsigned long old_word, word;
|
|
|
|
|
|
|
|
BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
|
|
|
|
|
|
|
|
bitmap = get_pageblock_bitmap(page, pfn);
|
|
|
|
bitidx = pfn_to_bitidx(page, pfn);
|
|
|
|
word_bitidx = bitidx / BITS_PER_LONG;
|
|
|
|
bitidx &= (BITS_PER_LONG-1);
|
|
|
|
|
|
|
|
VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
|
|
|
|
|
|
|
|
bitidx += end_bitidx;
|
|
|
|
mask <<= (BITS_PER_LONG - bitidx - 1);
|
|
|
|
flags <<= (BITS_PER_LONG - bitidx - 1);
|
|
|
|
|
|
|
|
word = READ_ONCE(bitmap[word_bitidx]);
|
|
|
|
for (;;) {
|
|
|
|
old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
|
|
|
|
if (word == old_word)
|
|
|
|
break;
|
|
|
|
word = old_word;
|
|
|
|
}
|
|
|
|
}
|
2015-06-30 15:57:02 -06:00
|
|
|
|
2012-07-31 17:43:50 -06:00
|
|
|
void set_pageblock_migratetype(struct page *page, int migratetype)
|
2007-10-16 02:25:48 -06:00
|
|
|
{
|
2013-11-12 16:08:18 -07:00
|
|
|
if (unlikely(page_group_by_mobility_disabled &&
|
|
|
|
migratetype < MIGRATE_PCPTYPES))
|
2009-06-16 16:31:58 -06:00
|
|
|
migratetype = MIGRATE_UNMOVABLE;
|
|
|
|
|
2007-10-16 02:25:48 -06:00
|
|
|
set_pageblock_flags_group(page, (unsigned long)migratetype,
|
|
|
|
PB_migrate, PB_migrate_end);
|
|
|
|
}
|
|
|
|
|
2006-01-06 01:10:58 -07:00
|
|
|
#ifdef CONFIG_DEBUG_VM
|
2005-10-29 19:16:52 -06:00
|
|
|
static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2005-10-29 19:16:53 -06:00
|
|
|
int ret = 0;
|
|
|
|
unsigned seq;
|
|
|
|
unsigned long pfn = page_to_pfn(page);
|
2013-02-22 17:35:28 -07:00
|
|
|
unsigned long sp, start_pfn;
|
2005-10-29 19:16:52 -06:00
|
|
|
|
2005-10-29 19:16:53 -06:00
|
|
|
do {
|
|
|
|
seq = zone_span_seqbegin(zone);
|
2013-02-22 17:35:28 -07:00
|
|
|
start_pfn = zone->zone_start_pfn;
|
|
|
|
sp = zone->spanned_pages;
|
2013-02-22 17:35:23 -07:00
|
|
|
if (!zone_spans_pfn(zone, pfn))
|
2005-10-29 19:16:53 -06:00
|
|
|
ret = 1;
|
|
|
|
} while (zone_span_seqretry(zone, seq));
|
|
|
|
|
2013-02-22 17:35:28 -07:00
|
|
|
if (ret)
|
2014-06-04 17:07:27 -06:00
|
|
|
pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
|
|
|
|
pfn, zone_to_nid(zone), zone->name,
|
|
|
|
start_pfn, start_pfn + sp);
|
2013-02-22 17:35:28 -07:00
|
|
|
|
2005-10-29 19:16:53 -06:00
|
|
|
return ret;
|
2005-10-29 19:16:52 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int page_is_consistent(struct zone *zone, struct page *page)
|
|
|
|
{
|
2007-05-06 15:49:14 -06:00
|
|
|
if (!pfn_valid_within(page_to_pfn(page)))
|
2005-10-29 19:16:52 -06:00
|
|
|
return 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
if (zone != page_zone(page))
|
2005-10-29 19:16:52 -06:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Temporary debugging check for pages not lying within a given zone.
|
|
|
|
*/
|
|
|
|
static int bad_range(struct zone *zone, struct page *page)
|
|
|
|
{
|
|
|
|
if (page_outside_zone_boundaries(zone, page))
|
2005-04-16 16:20:36 -06:00
|
|
|
return 1;
|
2005-10-29 19:16:52 -06:00
|
|
|
if (!page_is_consistent(zone, page))
|
|
|
|
return 1;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
|
|
|
}
|
2006-01-06 01:10:58 -07:00
|
|
|
#else
|
|
|
|
static inline int bad_range(struct zone *zone, struct page *page)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2014-04-07 16:37:38 -06:00
|
|
|
static void bad_page(struct page *page, const char *reason,
|
|
|
|
unsigned long bad_flags)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2009-01-06 15:40:12 -07:00
|
|
|
static unsigned long resume;
|
|
|
|
static unsigned long nr_shown;
|
|
|
|
static unsigned long nr_unshown;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Allow a burst of 60 reports, then keep quiet for that minute;
|
|
|
|
* or allow a steady drip of one report per second.
|
|
|
|
*/
|
|
|
|
if (nr_shown == 60) {
|
|
|
|
if (time_before(jiffies, resume)) {
|
|
|
|
nr_unshown++;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (nr_unshown) {
|
2016-03-15 15:56:24 -06:00
|
|
|
pr_alert(
|
2009-01-06 15:40:13 -07:00
|
|
|
"BUG: Bad page state: %lu messages suppressed\n",
|
2009-01-06 15:40:12 -07:00
|
|
|
nr_unshown);
|
|
|
|
nr_unshown = 0;
|
|
|
|
}
|
|
|
|
nr_shown = 0;
|
|
|
|
}
|
|
|
|
if (nr_shown++ == 0)
|
|
|
|
resume = jiffies + 60 * HZ;
|
|
|
|
|
2016-03-15 15:56:24 -06:00
|
|
|
pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
|
badpage: replace page_remove_rmap Eeek and BUG
Now that bad pages are kept out of circulation, there is no need for the
infamous page_remove_rmap() BUG() - once that page is freed, its negative
mapcount will issue a "Bad page state" message and the page won't be
freed. Removing the BUG() allows more info, on subsequent pages, to be
gathered.
We do have more info about the page at this point than bad_page() can know
- notably, what the pmd is, which might pinpoint something like low 64kB
corruption - but page_remove_rmap() isn't given the address to find that.
In practice, there is only one call to page_remove_rmap() which has ever
reported anything, that from zap_pte_range() (usually on exit, sometimes
on munmap). It has all the info, so remove page_remove_rmap()'s "Eeek"
message and leave it all to zap_pte_range().
mm/memory.c already has a hardly used print_bad_pte() function, showing
some of the appropriate info: extend it to show what we want for the rmap
case: pte info, page info (when there is a page) and vma info to compare.
zap_pte_range() already knows the pmd, but print_bad_pte() is easier to
use if it works that out for itself.
Some of this info is also shown in bad_page()'s "Bad page state" message.
Keep them separate, but adjust them to match each other as far as
possible. Say "Bad page map" in print_bad_pte(), and add a TAINT_BAD_PAGE
there too.
print_bad_pte() show current->comm unconditionally (though it should get
repeated in the usually irrelevant stack trace): sorry, I misled Nick
Piggin to make it conditional on vm_mm == current->mm, but current->mm is
already NULL in the exit case. Usually current->comm is good, though
exceptionally it may not be that of the mm (when "swapoff" for example).
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-01-06 15:40:08 -07:00
|
|
|
current->comm, page_to_pfn(page));
|
2016-03-15 15:56:24 -06:00
|
|
|
__dump_page(page, reason);
|
|
|
|
bad_flags &= page->flags;
|
|
|
|
if (bad_flags)
|
|
|
|
pr_alert("bad because of flags: %#lx(%pGp)\n",
|
|
|
|
bad_flags, &bad_flags);
|
2016-03-15 15:56:21 -06:00
|
|
|
dump_page_owner(page);
|
badpage: replace page_remove_rmap Eeek and BUG
Now that bad pages are kept out of circulation, there is no need for the
infamous page_remove_rmap() BUG() - once that page is freed, its negative
mapcount will issue a "Bad page state" message and the page won't be
freed. Removing the BUG() allows more info, on subsequent pages, to be
gathered.
We do have more info about the page at this point than bad_page() can know
- notably, what the pmd is, which might pinpoint something like low 64kB
corruption - but page_remove_rmap() isn't given the address to find that.
In practice, there is only one call to page_remove_rmap() which has ever
reported anything, that from zap_pte_range() (usually on exit, sometimes
on munmap). It has all the info, so remove page_remove_rmap()'s "Eeek"
message and leave it all to zap_pte_range().
mm/memory.c already has a hardly used print_bad_pte() function, showing
some of the appropriate info: extend it to show what we want for the rmap
case: pte info, page info (when there is a page) and vma info to compare.
zap_pte_range() already knows the pmd, but print_bad_pte() is easier to
use if it works that out for itself.
Some of this info is also shown in bad_page()'s "Bad page state" message.
Keep them separate, but adjust them to match each other as far as
possible. Say "Bad page map" in print_bad_pte(), and add a TAINT_BAD_PAGE
there too.
print_bad_pte() show current->comm unconditionally (though it should get
repeated in the usually irrelevant stack trace): sorry, I misled Nick
Piggin to make it conditional on vm_mm == current->mm, but current->mm is
already NULL in the exit case. Usually current->comm is good, though
exceptionally it may not be that of the mm (when "swapoff" for example).
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-01-06 15:40:08 -07:00
|
|
|
|
2011-10-31 18:07:24 -06:00
|
|
|
print_modules();
|
2005-04-16 16:20:36 -06:00
|
|
|
dump_stack();
|
2009-01-06 15:40:12 -07:00
|
|
|
out:
|
2009-01-06 15:40:06 -07:00
|
|
|
/* Leave bad fields for debug, except PageBuddy could make trouble */
|
2013-02-22 17:34:59 -07:00
|
|
|
page_mapcount_reset(page); /* remove PageBuddy */
|
2013-01-20 23:47:39 -07:00
|
|
|
add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Higher-order pages are called "compound pages". They are structured thusly:
|
|
|
|
*
|
2015-11-06 17:29:54 -07:00
|
|
|
* The first PAGE_SIZE page is called the "head page" and have PG_head set.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
2015-11-06 17:29:54 -07:00
|
|
|
* The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
|
|
|
|
* in bit 0 of page->compound_head. The rest of bits is pointer to head page.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
2015-11-06 17:29:54 -07:00
|
|
|
* The first tail page's ->compound_dtor holds the offset in array of compound
|
|
|
|
* page destructors. See compound_page_dtors.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
2015-11-06 17:29:54 -07:00
|
|
|
* The first tail page's ->compound_order holds the order of allocation.
|
[PATCH] compound page: use page[1].lru
If a compound page has its own put_page_testzero destructor (the only current
example is free_huge_page), that is noted in page[1].mapping of the compound
page. But that's rather a poor place to keep it: functions which call
set_page_dirty_lock after get_user_pages (e.g. Infiniband's
__ib_umem_release) ought to be checking first, otherwise set_page_dirty is
liable to crash on what's not the address of a struct address_space.
And now I'm about to make that worse: it turns out that every compound page
needs a destructor, so we can no longer rely on hugetlb pages going their own
special way, to avoid further problems of page->mapping reuse. For example,
not many people know that: on 50% of i386 -Os builds, the first tail page of a
compound page purports to be PageAnon (when its destructor has an odd
address), which surprises page_add_file_rmap.
Keep the compound page destructor in page[1].lru.next instead. And to free up
the common pairing of mapping and index, also move compound page order from
index to lru.prev. Slab reuses page->lru too: but if we ever need slab to use
compound pages, it can easily stack its use above this.
(akpm: decoded version of the above: the tail pages of a compound page now
have ->mapping==NULL, so there's no need for the set_page_dirty[_lock]()
caller to check that they're not compund pages before doing the dirty).
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-02-14 14:52:58 -07:00
|
|
|
* This usage means that zero-order pages may not be compound.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2006-02-14 14:52:59 -07:00
|
|
|
|
2016-01-15 17:54:17 -07:00
|
|
|
void free_compound_page(struct page *page)
|
2006-02-14 14:52:59 -07:00
|
|
|
{
|
2007-05-06 15:49:39 -06:00
|
|
|
__free_pages_ok(page, compound_order(page));
|
2006-02-14 14:52:59 -07:00
|
|
|
}
|
|
|
|
|
2015-11-06 17:29:57 -07:00
|
|
|
void prep_compound_page(struct page *page, unsigned int order)
|
2008-11-06 13:53:27 -07:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
int nr_pages = 1 << order;
|
|
|
|
|
2015-11-06 17:29:50 -07:00
|
|
|
set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
|
2008-11-06 13:53:27 -07:00
|
|
|
set_compound_order(page, order);
|
|
|
|
__SetPageHead(page);
|
|
|
|
for (i = 1; i < nr_pages; i++) {
|
|
|
|
struct page *p = page + i;
|
2011-12-08 15:34:18 -07:00
|
|
|
set_page_count(p, 0);
|
2016-01-15 17:52:07 -07:00
|
|
|
p->mapping = TAIL_MAPPING;
|
2015-11-06 17:29:54 -07:00
|
|
|
set_compound_head(p, page);
|
2008-11-06 13:53:27 -07:00
|
|
|
}
|
2016-01-15 17:53:42 -07:00
|
|
|
atomic_set(compound_mapcount_ptr(page), -1);
|
2008-11-06 13:53:27 -07:00
|
|
|
}
|
|
|
|
|
2012-01-10 16:07:28 -07:00
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
|
|
|
unsigned int _debug_guardpage_minorder;
|
2016-03-15 15:55:30 -06:00
|
|
|
bool _debug_pagealloc_enabled __read_mostly
|
|
|
|
= IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
|
2016-03-17 15:17:56 -06:00
|
|
|
EXPORT_SYMBOL(_debug_pagealloc_enabled);
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
bool _debug_guardpage_enabled __read_mostly;
|
|
|
|
|
2014-12-12 17:55:52 -07:00
|
|
|
static int __init early_debug_pagealloc(char *buf)
|
|
|
|
{
|
|
|
|
if (!buf)
|
|
|
|
return -EINVAL;
|
2016-05-20 17:58:13 -06:00
|
|
|
return kstrtobool(buf, &_debug_pagealloc_enabled);
|
2014-12-12 17:55:52 -07:00
|
|
|
}
|
|
|
|
early_param("debug_pagealloc", early_debug_pagealloc);
|
|
|
|
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
static bool need_debug_guardpage(void)
|
|
|
|
{
|
2014-12-12 17:55:52 -07:00
|
|
|
/* If we don't use debug_pagealloc, we don't need guard page */
|
|
|
|
if (!debug_pagealloc_enabled())
|
|
|
|
return false;
|
|
|
|
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void init_debug_guardpage(void)
|
|
|
|
{
|
2014-12-12 17:55:52 -07:00
|
|
|
if (!debug_pagealloc_enabled())
|
|
|
|
return;
|
|
|
|
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
_debug_guardpage_enabled = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct page_ext_operations debug_guardpage_ops = {
|
|
|
|
.need = need_debug_guardpage,
|
|
|
|
.init = init_debug_guardpage,
|
|
|
|
};
|
2012-01-10 16:07:28 -07:00
|
|
|
|
|
|
|
static int __init debug_guardpage_minorder_setup(char *buf)
|
|
|
|
{
|
|
|
|
unsigned long res;
|
|
|
|
|
|
|
|
if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
|
2016-03-17 15:19:50 -06:00
|
|
|
pr_err("Bad debug_guardpage_minorder value\n");
|
2012-01-10 16:07:28 -07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
_debug_guardpage_minorder = res;
|
2016-03-17 15:19:50 -06:00
|
|
|
pr_info("Setting debug_guardpage_minorder to %lu\n", res);
|
2012-01-10 16:07:28 -07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
|
|
|
|
|
2014-12-12 17:55:01 -07:00
|
|
|
static inline void set_page_guard(struct zone *zone, struct page *page,
|
|
|
|
unsigned int order, int migratetype)
|
2012-01-10 16:07:28 -07:00
|
|
|
{
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
struct page_ext *page_ext;
|
|
|
|
|
|
|
|
if (!debug_guardpage_enabled())
|
|
|
|
return;
|
|
|
|
|
|
|
|
page_ext = lookup_page_ext(page);
|
2016-06-03 15:55:38 -06:00
|
|
|
if (unlikely(!page_ext))
|
|
|
|
return;
|
|
|
|
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
__set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
|
|
|
|
|
2014-12-12 17:55:01 -07:00
|
|
|
INIT_LIST_HEAD(&page->lru);
|
|
|
|
set_page_private(page, order);
|
|
|
|
/* Guard pages are not available for any usage */
|
|
|
|
__mod_zone_freepage_state(zone, -(1 << order), migratetype);
|
2012-01-10 16:07:28 -07:00
|
|
|
}
|
|
|
|
|
2014-12-12 17:55:01 -07:00
|
|
|
static inline void clear_page_guard(struct zone *zone, struct page *page,
|
|
|
|
unsigned int order, int migratetype)
|
2012-01-10 16:07:28 -07:00
|
|
|
{
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
struct page_ext *page_ext;
|
|
|
|
|
|
|
|
if (!debug_guardpage_enabled())
|
|
|
|
return;
|
|
|
|
|
|
|
|
page_ext = lookup_page_ext(page);
|
2016-06-03 15:55:38 -06:00
|
|
|
if (unlikely(!page_ext))
|
|
|
|
return;
|
|
|
|
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
__clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
|
|
|
|
|
2014-12-12 17:55:01 -07:00
|
|
|
set_page_private(page, 0);
|
|
|
|
if (!is_migrate_isolate(migratetype))
|
|
|
|
__mod_zone_freepage_state(zone, (1 << order), migratetype);
|
2012-01-10 16:07:28 -07:00
|
|
|
}
|
|
|
|
#else
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
struct page_ext_operations debug_guardpage_ops = { NULL, };
|
2014-12-12 17:55:01 -07:00
|
|
|
static inline void set_page_guard(struct zone *zone, struct page *page,
|
|
|
|
unsigned int order, int migratetype) {}
|
|
|
|
static inline void clear_page_guard(struct zone *zone, struct page *page,
|
|
|
|
unsigned int order, int migratetype) {}
|
2012-01-10 16:07:28 -07:00
|
|
|
#endif
|
|
|
|
|
2014-06-04 17:10:21 -06:00
|
|
|
static inline void set_page_order(struct page *page, unsigned int order)
|
2006-04-18 23:20:52 -06:00
|
|
|
{
|
[PATCH] mm: split page table lock
Christoph Lameter demonstrated very poor scalability on the SGI 512-way, with
a many-threaded application which concurrently initializes different parts of
a large anonymous area.
This patch corrects that, by using a separate spinlock per page table page, to
guard the page table entries in that page, instead of using the mm's single
page_table_lock. (But even then, page_table_lock is still used to guard page
table allocation, and anon_vma allocation.)
In this implementation, the spinlock is tucked inside the struct page of the
page table page: with a BUILD_BUG_ON in case it overflows - which it would in
the case of 32-bit PA-RISC with spinlock debugging enabled.
Splitting the lock is not quite for free: another cacheline access. Ideally,
I suppose we would use split ptlock only for multi-threaded processes on
multi-cpu machines; but deciding that dynamically would have its own costs.
So for now enable it by config, at some number of cpus - since the Kconfig
language doesn't support inequalities, let preprocessor compare that with
NR_CPUS. But I don't think it's worth being user-configurable: for good
testing of both split and unsplit configs, split now at 4 cpus, and perhaps
change that to 8 later.
There is a benefit even for singly threaded processes: kswapd can be attacking
one part of the mm while another part is busy faulting.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-29 19:16:40 -06:00
|
|
|
set_page_private(page, order);
|
2006-04-09 19:21:48 -06:00
|
|
|
__SetPageBuddy(page);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void rmv_page_order(struct page *page)
|
|
|
|
{
|
2006-04-09 19:21:48 -06:00
|
|
|
__ClearPageBuddy(page);
|
[PATCH] mm: split page table lock
Christoph Lameter demonstrated very poor scalability on the SGI 512-way, with
a many-threaded application which concurrently initializes different parts of
a large anonymous area.
This patch corrects that, by using a separate spinlock per page table page, to
guard the page table entries in that page, instead of using the mm's single
page_table_lock. (But even then, page_table_lock is still used to guard page
table allocation, and anon_vma allocation.)
In this implementation, the spinlock is tucked inside the struct page of the
page table page: with a BUILD_BUG_ON in case it overflows - which it would in
the case of 32-bit PA-RISC with spinlock debugging enabled.
Splitting the lock is not quite for free: another cacheline access. Ideally,
I suppose we would use split ptlock only for multi-threaded processes on
multi-cpu machines; but deciding that dynamically would have its own costs.
So for now enable it by config, at some number of cpus - since the Kconfig
language doesn't support inequalities, let preprocessor compare that with
NR_CPUS. But I don't think it's worth being user-configurable: for good
testing of both split and unsplit configs, split now at 4 cpus, and perhaps
change that to 8 later.
There is a benefit even for singly threaded processes: kswapd can be attacking
one part of the mm while another part is busy faulting.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-29 19:16:40 -06:00
|
|
|
set_page_private(page, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This function checks whether a page is free && is the buddy
|
|
|
|
* we can do coalesce a page and its buddy if
|
2006-01-06 01:10:58 -07:00
|
|
|
* (a) the buddy is not in a hole &&
|
2006-04-09 19:21:48 -06:00
|
|
|
* (b) the buddy is in the buddy system &&
|
2006-06-23 03:03:01 -06:00
|
|
|
* (c) a page and its buddy have the same order &&
|
|
|
|
* (d) a page and its buddy are in the same zone.
|
2006-04-09 19:21:48 -06:00
|
|
|
*
|
2013-09-11 15:22:48 -06:00
|
|
|
* For recording whether a page is in the buddy system, we set ->_mapcount
|
|
|
|
* PAGE_BUDDY_MAPCOUNT_VALUE.
|
|
|
|
* Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is
|
|
|
|
* serialized by zone->lock.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
2006-04-09 19:21:48 -06:00
|
|
|
* For recording page's order, we use page_private(page).
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2006-06-23 03:03:01 -06:00
|
|
|
static inline int page_is_buddy(struct page *page, struct page *buddy,
|
2014-06-04 17:10:21 -06:00
|
|
|
unsigned int order)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-05-06 15:49:14 -06:00
|
|
|
if (!pfn_valid_within(page_to_pfn(buddy)))
|
2006-01-06 01:10:58 -07:00
|
|
|
return 0;
|
|
|
|
|
2012-01-10 16:07:28 -07:00
|
|
|
if (page_is_guard(buddy) && page_order(buddy) == order) {
|
2014-06-04 17:10:10 -06:00
|
|
|
if (page_zone_id(page) != page_zone_id(buddy))
|
|
|
|
return 0;
|
|
|
|
|
2015-02-10 15:11:39 -07:00
|
|
|
VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
|
|
|
|
|
2012-01-10 16:07:28 -07:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2006-06-23 03:03:01 -06:00
|
|
|
if (PageBuddy(buddy) && page_order(buddy) == order) {
|
2014-06-04 17:10:10 -06:00
|
|
|
/*
|
|
|
|
* zone check is done late to avoid uselessly
|
|
|
|
* calculating zone/node ids for pages that could
|
|
|
|
* never merge.
|
|
|
|
*/
|
|
|
|
if (page_zone_id(page) != page_zone_id(buddy))
|
|
|
|
return 0;
|
|
|
|
|
2015-02-10 15:11:39 -07:00
|
|
|
VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
|
|
|
|
|
2006-04-18 23:20:52 -06:00
|
|
|
return 1;
|
2006-04-09 19:21:48 -06:00
|
|
|
}
|
2006-04-18 23:20:52 -06:00
|
|
|
return 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Freeing function for a buddy system allocator.
|
|
|
|
*
|
|
|
|
* The concept of a buddy system is to maintain direct-mapped table
|
|
|
|
* (containing bit values) for memory blocks of various "orders".
|
|
|
|
* The bottom level table contains the map for the smallest allocatable
|
|
|
|
* units of memory (here, pages), and each level above it describes
|
|
|
|
* pairs of units from the levels below, hence, "buddies".
|
|
|
|
* At a high level, all that happens here is marking the table entry
|
|
|
|
* at the bottom level available, and propagating the changes upward
|
|
|
|
* as necessary, plus some accounting needed to play nicely with other
|
|
|
|
* parts of the VM system.
|
|
|
|
* At each level, we keep a list of pages, which are heads of continuous
|
2013-09-11 15:22:48 -06:00
|
|
|
* free pages of length of (1 << order) and marked with _mapcount
|
|
|
|
* PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page)
|
|
|
|
* field.
|
2005-04-16 16:20:36 -06:00
|
|
|
* So when we are allocating or freeing one, we can derive the state of the
|
2012-01-11 07:16:11 -07:00
|
|
|
* other. That is, if we allocate a small block, and both were
|
|
|
|
* free, the remainder of the region must be split into blocks.
|
2005-04-16 16:20:36 -06:00
|
|
|
* If a block is freed, and its buddy is also free, then this
|
2012-01-11 07:16:11 -07:00
|
|
|
* triggers coalescing into a block of larger size.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
2012-12-06 02:39:54 -07:00
|
|
|
* -- nyc
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
|
2006-01-08 02:00:42 -07:00
|
|
|
static inline void __free_one_page(struct page *page,
|
2014-06-04 17:10:17 -06:00
|
|
|
unsigned long pfn,
|
2009-06-16 16:32:07 -06:00
|
|
|
struct zone *zone, unsigned int order,
|
|
|
|
int migratetype)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
unsigned long page_idx;
|
page allocator: reduce fragmentation in buddy allocator by adding buddies that are merging to the tail of the free lists
In order to reduce fragmentation, this patch classifies freed pages in two
groups according to their probability of being part of a high order merge.
Pages belonging to a compound whose next-highest buddy is free are more
likely to be part of a high order merge in the near future, so they will
be added at the tail of the freelist. The remaining pages are put at the
front of the freelist.
In this way, the pages that are more likely to cause a big merge are kept
free longer. Consequently there is a tendency to aggregate the
long-living allocations on a subset of the compounds, reducing the
fragmentation.
This heuristic was tested on three machines, x86, x86-64 and ppc64 with
3GB of RAM in each machine. The tests were kernbench, netperf, sysbench
and STREAM for performance and a high-order stress test for huge page
allocations.
KernBench X86
Elapsed mean 374.77 ( 0.00%) 375.10 (-0.09%)
User mean 649.53 ( 0.00%) 650.44 (-0.14%)
System mean 54.75 ( 0.00%) 54.18 ( 1.05%)
CPU mean 187.75 ( 0.00%) 187.25 ( 0.27%)
KernBench X86-64
Elapsed mean 94.45 ( 0.00%) 94.01 ( 0.47%)
User mean 323.27 ( 0.00%) 322.66 ( 0.19%)
System mean 36.71 ( 0.00%) 36.50 ( 0.57%)
CPU mean 380.75 ( 0.00%) 381.75 (-0.26%)
KernBench PPC64
Elapsed mean 173.45 ( 0.00%) 173.74 (-0.17%)
User mean 587.99 ( 0.00%) 587.95 ( 0.01%)
System mean 60.60 ( 0.00%) 60.57 ( 0.05%)
CPU mean 373.50 ( 0.00%) 372.75 ( 0.20%)
Nothing notable for kernbench.
NetPerf UDP X86
64 42.68 ( 0.00%) 42.77 ( 0.21%)
128 85.62 ( 0.00%) 85.32 (-0.35%)
256 170.01 ( 0.00%) 168.76 (-0.74%)
1024 655.68 ( 0.00%) 652.33 (-0.51%)
2048 1262.39 ( 0.00%) 1248.61 (-1.10%)
3312 1958.41 ( 0.00%) 1944.61 (-0.71%)
4096 2345.63 ( 0.00%) 2318.83 (-1.16%)
8192 4132.90 ( 0.00%) 4089.50 (-1.06%)
16384 6770.88 ( 0.00%) 6642.05 (-1.94%)*
NetPerf UDP X86-64
64 148.82 ( 0.00%) 154.92 ( 3.94%)
128 298.96 ( 0.00%) 312.95 ( 4.47%)
256 583.67 ( 0.00%) 626.39 ( 6.82%)
1024 2293.18 ( 0.00%) 2371.10 ( 3.29%)
2048 4274.16 ( 0.00%) 4396.83 ( 2.79%)
3312 6356.94 ( 0.00%) 6571.35 ( 3.26%)
4096 7422.68 ( 0.00%) 7635.42 ( 2.79%)*
8192 12114.81 ( 0.00%)* 12346.88 ( 1.88%)
16384 17022.28 ( 0.00%)* 17033.19 ( 0.06%)*
1.64% 2.73%
NetPerf UDP PPC64
64 49.98 ( 0.00%) 50.25 ( 0.54%)
128 98.66 ( 0.00%) 100.95 ( 2.27%)
256 197.33 ( 0.00%) 191.03 (-3.30%)
1024 761.98 ( 0.00%) 785.07 ( 2.94%)
2048 1493.50 ( 0.00%) 1510.85 ( 1.15%)
3312 2303.95 ( 0.00%) 2271.72 (-1.42%)
4096 2774.56 ( 0.00%) 2773.06 (-0.05%)
8192 4918.31 ( 0.00%) 4793.59 (-2.60%)
16384 7497.98 ( 0.00%) 7749.52 ( 3.25%)
The tests are run to have confidence limits within 1%. Results marked
with a * were not confident although in this case, it's only outside by
small amounts. Even with some results that were not confident, the
netperf UDP results were generally positive.
NetPerf TCP X86
64 652.25 ( 0.00%)* 648.12 (-0.64%)*
23.80% 22.82%
128 1229.98 ( 0.00%)* 1220.56 (-0.77%)*
21.03% 18.90%
256 2105.88 ( 0.00%) 1872.03 (-12.49%)*
1.00% 16.46%
1024 3476.46 ( 0.00%)* 3548.28 ( 2.02%)*
13.37% 11.39%
2048 4023.44 ( 0.00%)* 4231.45 ( 4.92%)*
9.76% 12.48%
3312 4348.88 ( 0.00%)* 4396.96 ( 1.09%)*
6.49% 8.75%
4096 4726.56 ( 0.00%)* 4877.71 ( 3.10%)*
9.85% 8.50%
8192 4732.28 ( 0.00%)* 5777.77 (18.10%)*
9.13% 13.04%
16384 5543.05 ( 0.00%)* 5906.24 ( 6.15%)*
7.73% 8.68%
NETPERF TCP X86-64
netperf-tcp-vanilla-netperf netperf-tcp
tcp-vanilla pgalloc-delay
64 1895.87 ( 0.00%)* 1775.07 (-6.81%)*
5.79% 4.78%
128 3571.03 ( 0.00%)* 3342.20 (-6.85%)*
3.68% 6.06%
256 5097.21 ( 0.00%)* 4859.43 (-4.89%)*
3.02% 2.10%
1024 8919.10 ( 0.00%)* 8892.49 (-0.30%)*
5.89% 6.55%
2048 10255.46 ( 0.00%)* 10449.39 ( 1.86%)*
7.08% 7.44%
3312 10839.90 ( 0.00%)* 10740.15 (-0.93%)*
6.87% 7.33%
4096 10814.84 ( 0.00%)* 10766.97 (-0.44%)*
6.86% 8.18%
8192 11606.89 ( 0.00%)* 11189.28 (-3.73%)*
7.49% 5.55%
16384 12554.88 ( 0.00%)* 12361.22 (-1.57%)*
7.36% 6.49%
NETPERF TCP PPC64
netperf-tcp-vanilla-netperf netperf-tcp
tcp-vanilla pgalloc-delay
64 594.17 ( 0.00%) 596.04 ( 0.31%)*
1.00% 2.29%
128 1064.87 ( 0.00%)* 1074.77 ( 0.92%)*
1.30% 1.40%
256 1852.46 ( 0.00%)* 1856.95 ( 0.24%)
1.25% 1.00%
1024 3839.46 ( 0.00%)* 3813.05 (-0.69%)
1.02% 1.00%
2048 4885.04 ( 0.00%)* 4881.97 (-0.06%)*
1.15% 1.04%
3312 5506.90 ( 0.00%) 5459.72 (-0.86%)
4096 6449.19 ( 0.00%) 6345.46 (-1.63%)
8192 7501.17 ( 0.00%) 7508.79 ( 0.10%)
16384 9618.65 ( 0.00%) 9490.10 (-1.35%)
There was a distinct lack of confidence in the X86* figures so I included
what the devation was where the results were not confident. Many of the
results, whether gains or losses were within the standard deviation so no
solid conclusion can be reached on performance impact. Looking at the
figures, only the X86-64 ones look suspicious with a few losses that were
outside the noise. However, the results were so unstable that without
knowing why they vary so much, a solid conclusion cannot be reached.
SYSBENCH X86
sysbench-vanilla pgalloc-delay
1 7722.85 ( 0.00%) 7756.79 ( 0.44%)
2 14901.11 ( 0.00%) 13683.44 (-8.90%)
3 15171.71 ( 0.00%) 14888.25 (-1.90%)
4 14966.98 ( 0.00%) 15029.67 ( 0.42%)
5 14370.47 ( 0.00%) 14865.00 ( 3.33%)
6 14870.33 ( 0.00%) 14845.57 (-0.17%)
7 14429.45 ( 0.00%) 14520.85 ( 0.63%)
8 14354.35 ( 0.00%) 14362.31 ( 0.06%)
SYSBENCH X86-64
1 17448.70 ( 0.00%) 17484.41 ( 0.20%)
2 34276.39 ( 0.00%) 34251.00 (-0.07%)
3 50805.25 ( 0.00%) 50854.80 ( 0.10%)
4 66667.10 ( 0.00%) 66174.69 (-0.74%)
5 66003.91 ( 0.00%) 65685.25 (-0.49%)
6 64981.90 ( 0.00%) 65125.60 ( 0.22%)
7 64933.16 ( 0.00%) 64379.23 (-0.86%)
8 63353.30 ( 0.00%) 63281.22 (-0.11%)
9 63511.84 ( 0.00%) 63570.37 ( 0.09%)
10 62708.27 ( 0.00%) 63166.25 ( 0.73%)
11 62092.81 ( 0.00%) 61787.75 (-0.49%)
12 61330.11 ( 0.00%) 61036.34 (-0.48%)
13 61438.37 ( 0.00%) 61994.47 ( 0.90%)
14 62304.48 ( 0.00%) 62064.90 (-0.39%)
15 63296.48 ( 0.00%) 62875.16 (-0.67%)
16 63951.76 ( 0.00%) 63769.09 (-0.29%)
SYSBENCH PPC64
-sysbench-pgalloc-delay-sysbench
sysbench-vanilla pgalloc-delay
1 7645.08 ( 0.00%) 7467.43 (-2.38%)
2 14856.67 ( 0.00%) 14558.73 (-2.05%)
3 21952.31 ( 0.00%) 21683.64 (-1.24%)
4 27946.09 ( 0.00%) 28623.29 ( 2.37%)
5 28045.11 ( 0.00%) 28143.69 ( 0.35%)
6 27477.10 ( 0.00%) 27337.45 (-0.51%)
7 26489.17 ( 0.00%) 26590.06 ( 0.38%)
8 26642.91 ( 0.00%) 25274.33 (-5.41%)
9 25137.27 ( 0.00%) 24810.06 (-1.32%)
10 24451.99 ( 0.00%) 24275.85 (-0.73%)
11 23262.20 ( 0.00%) 23674.88 ( 1.74%)
12 24234.81 ( 0.00%) 23640.89 (-2.51%)
13 24577.75 ( 0.00%) 24433.50 (-0.59%)
14 25640.19 ( 0.00%) 25116.52 (-2.08%)
15 26188.84 ( 0.00%) 26181.36 (-0.03%)
16 26782.37 ( 0.00%) 26255.99 (-2.00%)
Again, there is little to conclude here. While there are a few losses,
the results vary by +/- 8% in some cases. They are the results of most
concern as there are some large losses but it's also within the variance
typically seen between kernel releases.
The STREAM results varied so little and are so verbose that I didn't
include them here.
The final test stressed how many huge pages can be allocated. The
absolute number of huge pages allocated are the same with or without the
page. However, the "unusability free space index" which is a measure of
external fragmentation was slightly lower (lower is better) throughout the
lifetime of the system. I also measured the latency of how long it took
to successfully allocate a huge page. The latency was slightly lower and
on X86 and PPC64, more huge pages were allocated almost immediately from
the free lists. The improvement is slight but there.
[mel@csn.ul.ie: Tested, reworked for less branches]
[czoccolo@gmail.com: fix oops by checking pfn_valid_within()]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Acked-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Corrado Zoccolo <czoccolo@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-24 15:31:54 -06:00
|
|
|
unsigned long combined_idx;
|
2011-01-13 16:47:24 -07:00
|
|
|
unsigned long uninitialized_var(buddy_idx);
|
page allocator: reduce fragmentation in buddy allocator by adding buddies that are merging to the tail of the free lists
In order to reduce fragmentation, this patch classifies freed pages in two
groups according to their probability of being part of a high order merge.
Pages belonging to a compound whose next-highest buddy is free are more
likely to be part of a high order merge in the near future, so they will
be added at the tail of the freelist. The remaining pages are put at the
front of the freelist.
In this way, the pages that are more likely to cause a big merge are kept
free longer. Consequently there is a tendency to aggregate the
long-living allocations on a subset of the compounds, reducing the
fragmentation.
This heuristic was tested on three machines, x86, x86-64 and ppc64 with
3GB of RAM in each machine. The tests were kernbench, netperf, sysbench
and STREAM for performance and a high-order stress test for huge page
allocations.
KernBench X86
Elapsed mean 374.77 ( 0.00%) 375.10 (-0.09%)
User mean 649.53 ( 0.00%) 650.44 (-0.14%)
System mean 54.75 ( 0.00%) 54.18 ( 1.05%)
CPU mean 187.75 ( 0.00%) 187.25 ( 0.27%)
KernBench X86-64
Elapsed mean 94.45 ( 0.00%) 94.01 ( 0.47%)
User mean 323.27 ( 0.00%) 322.66 ( 0.19%)
System mean 36.71 ( 0.00%) 36.50 ( 0.57%)
CPU mean 380.75 ( 0.00%) 381.75 (-0.26%)
KernBench PPC64
Elapsed mean 173.45 ( 0.00%) 173.74 (-0.17%)
User mean 587.99 ( 0.00%) 587.95 ( 0.01%)
System mean 60.60 ( 0.00%) 60.57 ( 0.05%)
CPU mean 373.50 ( 0.00%) 372.75 ( 0.20%)
Nothing notable for kernbench.
NetPerf UDP X86
64 42.68 ( 0.00%) 42.77 ( 0.21%)
128 85.62 ( 0.00%) 85.32 (-0.35%)
256 170.01 ( 0.00%) 168.76 (-0.74%)
1024 655.68 ( 0.00%) 652.33 (-0.51%)
2048 1262.39 ( 0.00%) 1248.61 (-1.10%)
3312 1958.41 ( 0.00%) 1944.61 (-0.71%)
4096 2345.63 ( 0.00%) 2318.83 (-1.16%)
8192 4132.90 ( 0.00%) 4089.50 (-1.06%)
16384 6770.88 ( 0.00%) 6642.05 (-1.94%)*
NetPerf UDP X86-64
64 148.82 ( 0.00%) 154.92 ( 3.94%)
128 298.96 ( 0.00%) 312.95 ( 4.47%)
256 583.67 ( 0.00%) 626.39 ( 6.82%)
1024 2293.18 ( 0.00%) 2371.10 ( 3.29%)
2048 4274.16 ( 0.00%) 4396.83 ( 2.79%)
3312 6356.94 ( 0.00%) 6571.35 ( 3.26%)
4096 7422.68 ( 0.00%) 7635.42 ( 2.79%)*
8192 12114.81 ( 0.00%)* 12346.88 ( 1.88%)
16384 17022.28 ( 0.00%)* 17033.19 ( 0.06%)*
1.64% 2.73%
NetPerf UDP PPC64
64 49.98 ( 0.00%) 50.25 ( 0.54%)
128 98.66 ( 0.00%) 100.95 ( 2.27%)
256 197.33 ( 0.00%) 191.03 (-3.30%)
1024 761.98 ( 0.00%) 785.07 ( 2.94%)
2048 1493.50 ( 0.00%) 1510.85 ( 1.15%)
3312 2303.95 ( 0.00%) 2271.72 (-1.42%)
4096 2774.56 ( 0.00%) 2773.06 (-0.05%)
8192 4918.31 ( 0.00%) 4793.59 (-2.60%)
16384 7497.98 ( 0.00%) 7749.52 ( 3.25%)
The tests are run to have confidence limits within 1%. Results marked
with a * were not confident although in this case, it's only outside by
small amounts. Even with some results that were not confident, the
netperf UDP results were generally positive.
NetPerf TCP X86
64 652.25 ( 0.00%)* 648.12 (-0.64%)*
23.80% 22.82%
128 1229.98 ( 0.00%)* 1220.56 (-0.77%)*
21.03% 18.90%
256 2105.88 ( 0.00%) 1872.03 (-12.49%)*
1.00% 16.46%
1024 3476.46 ( 0.00%)* 3548.28 ( 2.02%)*
13.37% 11.39%
2048 4023.44 ( 0.00%)* 4231.45 ( 4.92%)*
9.76% 12.48%
3312 4348.88 ( 0.00%)* 4396.96 ( 1.09%)*
6.49% 8.75%
4096 4726.56 ( 0.00%)* 4877.71 ( 3.10%)*
9.85% 8.50%
8192 4732.28 ( 0.00%)* 5777.77 (18.10%)*
9.13% 13.04%
16384 5543.05 ( 0.00%)* 5906.24 ( 6.15%)*
7.73% 8.68%
NETPERF TCP X86-64
netperf-tcp-vanilla-netperf netperf-tcp
tcp-vanilla pgalloc-delay
64 1895.87 ( 0.00%)* 1775.07 (-6.81%)*
5.79% 4.78%
128 3571.03 ( 0.00%)* 3342.20 (-6.85%)*
3.68% 6.06%
256 5097.21 ( 0.00%)* 4859.43 (-4.89%)*
3.02% 2.10%
1024 8919.10 ( 0.00%)* 8892.49 (-0.30%)*
5.89% 6.55%
2048 10255.46 ( 0.00%)* 10449.39 ( 1.86%)*
7.08% 7.44%
3312 10839.90 ( 0.00%)* 10740.15 (-0.93%)*
6.87% 7.33%
4096 10814.84 ( 0.00%)* 10766.97 (-0.44%)*
6.86% 8.18%
8192 11606.89 ( 0.00%)* 11189.28 (-3.73%)*
7.49% 5.55%
16384 12554.88 ( 0.00%)* 12361.22 (-1.57%)*
7.36% 6.49%
NETPERF TCP PPC64
netperf-tcp-vanilla-netperf netperf-tcp
tcp-vanilla pgalloc-delay
64 594.17 ( 0.00%) 596.04 ( 0.31%)*
1.00% 2.29%
128 1064.87 ( 0.00%)* 1074.77 ( 0.92%)*
1.30% 1.40%
256 1852.46 ( 0.00%)* 1856.95 ( 0.24%)
1.25% 1.00%
1024 3839.46 ( 0.00%)* 3813.05 (-0.69%)
1.02% 1.00%
2048 4885.04 ( 0.00%)* 4881.97 (-0.06%)*
1.15% 1.04%
3312 5506.90 ( 0.00%) 5459.72 (-0.86%)
4096 6449.19 ( 0.00%) 6345.46 (-1.63%)
8192 7501.17 ( 0.00%) 7508.79 ( 0.10%)
16384 9618.65 ( 0.00%) 9490.10 (-1.35%)
There was a distinct lack of confidence in the X86* figures so I included
what the devation was where the results were not confident. Many of the
results, whether gains or losses were within the standard deviation so no
solid conclusion can be reached on performance impact. Looking at the
figures, only the X86-64 ones look suspicious with a few losses that were
outside the noise. However, the results were so unstable that without
knowing why they vary so much, a solid conclusion cannot be reached.
SYSBENCH X86
sysbench-vanilla pgalloc-delay
1 7722.85 ( 0.00%) 7756.79 ( 0.44%)
2 14901.11 ( 0.00%) 13683.44 (-8.90%)
3 15171.71 ( 0.00%) 14888.25 (-1.90%)
4 14966.98 ( 0.00%) 15029.67 ( 0.42%)
5 14370.47 ( 0.00%) 14865.00 ( 3.33%)
6 14870.33 ( 0.00%) 14845.57 (-0.17%)
7 14429.45 ( 0.00%) 14520.85 ( 0.63%)
8 14354.35 ( 0.00%) 14362.31 ( 0.06%)
SYSBENCH X86-64
1 17448.70 ( 0.00%) 17484.41 ( 0.20%)
2 34276.39 ( 0.00%) 34251.00 (-0.07%)
3 50805.25 ( 0.00%) 50854.80 ( 0.10%)
4 66667.10 ( 0.00%) 66174.69 (-0.74%)
5 66003.91 ( 0.00%) 65685.25 (-0.49%)
6 64981.90 ( 0.00%) 65125.60 ( 0.22%)
7 64933.16 ( 0.00%) 64379.23 (-0.86%)
8 63353.30 ( 0.00%) 63281.22 (-0.11%)
9 63511.84 ( 0.00%) 63570.37 ( 0.09%)
10 62708.27 ( 0.00%) 63166.25 ( 0.73%)
11 62092.81 ( 0.00%) 61787.75 (-0.49%)
12 61330.11 ( 0.00%) 61036.34 (-0.48%)
13 61438.37 ( 0.00%) 61994.47 ( 0.90%)
14 62304.48 ( 0.00%) 62064.90 (-0.39%)
15 63296.48 ( 0.00%) 62875.16 (-0.67%)
16 63951.76 ( 0.00%) 63769.09 (-0.29%)
SYSBENCH PPC64
-sysbench-pgalloc-delay-sysbench
sysbench-vanilla pgalloc-delay
1 7645.08 ( 0.00%) 7467.43 (-2.38%)
2 14856.67 ( 0.00%) 14558.73 (-2.05%)
3 21952.31 ( 0.00%) 21683.64 (-1.24%)
4 27946.09 ( 0.00%) 28623.29 ( 2.37%)
5 28045.11 ( 0.00%) 28143.69 ( 0.35%)
6 27477.10 ( 0.00%) 27337.45 (-0.51%)
7 26489.17 ( 0.00%) 26590.06 ( 0.38%)
8 26642.91 ( 0.00%) 25274.33 (-5.41%)
9 25137.27 ( 0.00%) 24810.06 (-1.32%)
10 24451.99 ( 0.00%) 24275.85 (-0.73%)
11 23262.20 ( 0.00%) 23674.88 ( 1.74%)
12 24234.81 ( 0.00%) 23640.89 (-2.51%)
13 24577.75 ( 0.00%) 24433.50 (-0.59%)
14 25640.19 ( 0.00%) 25116.52 (-2.08%)
15 26188.84 ( 0.00%) 26181.36 (-0.03%)
16 26782.37 ( 0.00%) 26255.99 (-2.00%)
Again, there is little to conclude here. While there are a few losses,
the results vary by +/- 8% in some cases. They are the results of most
concern as there are some large losses but it's also within the variance
typically seen between kernel releases.
The STREAM results varied so little and are so verbose that I didn't
include them here.
The final test stressed how many huge pages can be allocated. The
absolute number of huge pages allocated are the same with or without the
page. However, the "unusability free space index" which is a measure of
external fragmentation was slightly lower (lower is better) throughout the
lifetime of the system. I also measured the latency of how long it took
to successfully allocate a huge page. The latency was slightly lower and
on X86 and PPC64, more huge pages were allocated almost immediately from
the free lists. The improvement is slight but there.
[mel@csn.ul.ie: Tested, reworked for less branches]
[czoccolo@gmail.com: fix oops by checking pfn_valid_within()]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Acked-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Corrado Zoccolo <czoccolo@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-24 15:31:54 -06:00
|
|
|
struct page *buddy;
|
mm/page_alloc: prevent merging between isolated and other pageblocks
Hanjun Guo has reported that a CMA stress test causes broken accounting of
CMA and free pages:
> Before the test, I got:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 195044 kB
>
>
> After running the test:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 6602584 kB
>
> So the freed CMA memory is more than total..
>
> Also the the MemFree is more than mem total:
>
> -bash-4.3# cat /proc/meminfo
> MemTotal: 16342016 kB
> MemFree: 22367268 kB
> MemAvailable: 22370528 kB
Laura Abbott has confirmed the issue and suspected the freepage accounting
rewrite around 3.18/4.0 by Joonsoo Kim. Joonsoo had a theory that this is
caused by unexpected merging between MIGRATE_ISOLATE and MIGRATE_CMA
pageblocks:
> CMA isolates MAX_ORDER aligned blocks, but, during the process,
> partialy isolated block exists. If MAX_ORDER is 11 and
> pageblock_order is 9, two pageblocks make up MAX_ORDER
> aligned block and I can think following scenario because pageblock
> (un)isolation would be done one by one.
>
> (each character means one pageblock. 'C', 'I' means MIGRATE_CMA,
> MIGRATE_ISOLATE, respectively.
>
> CC -> IC -> II (Isolation)
> II -> CI -> CC (Un-isolation)
>
> If some pages are freed at this intermediate state such as IC or CI,
> that page could be merged to the other page that is resident on
> different type of pageblock and it will cause wrong freepage count.
This was supposed to be prevented by CMA operating on MAX_ORDER blocks,
but since it doesn't hold the zone->lock between pageblocks, a race
window does exist.
It's also likely that unexpected merging can occur between
MIGRATE_ISOLATE and non-CMA pageblocks. This should be prevented in
__free_one_page() since commit 3c605096d315 ("mm/page_alloc: restrict
max order of merging on isolated pageblock"). However, we only check
the migratetype of the pageblock where buddy merging has been initiated,
not the migratetype of the buddy pageblock (or group of pageblocks)
which can be MIGRATE_ISOLATE.
Joonsoo has suggested checking for buddy migratetype as part of
page_is_buddy(), but that would add extra checks in allocator hotpath
and bloat-o-meter has shown significant code bloat (the function is
inline).
This patch reduces the bloat at some expense of more complicated code.
The buddy-merging while-loop in __free_one_page() is initially bounded
to pageblock_border and without any migratetype checks. The checks are
placed outside, bumping the max_order if merging is allowed, and
returning to the while-loop with a statement which can't be possibly
considered harmful.
This fixes the accounting bug and also removes the arguably weird state
in the original commit 3c605096d315 where buddies could be left
unmerged.
Fixes: 3c605096d315 ("mm/page_alloc: restrict max order of merging on isolated pageblock")
Link: https://lkml.org/lkml/2016/3/2/280
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reported-by: Hanjun Guo <guohanjun@huawei.com>
Tested-by: Hanjun Guo <guohanjun@huawei.com>
Acked-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Debugged-by: Laura Abbott <labbott@redhat.com>
Debugged-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: <stable@vger.kernel.org> [3.18+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-25 15:21:50 -06:00
|
|
|
unsigned int max_order;
|
|
|
|
|
|
|
|
max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-02-22 17:35:25 -07:00
|
|
|
VM_BUG_ON(!zone_is_initialized(zone));
|
2015-02-11 16:25:50 -07:00
|
|
|
VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-06-16 16:32:07 -06:00
|
|
|
VM_BUG_ON(migratetype == -1);
|
mm/page_alloc: prevent merging between isolated and other pageblocks
Hanjun Guo has reported that a CMA stress test causes broken accounting of
CMA and free pages:
> Before the test, I got:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 195044 kB
>
>
> After running the test:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 6602584 kB
>
> So the freed CMA memory is more than total..
>
> Also the the MemFree is more than mem total:
>
> -bash-4.3# cat /proc/meminfo
> MemTotal: 16342016 kB
> MemFree: 22367268 kB
> MemAvailable: 22370528 kB
Laura Abbott has confirmed the issue and suspected the freepage accounting
rewrite around 3.18/4.0 by Joonsoo Kim. Joonsoo had a theory that this is
caused by unexpected merging between MIGRATE_ISOLATE and MIGRATE_CMA
pageblocks:
> CMA isolates MAX_ORDER aligned blocks, but, during the process,
> partialy isolated block exists. If MAX_ORDER is 11 and
> pageblock_order is 9, two pageblocks make up MAX_ORDER
> aligned block and I can think following scenario because pageblock
> (un)isolation would be done one by one.
>
> (each character means one pageblock. 'C', 'I' means MIGRATE_CMA,
> MIGRATE_ISOLATE, respectively.
>
> CC -> IC -> II (Isolation)
> II -> CI -> CC (Un-isolation)
>
> If some pages are freed at this intermediate state such as IC or CI,
> that page could be merged to the other page that is resident on
> different type of pageblock and it will cause wrong freepage count.
This was supposed to be prevented by CMA operating on MAX_ORDER blocks,
but since it doesn't hold the zone->lock between pageblocks, a race
window does exist.
It's also likely that unexpected merging can occur between
MIGRATE_ISOLATE and non-CMA pageblocks. This should be prevented in
__free_one_page() since commit 3c605096d315 ("mm/page_alloc: restrict
max order of merging on isolated pageblock"). However, we only check
the migratetype of the pageblock where buddy merging has been initiated,
not the migratetype of the buddy pageblock (or group of pageblocks)
which can be MIGRATE_ISOLATE.
Joonsoo has suggested checking for buddy migratetype as part of
page_is_buddy(), but that would add extra checks in allocator hotpath
and bloat-o-meter has shown significant code bloat (the function is
inline).
This patch reduces the bloat at some expense of more complicated code.
The buddy-merging while-loop in __free_one_page() is initially bounded
to pageblock_border and without any migratetype checks. The checks are
placed outside, bumping the max_order if merging is allowed, and
returning to the while-loop with a statement which can't be possibly
considered harmful.
This fixes the accounting bug and also removes the arguably weird state
in the original commit 3c605096d315 where buddies could be left
unmerged.
Fixes: 3c605096d315 ("mm/page_alloc: restrict max order of merging on isolated pageblock")
Link: https://lkml.org/lkml/2016/3/2/280
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reported-by: Hanjun Guo <guohanjun@huawei.com>
Tested-by: Hanjun Guo <guohanjun@huawei.com>
Acked-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Debugged-by: Laura Abbott <labbott@redhat.com>
Debugged-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: <stable@vger.kernel.org> [3.18+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-25 15:21:50 -06:00
|
|
|
if (likely(!is_migrate_isolate(migratetype)))
|
2014-11-13 16:19:18 -07:00
|
|
|
__mod_zone_freepage_state(zone, 1 << order, migratetype);
|
2009-06-16 16:32:07 -06:00
|
|
|
|
mm/page_alloc: prevent merging between isolated and other pageblocks
Hanjun Guo has reported that a CMA stress test causes broken accounting of
CMA and free pages:
> Before the test, I got:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 195044 kB
>
>
> After running the test:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 6602584 kB
>
> So the freed CMA memory is more than total..
>
> Also the the MemFree is more than mem total:
>
> -bash-4.3# cat /proc/meminfo
> MemTotal: 16342016 kB
> MemFree: 22367268 kB
> MemAvailable: 22370528 kB
Laura Abbott has confirmed the issue and suspected the freepage accounting
rewrite around 3.18/4.0 by Joonsoo Kim. Joonsoo had a theory that this is
caused by unexpected merging between MIGRATE_ISOLATE and MIGRATE_CMA
pageblocks:
> CMA isolates MAX_ORDER aligned blocks, but, during the process,
> partialy isolated block exists. If MAX_ORDER is 11 and
> pageblock_order is 9, two pageblocks make up MAX_ORDER
> aligned block and I can think following scenario because pageblock
> (un)isolation would be done one by one.
>
> (each character means one pageblock. 'C', 'I' means MIGRATE_CMA,
> MIGRATE_ISOLATE, respectively.
>
> CC -> IC -> II (Isolation)
> II -> CI -> CC (Un-isolation)
>
> If some pages are freed at this intermediate state such as IC or CI,
> that page could be merged to the other page that is resident on
> different type of pageblock and it will cause wrong freepage count.
This was supposed to be prevented by CMA operating on MAX_ORDER blocks,
but since it doesn't hold the zone->lock between pageblocks, a race
window does exist.
It's also likely that unexpected merging can occur between
MIGRATE_ISOLATE and non-CMA pageblocks. This should be prevented in
__free_one_page() since commit 3c605096d315 ("mm/page_alloc: restrict
max order of merging on isolated pageblock"). However, we only check
the migratetype of the pageblock where buddy merging has been initiated,
not the migratetype of the buddy pageblock (or group of pageblocks)
which can be MIGRATE_ISOLATE.
Joonsoo has suggested checking for buddy migratetype as part of
page_is_buddy(), but that would add extra checks in allocator hotpath
and bloat-o-meter has shown significant code bloat (the function is
inline).
This patch reduces the bloat at some expense of more complicated code.
The buddy-merging while-loop in __free_one_page() is initially bounded
to pageblock_border and without any migratetype checks. The checks are
placed outside, bumping the max_order if merging is allowed, and
returning to the while-loop with a statement which can't be possibly
considered harmful.
This fixes the accounting bug and also removes the arguably weird state
in the original commit 3c605096d315 where buddies could be left
unmerged.
Fixes: 3c605096d315 ("mm/page_alloc: restrict max order of merging on isolated pageblock")
Link: https://lkml.org/lkml/2016/3/2/280
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reported-by: Hanjun Guo <guohanjun@huawei.com>
Tested-by: Hanjun Guo <guohanjun@huawei.com>
Acked-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Debugged-by: Laura Abbott <labbott@redhat.com>
Debugged-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: <stable@vger.kernel.org> [3.18+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-25 15:21:50 -06:00
|
|
|
page_idx = pfn & ((1 << MAX_ORDER) - 1);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2014-01-23 16:52:54 -07:00
|
|
|
VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
|
|
|
|
VM_BUG_ON_PAGE(bad_range(zone, page), page);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
mm/page_alloc: prevent merging between isolated and other pageblocks
Hanjun Guo has reported that a CMA stress test causes broken accounting of
CMA and free pages:
> Before the test, I got:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 195044 kB
>
>
> After running the test:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 6602584 kB
>
> So the freed CMA memory is more than total..
>
> Also the the MemFree is more than mem total:
>
> -bash-4.3# cat /proc/meminfo
> MemTotal: 16342016 kB
> MemFree: 22367268 kB
> MemAvailable: 22370528 kB
Laura Abbott has confirmed the issue and suspected the freepage accounting
rewrite around 3.18/4.0 by Joonsoo Kim. Joonsoo had a theory that this is
caused by unexpected merging between MIGRATE_ISOLATE and MIGRATE_CMA
pageblocks:
> CMA isolates MAX_ORDER aligned blocks, but, during the process,
> partialy isolated block exists. If MAX_ORDER is 11 and
> pageblock_order is 9, two pageblocks make up MAX_ORDER
> aligned block and I can think following scenario because pageblock
> (un)isolation would be done one by one.
>
> (each character means one pageblock. 'C', 'I' means MIGRATE_CMA,
> MIGRATE_ISOLATE, respectively.
>
> CC -> IC -> II (Isolation)
> II -> CI -> CC (Un-isolation)
>
> If some pages are freed at this intermediate state such as IC or CI,
> that page could be merged to the other page that is resident on
> different type of pageblock and it will cause wrong freepage count.
This was supposed to be prevented by CMA operating on MAX_ORDER blocks,
but since it doesn't hold the zone->lock between pageblocks, a race
window does exist.
It's also likely that unexpected merging can occur between
MIGRATE_ISOLATE and non-CMA pageblocks. This should be prevented in
__free_one_page() since commit 3c605096d315 ("mm/page_alloc: restrict
max order of merging on isolated pageblock"). However, we only check
the migratetype of the pageblock where buddy merging has been initiated,
not the migratetype of the buddy pageblock (or group of pageblocks)
which can be MIGRATE_ISOLATE.
Joonsoo has suggested checking for buddy migratetype as part of
page_is_buddy(), but that would add extra checks in allocator hotpath
and bloat-o-meter has shown significant code bloat (the function is
inline).
This patch reduces the bloat at some expense of more complicated code.
The buddy-merging while-loop in __free_one_page() is initially bounded
to pageblock_border and without any migratetype checks. The checks are
placed outside, bumping the max_order if merging is allowed, and
returning to the while-loop with a statement which can't be possibly
considered harmful.
This fixes the accounting bug and also removes the arguably weird state
in the original commit 3c605096d315 where buddies could be left
unmerged.
Fixes: 3c605096d315 ("mm/page_alloc: restrict max order of merging on isolated pageblock")
Link: https://lkml.org/lkml/2016/3/2/280
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reported-by: Hanjun Guo <guohanjun@huawei.com>
Tested-by: Hanjun Guo <guohanjun@huawei.com>
Acked-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Debugged-by: Laura Abbott <labbott@redhat.com>
Debugged-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: <stable@vger.kernel.org> [3.18+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-25 15:21:50 -06:00
|
|
|
continue_merging:
|
2014-11-13 16:19:21 -07:00
|
|
|
while (order < max_order - 1) {
|
2011-01-13 16:47:24 -07:00
|
|
|
buddy_idx = __find_buddy_index(page_idx, order);
|
|
|
|
buddy = page + (buddy_idx - page_idx);
|
2006-06-23 03:03:01 -06:00
|
|
|
if (!page_is_buddy(page, buddy, order))
|
mm/page_alloc: prevent merging between isolated and other pageblocks
Hanjun Guo has reported that a CMA stress test causes broken accounting of
CMA and free pages:
> Before the test, I got:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 195044 kB
>
>
> After running the test:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 6602584 kB
>
> So the freed CMA memory is more than total..
>
> Also the the MemFree is more than mem total:
>
> -bash-4.3# cat /proc/meminfo
> MemTotal: 16342016 kB
> MemFree: 22367268 kB
> MemAvailable: 22370528 kB
Laura Abbott has confirmed the issue and suspected the freepage accounting
rewrite around 3.18/4.0 by Joonsoo Kim. Joonsoo had a theory that this is
caused by unexpected merging between MIGRATE_ISOLATE and MIGRATE_CMA
pageblocks:
> CMA isolates MAX_ORDER aligned blocks, but, during the process,
> partialy isolated block exists. If MAX_ORDER is 11 and
> pageblock_order is 9, two pageblocks make up MAX_ORDER
> aligned block and I can think following scenario because pageblock
> (un)isolation would be done one by one.
>
> (each character means one pageblock. 'C', 'I' means MIGRATE_CMA,
> MIGRATE_ISOLATE, respectively.
>
> CC -> IC -> II (Isolation)
> II -> CI -> CC (Un-isolation)
>
> If some pages are freed at this intermediate state such as IC or CI,
> that page could be merged to the other page that is resident on
> different type of pageblock and it will cause wrong freepage count.
This was supposed to be prevented by CMA operating on MAX_ORDER blocks,
but since it doesn't hold the zone->lock between pageblocks, a race
window does exist.
It's also likely that unexpected merging can occur between
MIGRATE_ISOLATE and non-CMA pageblocks. This should be prevented in
__free_one_page() since commit 3c605096d315 ("mm/page_alloc: restrict
max order of merging on isolated pageblock"). However, we only check
the migratetype of the pageblock where buddy merging has been initiated,
not the migratetype of the buddy pageblock (or group of pageblocks)
which can be MIGRATE_ISOLATE.
Joonsoo has suggested checking for buddy migratetype as part of
page_is_buddy(), but that would add extra checks in allocator hotpath
and bloat-o-meter has shown significant code bloat (the function is
inline).
This patch reduces the bloat at some expense of more complicated code.
The buddy-merging while-loop in __free_one_page() is initially bounded
to pageblock_border and without any migratetype checks. The checks are
placed outside, bumping the max_order if merging is allowed, and
returning to the while-loop with a statement which can't be possibly
considered harmful.
This fixes the accounting bug and also removes the arguably weird state
in the original commit 3c605096d315 where buddies could be left
unmerged.
Fixes: 3c605096d315 ("mm/page_alloc: restrict max order of merging on isolated pageblock")
Link: https://lkml.org/lkml/2016/3/2/280
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reported-by: Hanjun Guo <guohanjun@huawei.com>
Tested-by: Hanjun Guo <guohanjun@huawei.com>
Acked-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Debugged-by: Laura Abbott <labbott@redhat.com>
Debugged-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: <stable@vger.kernel.org> [3.18+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-25 15:21:50 -06:00
|
|
|
goto done_merging;
|
2012-01-10 16:07:28 -07:00
|
|
|
/*
|
|
|
|
* Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
|
|
|
|
* merge with it and move up one order.
|
|
|
|
*/
|
|
|
|
if (page_is_guard(buddy)) {
|
2014-12-12 17:55:01 -07:00
|
|
|
clear_page_guard(zone, buddy, order, migratetype);
|
2012-01-10 16:07:28 -07:00
|
|
|
} else {
|
|
|
|
list_del(&buddy->lru);
|
|
|
|
zone->free_area[order].nr_free--;
|
|
|
|
rmv_page_order(buddy);
|
|
|
|
}
|
2011-01-13 16:47:24 -07:00
|
|
|
combined_idx = buddy_idx & page_idx;
|
2005-04-16 16:20:36 -06:00
|
|
|
page = page + (combined_idx - page_idx);
|
|
|
|
page_idx = combined_idx;
|
|
|
|
order++;
|
|
|
|
}
|
mm/page_alloc: prevent merging between isolated and other pageblocks
Hanjun Guo has reported that a CMA stress test causes broken accounting of
CMA and free pages:
> Before the test, I got:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 195044 kB
>
>
> After running the test:
> -bash-4.3# cat /proc/meminfo | grep Cma
> CmaTotal: 204800 kB
> CmaFree: 6602584 kB
>
> So the freed CMA memory is more than total..
>
> Also the the MemFree is more than mem total:
>
> -bash-4.3# cat /proc/meminfo
> MemTotal: 16342016 kB
> MemFree: 22367268 kB
> MemAvailable: 22370528 kB
Laura Abbott has confirmed the issue and suspected the freepage accounting
rewrite around 3.18/4.0 by Joonsoo Kim. Joonsoo had a theory that this is
caused by unexpected merging between MIGRATE_ISOLATE and MIGRATE_CMA
pageblocks:
> CMA isolates MAX_ORDER aligned blocks, but, during the process,
> partialy isolated block exists. If MAX_ORDER is 11 and
> pageblock_order is 9, two pageblocks make up MAX_ORDER
> aligned block and I can think following scenario because pageblock
> (un)isolation would be done one by one.
>
> (each character means one pageblock. 'C', 'I' means MIGRATE_CMA,
> MIGRATE_ISOLATE, respectively.
>
> CC -> IC -> II (Isolation)
> II -> CI -> CC (Un-isolation)
>
> If some pages are freed at this intermediate state such as IC or CI,
> that page could be merged to the other page that is resident on
> different type of pageblock and it will cause wrong freepage count.
This was supposed to be prevented by CMA operating on MAX_ORDER blocks,
but since it doesn't hold the zone->lock between pageblocks, a race
window does exist.
It's also likely that unexpected merging can occur between
MIGRATE_ISOLATE and non-CMA pageblocks. This should be prevented in
__free_one_page() since commit 3c605096d315 ("mm/page_alloc: restrict
max order of merging on isolated pageblock"). However, we only check
the migratetype of the pageblock where buddy merging has been initiated,
not the migratetype of the buddy pageblock (or group of pageblocks)
which can be MIGRATE_ISOLATE.
Joonsoo has suggested checking for buddy migratetype as part of
page_is_buddy(), but that would add extra checks in allocator hotpath
and bloat-o-meter has shown significant code bloat (the function is
inline).
This patch reduces the bloat at some expense of more complicated code.
The buddy-merging while-loop in __free_one_page() is initially bounded
to pageblock_border and without any migratetype checks. The checks are
placed outside, bumping the max_order if merging is allowed, and
returning to the while-loop with a statement which can't be possibly
considered harmful.
This fixes the accounting bug and also removes the arguably weird state
in the original commit 3c605096d315 where buddies could be left
unmerged.
Fixes: 3c605096d315 ("mm/page_alloc: restrict max order of merging on isolated pageblock")
Link: https://lkml.org/lkml/2016/3/2/280
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reported-by: Hanjun Guo <guohanjun@huawei.com>
Tested-by: Hanjun Guo <guohanjun@huawei.com>
Acked-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Debugged-by: Laura Abbott <labbott@redhat.com>
Debugged-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: <stable@vger.kernel.org> [3.18+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-25 15:21:50 -06:00
|
|
|
if (max_order < MAX_ORDER) {
|
|
|
|
/* If we are here, it means order is >= pageblock_order.
|
|
|
|
* We want to prevent merge between freepages on isolate
|
|
|
|
* pageblock and normal pageblock. Without this, pageblock
|
|
|
|
* isolation could cause incorrect freepage or CMA accounting.
|
|
|
|
*
|
|
|
|
* We don't want to hit this code for the more frequent
|
|
|
|
* low-order merging.
|
|
|
|
*/
|
|
|
|
if (unlikely(has_isolate_pageblock(zone))) {
|
|
|
|
int buddy_mt;
|
|
|
|
|
|
|
|
buddy_idx = __find_buddy_index(page_idx, order);
|
|
|
|
buddy = page + (buddy_idx - page_idx);
|
|
|
|
buddy_mt = get_pageblock_migratetype(buddy);
|
|
|
|
|
|
|
|
if (migratetype != buddy_mt
|
|
|
|
&& (is_migrate_isolate(migratetype) ||
|
|
|
|
is_migrate_isolate(buddy_mt)))
|
|
|
|
goto done_merging;
|
|
|
|
}
|
|
|
|
max_order++;
|
|
|
|
goto continue_merging;
|
|
|
|
}
|
|
|
|
|
|
|
|
done_merging:
|
2005-04-16 16:20:36 -06:00
|
|
|
set_page_order(page, order);
|
page allocator: reduce fragmentation in buddy allocator by adding buddies that are merging to the tail of the free lists
In order to reduce fragmentation, this patch classifies freed pages in two
groups according to their probability of being part of a high order merge.
Pages belonging to a compound whose next-highest buddy is free are more
likely to be part of a high order merge in the near future, so they will
be added at the tail of the freelist. The remaining pages are put at the
front of the freelist.
In this way, the pages that are more likely to cause a big merge are kept
free longer. Consequently there is a tendency to aggregate the
long-living allocations on a subset of the compounds, reducing the
fragmentation.
This heuristic was tested on three machines, x86, x86-64 and ppc64 with
3GB of RAM in each machine. The tests were kernbench, netperf, sysbench
and STREAM for performance and a high-order stress test for huge page
allocations.
KernBench X86
Elapsed mean 374.77 ( 0.00%) 375.10 (-0.09%)
User mean 649.53 ( 0.00%) 650.44 (-0.14%)
System mean 54.75 ( 0.00%) 54.18 ( 1.05%)
CPU mean 187.75 ( 0.00%) 187.25 ( 0.27%)
KernBench X86-64
Elapsed mean 94.45 ( 0.00%) 94.01 ( 0.47%)
User mean 323.27 ( 0.00%) 322.66 ( 0.19%)
System mean 36.71 ( 0.00%) 36.50 ( 0.57%)
CPU mean 380.75 ( 0.00%) 381.75 (-0.26%)
KernBench PPC64
Elapsed mean 173.45 ( 0.00%) 173.74 (-0.17%)
User mean 587.99 ( 0.00%) 587.95 ( 0.01%)
System mean 60.60 ( 0.00%) 60.57 ( 0.05%)
CPU mean 373.50 ( 0.00%) 372.75 ( 0.20%)
Nothing notable for kernbench.
NetPerf UDP X86
64 42.68 ( 0.00%) 42.77 ( 0.21%)
128 85.62 ( 0.00%) 85.32 (-0.35%)
256 170.01 ( 0.00%) 168.76 (-0.74%)
1024 655.68 ( 0.00%) 652.33 (-0.51%)
2048 1262.39 ( 0.00%) 1248.61 (-1.10%)
3312 1958.41 ( 0.00%) 1944.61 (-0.71%)
4096 2345.63 ( 0.00%) 2318.83 (-1.16%)
8192 4132.90 ( 0.00%) 4089.50 (-1.06%)
16384 6770.88 ( 0.00%) 6642.05 (-1.94%)*
NetPerf UDP X86-64
64 148.82 ( 0.00%) 154.92 ( 3.94%)
128 298.96 ( 0.00%) 312.95 ( 4.47%)
256 583.67 ( 0.00%) 626.39 ( 6.82%)
1024 2293.18 ( 0.00%) 2371.10 ( 3.29%)
2048 4274.16 ( 0.00%) 4396.83 ( 2.79%)
3312 6356.94 ( 0.00%) 6571.35 ( 3.26%)
4096 7422.68 ( 0.00%) 7635.42 ( 2.79%)*
8192 12114.81 ( 0.00%)* 12346.88 ( 1.88%)
16384 17022.28 ( 0.00%)* 17033.19 ( 0.06%)*
1.64% 2.73%
NetPerf UDP PPC64
64 49.98 ( 0.00%) 50.25 ( 0.54%)
128 98.66 ( 0.00%) 100.95 ( 2.27%)
256 197.33 ( 0.00%) 191.03 (-3.30%)
1024 761.98 ( 0.00%) 785.07 ( 2.94%)
2048 1493.50 ( 0.00%) 1510.85 ( 1.15%)
3312 2303.95 ( 0.00%) 2271.72 (-1.42%)
4096 2774.56 ( 0.00%) 2773.06 (-0.05%)
8192 4918.31 ( 0.00%) 4793.59 (-2.60%)
16384 7497.98 ( 0.00%) 7749.52 ( 3.25%)
The tests are run to have confidence limits within 1%. Results marked
with a * were not confident although in this case, it's only outside by
small amounts. Even with some results that were not confident, the
netperf UDP results were generally positive.
NetPerf TCP X86
64 652.25 ( 0.00%)* 648.12 (-0.64%)*
23.80% 22.82%
128 1229.98 ( 0.00%)* 1220.56 (-0.77%)*
21.03% 18.90%
256 2105.88 ( 0.00%) 1872.03 (-12.49%)*
1.00% 16.46%
1024 3476.46 ( 0.00%)* 3548.28 ( 2.02%)*
13.37% 11.39%
2048 4023.44 ( 0.00%)* 4231.45 ( 4.92%)*
9.76% 12.48%
3312 4348.88 ( 0.00%)* 4396.96 ( 1.09%)*
6.49% 8.75%
4096 4726.56 ( 0.00%)* 4877.71 ( 3.10%)*
9.85% 8.50%
8192 4732.28 ( 0.00%)* 5777.77 (18.10%)*
9.13% 13.04%
16384 5543.05 ( 0.00%)* 5906.24 ( 6.15%)*
7.73% 8.68%
NETPERF TCP X86-64
netperf-tcp-vanilla-netperf netperf-tcp
tcp-vanilla pgalloc-delay
64 1895.87 ( 0.00%)* 1775.07 (-6.81%)*
5.79% 4.78%
128 3571.03 ( 0.00%)* 3342.20 (-6.85%)*
3.68% 6.06%
256 5097.21 ( 0.00%)* 4859.43 (-4.89%)*
3.02% 2.10%
1024 8919.10 ( 0.00%)* 8892.49 (-0.30%)*
5.89% 6.55%
2048 10255.46 ( 0.00%)* 10449.39 ( 1.86%)*
7.08% 7.44%
3312 10839.90 ( 0.00%)* 10740.15 (-0.93%)*
6.87% 7.33%
4096 10814.84 ( 0.00%)* 10766.97 (-0.44%)*
6.86% 8.18%
8192 11606.89 ( 0.00%)* 11189.28 (-3.73%)*
7.49% 5.55%
16384 12554.88 ( 0.00%)* 12361.22 (-1.57%)*
7.36% 6.49%
NETPERF TCP PPC64
netperf-tcp-vanilla-netperf netperf-tcp
tcp-vanilla pgalloc-delay
64 594.17 ( 0.00%) 596.04 ( 0.31%)*
1.00% 2.29%
128 1064.87 ( 0.00%)* 1074.77 ( 0.92%)*
1.30% 1.40%
256 1852.46 ( 0.00%)* 1856.95 ( 0.24%)
1.25% 1.00%
1024 3839.46 ( 0.00%)* 3813.05 (-0.69%)
1.02% 1.00%
2048 4885.04 ( 0.00%)* 4881.97 (-0.06%)*
1.15% 1.04%
3312 5506.90 ( 0.00%) 5459.72 (-0.86%)
4096 6449.19 ( 0.00%) 6345.46 (-1.63%)
8192 7501.17 ( 0.00%) 7508.79 ( 0.10%)
16384 9618.65 ( 0.00%) 9490.10 (-1.35%)
There was a distinct lack of confidence in the X86* figures so I included
what the devation was where the results were not confident. Many of the
results, whether gains or losses were within the standard deviation so no
solid conclusion can be reached on performance impact. Looking at the
figures, only the X86-64 ones look suspicious with a few losses that were
outside the noise. However, the results were so unstable that without
knowing why they vary so much, a solid conclusion cannot be reached.
SYSBENCH X86
sysbench-vanilla pgalloc-delay
1 7722.85 ( 0.00%) 7756.79 ( 0.44%)
2 14901.11 ( 0.00%) 13683.44 (-8.90%)
3 15171.71 ( 0.00%) 14888.25 (-1.90%)
4 14966.98 ( 0.00%) 15029.67 ( 0.42%)
5 14370.47 ( 0.00%) 14865.00 ( 3.33%)
6 14870.33 ( 0.00%) 14845.57 (-0.17%)
7 14429.45 ( 0.00%) 14520.85 ( 0.63%)
8 14354.35 ( 0.00%) 14362.31 ( 0.06%)
SYSBENCH X86-64
1 17448.70 ( 0.00%) 17484.41 ( 0.20%)
2 34276.39 ( 0.00%) 34251.00 (-0.07%)
3 50805.25 ( 0.00%) 50854.80 ( 0.10%)
4 66667.10 ( 0.00%) 66174.69 (-0.74%)
5 66003.91 ( 0.00%) 65685.25 (-0.49%)
6 64981.90 ( 0.00%) 65125.60 ( 0.22%)
7 64933.16 ( 0.00%) 64379.23 (-0.86%)
8 63353.30 ( 0.00%) 63281.22 (-0.11%)
9 63511.84 ( 0.00%) 63570.37 ( 0.09%)
10 62708.27 ( 0.00%) 63166.25 ( 0.73%)
11 62092.81 ( 0.00%) 61787.75 (-0.49%)
12 61330.11 ( 0.00%) 61036.34 (-0.48%)
13 61438.37 ( 0.00%) 61994.47 ( 0.90%)
14 62304.48 ( 0.00%) 62064.90 (-0.39%)
15 63296.48 ( 0.00%) 62875.16 (-0.67%)
16 63951.76 ( 0.00%) 63769.09 (-0.29%)
SYSBENCH PPC64
-sysbench-pgalloc-delay-sysbench
sysbench-vanilla pgalloc-delay
1 7645.08 ( 0.00%) 7467.43 (-2.38%)
2 14856.67 ( 0.00%) 14558.73 (-2.05%)
3 21952.31 ( 0.00%) 21683.64 (-1.24%)
4 27946.09 ( 0.00%) 28623.29 ( 2.37%)
5 28045.11 ( 0.00%) 28143.69 ( 0.35%)
6 27477.10 ( 0.00%) 27337.45 (-0.51%)
7 26489.17 ( 0.00%) 26590.06 ( 0.38%)
8 26642.91 ( 0.00%) 25274.33 (-5.41%)
9 25137.27 ( 0.00%) 24810.06 (-1.32%)
10 24451.99 ( 0.00%) 24275.85 (-0.73%)
11 23262.20 ( 0.00%) 23674.88 ( 1.74%)
12 24234.81 ( 0.00%) 23640.89 (-2.51%)
13 24577.75 ( 0.00%) 24433.50 (-0.59%)
14 25640.19 ( 0.00%) 25116.52 (-2.08%)
15 26188.84 ( 0.00%) 26181.36 (-0.03%)
16 26782.37 ( 0.00%) 26255.99 (-2.00%)
Again, there is little to conclude here. While there are a few losses,
the results vary by +/- 8% in some cases. They are the results of most
concern as there are some large losses but it's also within the variance
typically seen between kernel releases.
The STREAM results varied so little and are so verbose that I didn't
include them here.
The final test stressed how many huge pages can be allocated. The
absolute number of huge pages allocated are the same with or without the
page. However, the "unusability free space index" which is a measure of
external fragmentation was slightly lower (lower is better) throughout the
lifetime of the system. I also measured the latency of how long it took
to successfully allocate a huge page. The latency was slightly lower and
on X86 and PPC64, more huge pages were allocated almost immediately from
the free lists. The improvement is slight but there.
[mel@csn.ul.ie: Tested, reworked for less branches]
[czoccolo@gmail.com: fix oops by checking pfn_valid_within()]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Acked-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Corrado Zoccolo <czoccolo@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-24 15:31:54 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If this is not the largest possible page, check if the buddy
|
|
|
|
* of the next-highest order is free. If it is, it's possible
|
|
|
|
* that pages are being freed that will coalesce soon. In case,
|
|
|
|
* that is happening, add the free page to the tail of the list
|
|
|
|
* so it's less likely to be used soon and more likely to be merged
|
|
|
|
* as a higher order page
|
|
|
|
*/
|
2010-10-26 15:21:11 -06:00
|
|
|
if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
|
page allocator: reduce fragmentation in buddy allocator by adding buddies that are merging to the tail of the free lists
In order to reduce fragmentation, this patch classifies freed pages in two
groups according to their probability of being part of a high order merge.
Pages belonging to a compound whose next-highest buddy is free are more
likely to be part of a high order merge in the near future, so they will
be added at the tail of the freelist. The remaining pages are put at the
front of the freelist.
In this way, the pages that are more likely to cause a big merge are kept
free longer. Consequently there is a tendency to aggregate the
long-living allocations on a subset of the compounds, reducing the
fragmentation.
This heuristic was tested on three machines, x86, x86-64 and ppc64 with
3GB of RAM in each machine. The tests were kernbench, netperf, sysbench
and STREAM for performance and a high-order stress test for huge page
allocations.
KernBench X86
Elapsed mean 374.77 ( 0.00%) 375.10 (-0.09%)
User mean 649.53 ( 0.00%) 650.44 (-0.14%)
System mean 54.75 ( 0.00%) 54.18 ( 1.05%)
CPU mean 187.75 ( 0.00%) 187.25 ( 0.27%)
KernBench X86-64
Elapsed mean 94.45 ( 0.00%) 94.01 ( 0.47%)
User mean 323.27 ( 0.00%) 322.66 ( 0.19%)
System mean 36.71 ( 0.00%) 36.50 ( 0.57%)
CPU mean 380.75 ( 0.00%) 381.75 (-0.26%)
KernBench PPC64
Elapsed mean 173.45 ( 0.00%) 173.74 (-0.17%)
User mean 587.99 ( 0.00%) 587.95 ( 0.01%)
System mean 60.60 ( 0.00%) 60.57 ( 0.05%)
CPU mean 373.50 ( 0.00%) 372.75 ( 0.20%)
Nothing notable for kernbench.
NetPerf UDP X86
64 42.68 ( 0.00%) 42.77 ( 0.21%)
128 85.62 ( 0.00%) 85.32 (-0.35%)
256 170.01 ( 0.00%) 168.76 (-0.74%)
1024 655.68 ( 0.00%) 652.33 (-0.51%)
2048 1262.39 ( 0.00%) 1248.61 (-1.10%)
3312 1958.41 ( 0.00%) 1944.61 (-0.71%)
4096 2345.63 ( 0.00%) 2318.83 (-1.16%)
8192 4132.90 ( 0.00%) 4089.50 (-1.06%)
16384 6770.88 ( 0.00%) 6642.05 (-1.94%)*
NetPerf UDP X86-64
64 148.82 ( 0.00%) 154.92 ( 3.94%)
128 298.96 ( 0.00%) 312.95 ( 4.47%)
256 583.67 ( 0.00%) 626.39 ( 6.82%)
1024 2293.18 ( 0.00%) 2371.10 ( 3.29%)
2048 4274.16 ( 0.00%) 4396.83 ( 2.79%)
3312 6356.94 ( 0.00%) 6571.35 ( 3.26%)
4096 7422.68 ( 0.00%) 7635.42 ( 2.79%)*
8192 12114.81 ( 0.00%)* 12346.88 ( 1.88%)
16384 17022.28 ( 0.00%)* 17033.19 ( 0.06%)*
1.64% 2.73%
NetPerf UDP PPC64
64 49.98 ( 0.00%) 50.25 ( 0.54%)
128 98.66 ( 0.00%) 100.95 ( 2.27%)
256 197.33 ( 0.00%) 191.03 (-3.30%)
1024 761.98 ( 0.00%) 785.07 ( 2.94%)
2048 1493.50 ( 0.00%) 1510.85 ( 1.15%)
3312 2303.95 ( 0.00%) 2271.72 (-1.42%)
4096 2774.56 ( 0.00%) 2773.06 (-0.05%)
8192 4918.31 ( 0.00%) 4793.59 (-2.60%)
16384 7497.98 ( 0.00%) 7749.52 ( 3.25%)
The tests are run to have confidence limits within 1%. Results marked
with a * were not confident although in this case, it's only outside by
small amounts. Even with some results that were not confident, the
netperf UDP results were generally positive.
NetPerf TCP X86
64 652.25 ( 0.00%)* 648.12 (-0.64%)*
23.80% 22.82%
128 1229.98 ( 0.00%)* 1220.56 (-0.77%)*
21.03% 18.90%
256 2105.88 ( 0.00%) 1872.03 (-12.49%)*
1.00% 16.46%
1024 3476.46 ( 0.00%)* 3548.28 ( 2.02%)*
13.37% 11.39%
2048 4023.44 ( 0.00%)* 4231.45 ( 4.92%)*
9.76% 12.48%
3312 4348.88 ( 0.00%)* 4396.96 ( 1.09%)*
6.49% 8.75%
4096 4726.56 ( 0.00%)* 4877.71 ( 3.10%)*
9.85% 8.50%
8192 4732.28 ( 0.00%)* 5777.77 (18.10%)*
9.13% 13.04%
16384 5543.05 ( 0.00%)* 5906.24 ( 6.15%)*
7.73% 8.68%
NETPERF TCP X86-64
netperf-tcp-vanilla-netperf netperf-tcp
tcp-vanilla pgalloc-delay
64 1895.87 ( 0.00%)* 1775.07 (-6.81%)*
5.79% 4.78%
128 3571.03 ( 0.00%)* 3342.20 (-6.85%)*
3.68% 6.06%
256 5097.21 ( 0.00%)* 4859.43 (-4.89%)*
3.02% 2.10%
1024 8919.10 ( 0.00%)* 8892.49 (-0.30%)*
5.89% 6.55%
2048 10255.46 ( 0.00%)* 10449.39 ( 1.86%)*
7.08% 7.44%
3312 10839.90 ( 0.00%)* 10740.15 (-0.93%)*
6.87% 7.33%
4096 10814.84 ( 0.00%)* 10766.97 (-0.44%)*
6.86% 8.18%
8192 11606.89 ( 0.00%)* 11189.28 (-3.73%)*
7.49% 5.55%
16384 12554.88 ( 0.00%)* 12361.22 (-1.57%)*
7.36% 6.49%
NETPERF TCP PPC64
netperf-tcp-vanilla-netperf netperf-tcp
tcp-vanilla pgalloc-delay
64 594.17 ( 0.00%) 596.04 ( 0.31%)*
1.00% 2.29%
128 1064.87 ( 0.00%)* 1074.77 ( 0.92%)*
1.30% 1.40%
256 1852.46 ( 0.00%)* 1856.95 ( 0.24%)
1.25% 1.00%
1024 3839.46 ( 0.00%)* 3813.05 (-0.69%)
1.02% 1.00%
2048 4885.04 ( 0.00%)* 4881.97 (-0.06%)*
1.15% 1.04%
3312 5506.90 ( 0.00%) 5459.72 (-0.86%)
4096 6449.19 ( 0.00%) 6345.46 (-1.63%)
8192 7501.17 ( 0.00%) 7508.79 ( 0.10%)
16384 9618.65 ( 0.00%) 9490.10 (-1.35%)
There was a distinct lack of confidence in the X86* figures so I included
what the devation was where the results were not confident. Many of the
results, whether gains or losses were within the standard deviation so no
solid conclusion can be reached on performance impact. Looking at the
figures, only the X86-64 ones look suspicious with a few losses that were
outside the noise. However, the results were so unstable that without
knowing why they vary so much, a solid conclusion cannot be reached.
SYSBENCH X86
sysbench-vanilla pgalloc-delay
1 7722.85 ( 0.00%) 7756.79 ( 0.44%)
2 14901.11 ( 0.00%) 13683.44 (-8.90%)
3 15171.71 ( 0.00%) 14888.25 (-1.90%)
4 14966.98 ( 0.00%) 15029.67 ( 0.42%)
5 14370.47 ( 0.00%) 14865.00 ( 3.33%)
6 14870.33 ( 0.00%) 14845.57 (-0.17%)
7 14429.45 ( 0.00%) 14520.85 ( 0.63%)
8 14354.35 ( 0.00%) 14362.31 ( 0.06%)
SYSBENCH X86-64
1 17448.70 ( 0.00%) 17484.41 ( 0.20%)
2 34276.39 ( 0.00%) 34251.00 (-0.07%)
3 50805.25 ( 0.00%) 50854.80 ( 0.10%)
4 66667.10 ( 0.00%) 66174.69 (-0.74%)
5 66003.91 ( 0.00%) 65685.25 (-0.49%)
6 64981.90 ( 0.00%) 65125.60 ( 0.22%)
7 64933.16 ( 0.00%) 64379.23 (-0.86%)
8 63353.30 ( 0.00%) 63281.22 (-0.11%)
9 63511.84 ( 0.00%) 63570.37 ( 0.09%)
10 62708.27 ( 0.00%) 63166.25 ( 0.73%)
11 62092.81 ( 0.00%) 61787.75 (-0.49%)
12 61330.11 ( 0.00%) 61036.34 (-0.48%)
13 61438.37 ( 0.00%) 61994.47 ( 0.90%)
14 62304.48 ( 0.00%) 62064.90 (-0.39%)
15 63296.48 ( 0.00%) 62875.16 (-0.67%)
16 63951.76 ( 0.00%) 63769.09 (-0.29%)
SYSBENCH PPC64
-sysbench-pgalloc-delay-sysbench
sysbench-vanilla pgalloc-delay
1 7645.08 ( 0.00%) 7467.43 (-2.38%)
2 14856.67 ( 0.00%) 14558.73 (-2.05%)
3 21952.31 ( 0.00%) 21683.64 (-1.24%)
4 27946.09 ( 0.00%) 28623.29 ( 2.37%)
5 28045.11 ( 0.00%) 28143.69 ( 0.35%)
6 27477.10 ( 0.00%) 27337.45 (-0.51%)
7 26489.17 ( 0.00%) 26590.06 ( 0.38%)
8 26642.91 ( 0.00%) 25274.33 (-5.41%)
9 25137.27 ( 0.00%) 24810.06 (-1.32%)
10 24451.99 ( 0.00%) 24275.85 (-0.73%)
11 23262.20 ( 0.00%) 23674.88 ( 1.74%)
12 24234.81 ( 0.00%) 23640.89 (-2.51%)
13 24577.75 ( 0.00%) 24433.50 (-0.59%)
14 25640.19 ( 0.00%) 25116.52 (-2.08%)
15 26188.84 ( 0.00%) 26181.36 (-0.03%)
16 26782.37 ( 0.00%) 26255.99 (-2.00%)
Again, there is little to conclude here. While there are a few losses,
the results vary by +/- 8% in some cases. They are the results of most
concern as there are some large losses but it's also within the variance
typically seen between kernel releases.
The STREAM results varied so little and are so verbose that I didn't
include them here.
The final test stressed how many huge pages can be allocated. The
absolute number of huge pages allocated are the same with or without the
page. However, the "unusability free space index" which is a measure of
external fragmentation was slightly lower (lower is better) throughout the
lifetime of the system. I also measured the latency of how long it took
to successfully allocate a huge page. The latency was slightly lower and
on X86 and PPC64, more huge pages were allocated almost immediately from
the free lists. The improvement is slight but there.
[mel@csn.ul.ie: Tested, reworked for less branches]
[czoccolo@gmail.com: fix oops by checking pfn_valid_within()]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Acked-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Corrado Zoccolo <czoccolo@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-24 15:31:54 -06:00
|
|
|
struct page *higher_page, *higher_buddy;
|
2011-01-13 16:47:24 -07:00
|
|
|
combined_idx = buddy_idx & page_idx;
|
|
|
|
higher_page = page + (combined_idx - page_idx);
|
|
|
|
buddy_idx = __find_buddy_index(combined_idx, order + 1);
|
2012-09-17 15:09:21 -06:00
|
|
|
higher_buddy = higher_page + (buddy_idx - combined_idx);
|
page allocator: reduce fragmentation in buddy allocator by adding buddies that are merging to the tail of the free lists
In order to reduce fragmentation, this patch classifies freed pages in two
groups according to their probability of being part of a high order merge.
Pages belonging to a compound whose next-highest buddy is free are more
likely to be part of a high order merge in the near future, so they will
be added at the tail of the freelist. The remaining pages are put at the
front of the freelist.
In this way, the pages that are more likely to cause a big merge are kept
free longer. Consequently there is a tendency to aggregate the
long-living allocations on a subset of the compounds, reducing the
fragmentation.
This heuristic was tested on three machines, x86, x86-64 and ppc64 with
3GB of RAM in each machine. The tests were kernbench, netperf, sysbench
and STREAM for performance and a high-order stress test for huge page
allocations.
KernBench X86
Elapsed mean 374.77 ( 0.00%) 375.10 (-0.09%)
User mean 649.53 ( 0.00%) 650.44 (-0.14%)
System mean 54.75 ( 0.00%) 54.18 ( 1.05%)
CPU mean 187.75 ( 0.00%) 187.25 ( 0.27%)
KernBench X86-64
Elapsed mean 94.45 ( 0.00%) 94.01 ( 0.47%)
User mean 323.27 ( 0.00%) 322.66 ( 0.19%)
System mean 36.71 ( 0.00%) 36.50 ( 0.57%)
CPU mean 380.75 ( 0.00%) 381.75 (-0.26%)
KernBench PPC64
Elapsed mean 173.45 ( 0.00%) 173.74 (-0.17%)
User mean 587.99 ( 0.00%) 587.95 ( 0.01%)
System mean 60.60 ( 0.00%) 60.57 ( 0.05%)
CPU mean 373.50 ( 0.00%) 372.75 ( 0.20%)
Nothing notable for kernbench.
NetPerf UDP X86
64 42.68 ( 0.00%) 42.77 ( 0.21%)
128 85.62 ( 0.00%) 85.32 (-0.35%)
256 170.01 ( 0.00%) 168.76 (-0.74%)
1024 655.68 ( 0.00%) 652.33 (-0.51%)
2048 1262.39 ( 0.00%) 1248.61 (-1.10%)
3312 1958.41 ( 0.00%) 1944.61 (-0.71%)
4096 2345.63 ( 0.00%) 2318.83 (-1.16%)
8192 4132.90 ( 0.00%) 4089.50 (-1.06%)
16384 6770.88 ( 0.00%) 6642.05 (-1.94%)*
NetPerf UDP X86-64
64 148.82 ( 0.00%) 154.92 ( 3.94%)
128 298.96 ( 0.00%) 312.95 ( 4.47%)
256 583.67 ( 0.00%) 626.39 ( 6.82%)
1024 2293.18 ( 0.00%) 2371.10 ( 3.29%)
2048 4274.16 ( 0.00%) 4396.83 ( 2.79%)
3312 6356.94 ( 0.00%) 6571.35 ( 3.26%)
4096 7422.68 ( 0.00%) 7635.42 ( 2.79%)*
8192 12114.81 ( 0.00%)* 12346.88 ( 1.88%)
16384 17022.28 ( 0.00%)* 17033.19 ( 0.06%)*
1.64% 2.73%
NetPerf UDP PPC64
64 49.98 ( 0.00%) 50.25 ( 0.54%)
128 98.66 ( 0.00%) 100.95 ( 2.27%)
256 197.33 ( 0.00%) 191.03 (-3.30%)
1024 761.98 ( 0.00%) 785.07 ( 2.94%)
2048 1493.50 ( 0.00%) 1510.85 ( 1.15%)
3312 2303.95 ( 0.00%) 2271.72 (-1.42%)
4096 2774.56 ( 0.00%) 2773.06 (-0.05%)
8192 4918.31 ( 0.00%) 4793.59 (-2.60%)
16384 7497.98 ( 0.00%) 7749.52 ( 3.25%)
The tests are run to have confidence limits within 1%. Results marked
with a * were not confident although in this case, it's only outside by
small amounts. Even with some results that were not confident, the
netperf UDP results were generally positive.
NetPerf TCP X86
64 652.25 ( 0.00%)* 648.12 (-0.64%)*
23.80% 22.82%
128 1229.98 ( 0.00%)* 1220.56 (-0.77%)*
21.03% 18.90%
256 2105.88 ( 0.00%) 1872.03 (-12.49%)*
1.00% 16.46%
1024 3476.46 ( 0.00%)* 3548.28 ( 2.02%)*
13.37% 11.39%
2048 4023.44 ( 0.00%)* 4231.45 ( 4.92%)*
9.76% 12.48%
3312 4348.88 ( 0.00%)* 4396.96 ( 1.09%)*
6.49% 8.75%
4096 4726.56 ( 0.00%)* 4877.71 ( 3.10%)*
9.85% 8.50%
8192 4732.28 ( 0.00%)* 5777.77 (18.10%)*
9.13% 13.04%
16384 5543.05 ( 0.00%)* 5906.24 ( 6.15%)*
7.73% 8.68%
NETPERF TCP X86-64
netperf-tcp-vanilla-netperf netperf-tcp
tcp-vanilla pgalloc-delay
64 1895.87 ( 0.00%)* 1775.07 (-6.81%)*
5.79% 4.78%
128 3571.03 ( 0.00%)* 3342.20 (-6.85%)*
3.68% 6.06%
256 5097.21 ( 0.00%)* 4859.43 (-4.89%)*
3.02% 2.10%
1024 8919.10 ( 0.00%)* 8892.49 (-0.30%)*
5.89% 6.55%
2048 10255.46 ( 0.00%)* 10449.39 ( 1.86%)*
7.08% 7.44%
3312 10839.90 ( 0.00%)* 10740.15 (-0.93%)*
6.87% 7.33%
4096 10814.84 ( 0.00%)* 10766.97 (-0.44%)*
6.86% 8.18%
8192 11606.89 ( 0.00%)* 11189.28 (-3.73%)*
7.49% 5.55%
16384 12554.88 ( 0.00%)* 12361.22 (-1.57%)*
7.36% 6.49%
NETPERF TCP PPC64
netperf-tcp-vanilla-netperf netperf-tcp
tcp-vanilla pgalloc-delay
64 594.17 ( 0.00%) 596.04 ( 0.31%)*
1.00% 2.29%
128 1064.87 ( 0.00%)* 1074.77 ( 0.92%)*
1.30% 1.40%
256 1852.46 ( 0.00%)* 1856.95 ( 0.24%)
1.25% 1.00%
1024 3839.46 ( 0.00%)* 3813.05 (-0.69%)
1.02% 1.00%
2048 4885.04 ( 0.00%)* 4881.97 (-0.06%)*
1.15% 1.04%
3312 5506.90 ( 0.00%) 5459.72 (-0.86%)
4096 6449.19 ( 0.00%) 6345.46 (-1.63%)
8192 7501.17 ( 0.00%) 7508.79 ( 0.10%)
16384 9618.65 ( 0.00%) 9490.10 (-1.35%)
There was a distinct lack of confidence in the X86* figures so I included
what the devation was where the results were not confident. Many of the
results, whether gains or losses were within the standard deviation so no
solid conclusion can be reached on performance impact. Looking at the
figures, only the X86-64 ones look suspicious with a few losses that were
outside the noise. However, the results were so unstable that without
knowing why they vary so much, a solid conclusion cannot be reached.
SYSBENCH X86
sysbench-vanilla pgalloc-delay
1 7722.85 ( 0.00%) 7756.79 ( 0.44%)
2 14901.11 ( 0.00%) 13683.44 (-8.90%)
3 15171.71 ( 0.00%) 14888.25 (-1.90%)
4 14966.98 ( 0.00%) 15029.67 ( 0.42%)
5 14370.47 ( 0.00%) 14865.00 ( 3.33%)
6 14870.33 ( 0.00%) 14845.57 (-0.17%)
7 14429.45 ( 0.00%) 14520.85 ( 0.63%)
8 14354.35 ( 0.00%) 14362.31 ( 0.06%)
SYSBENCH X86-64
1 17448.70 ( 0.00%) 17484.41 ( 0.20%)
2 34276.39 ( 0.00%) 34251.00 (-0.07%)
3 50805.25 ( 0.00%) 50854.80 ( 0.10%)
4 66667.10 ( 0.00%) 66174.69 (-0.74%)
5 66003.91 ( 0.00%) 65685.25 (-0.49%)
6 64981.90 ( 0.00%) 65125.60 ( 0.22%)
7 64933.16 ( 0.00%) 64379.23 (-0.86%)
8 63353.30 ( 0.00%) 63281.22 (-0.11%)
9 63511.84 ( 0.00%) 63570.37 ( 0.09%)
10 62708.27 ( 0.00%) 63166.25 ( 0.73%)
11 62092.81 ( 0.00%) 61787.75 (-0.49%)
12 61330.11 ( 0.00%) 61036.34 (-0.48%)
13 61438.37 ( 0.00%) 61994.47 ( 0.90%)
14 62304.48 ( 0.00%) 62064.90 (-0.39%)
15 63296.48 ( 0.00%) 62875.16 (-0.67%)
16 63951.76 ( 0.00%) 63769.09 (-0.29%)
SYSBENCH PPC64
-sysbench-pgalloc-delay-sysbench
sysbench-vanilla pgalloc-delay
1 7645.08 ( 0.00%) 7467.43 (-2.38%)
2 14856.67 ( 0.00%) 14558.73 (-2.05%)
3 21952.31 ( 0.00%) 21683.64 (-1.24%)
4 27946.09 ( 0.00%) 28623.29 ( 2.37%)
5 28045.11 ( 0.00%) 28143.69 ( 0.35%)
6 27477.10 ( 0.00%) 27337.45 (-0.51%)
7 26489.17 ( 0.00%) 26590.06 ( 0.38%)
8 26642.91 ( 0.00%) 25274.33 (-5.41%)
9 25137.27 ( 0.00%) 24810.06 (-1.32%)
10 24451.99 ( 0.00%) 24275.85 (-0.73%)
11 23262.20 ( 0.00%) 23674.88 ( 1.74%)
12 24234.81 ( 0.00%) 23640.89 (-2.51%)
13 24577.75 ( 0.00%) 24433.50 (-0.59%)
14 25640.19 ( 0.00%) 25116.52 (-2.08%)
15 26188.84 ( 0.00%) 26181.36 (-0.03%)
16 26782.37 ( 0.00%) 26255.99 (-2.00%)
Again, there is little to conclude here. While there are a few losses,
the results vary by +/- 8% in some cases. They are the results of most
concern as there are some large losses but it's also within the variance
typically seen between kernel releases.
The STREAM results varied so little and are so verbose that I didn't
include them here.
The final test stressed how many huge pages can be allocated. The
absolute number of huge pages allocated are the same with or without the
page. However, the "unusability free space index" which is a measure of
external fragmentation was slightly lower (lower is better) throughout the
lifetime of the system. I also measured the latency of how long it took
to successfully allocate a huge page. The latency was slightly lower and
on X86 and PPC64, more huge pages were allocated almost immediately from
the free lists. The improvement is slight but there.
[mel@csn.ul.ie: Tested, reworked for less branches]
[czoccolo@gmail.com: fix oops by checking pfn_valid_within()]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Acked-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Corrado Zoccolo <czoccolo@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-24 15:31:54 -06:00
|
|
|
if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
|
|
|
|
list_add_tail(&page->lru,
|
|
|
|
&zone->free_area[order].free_list[migratetype]);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
|
|
|
|
out:
|
2005-04-16 16:20:36 -06:00
|
|
|
zone->free_area[order].nr_free++;
|
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:15 -06:00
|
|
|
/*
|
|
|
|
* A bad page could be due to a number of fields. Instead of multiple branches,
|
|
|
|
* try and check multiple fields with one check. The caller must do a detailed
|
|
|
|
* check if necessary.
|
|
|
|
*/
|
|
|
|
static inline bool page_expected_state(struct page *page,
|
|
|
|
unsigned long check_flags)
|
|
|
|
{
|
|
|
|
if (unlikely(atomic_read(&page->_mapcount) != -1))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (unlikely((unsigned long)page->mapping |
|
|
|
|
page_ref_count(page) |
|
|
|
|
#ifdef CONFIG_MEMCG
|
|
|
|
(unsigned long)page->mem_cgroup |
|
|
|
|
#endif
|
|
|
|
(page->flags & check_flags)))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:18 -06:00
|
|
|
static void free_pages_check_bad(struct page *page)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2016-05-19 18:14:15 -06:00
|
|
|
const char *bad_reason;
|
|
|
|
unsigned long bad_flags;
|
|
|
|
|
|
|
|
bad_reason = NULL;
|
|
|
|
bad_flags = 0;
|
2014-01-23 16:52:49 -07:00
|
|
|
|
2016-01-15 17:53:42 -07:00
|
|
|
if (unlikely(atomic_read(&page->_mapcount) != -1))
|
2014-01-23 16:52:49 -07:00
|
|
|
bad_reason = "nonzero mapcount";
|
|
|
|
if (unlikely(page->mapping != NULL))
|
|
|
|
bad_reason = "non-NULL mapping";
|
2016-03-17 15:19:26 -06:00
|
|
|
if (unlikely(page_ref_count(page) != 0))
|
2016-05-19 18:10:49 -06:00
|
|
|
bad_reason = "nonzero _refcount";
|
2014-01-23 16:52:49 -07:00
|
|
|
if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
|
|
|
|
bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
|
|
|
|
bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
|
|
|
|
}
|
2014-12-10 16:44:58 -07:00
|
|
|
#ifdef CONFIG_MEMCG
|
|
|
|
if (unlikely(page->mem_cgroup))
|
|
|
|
bad_reason = "page still charged to cgroup";
|
|
|
|
#endif
|
2016-05-19 18:14:15 -06:00
|
|
|
bad_page(page, bad_reason, bad_flags);
|
2016-05-19 18:14:18 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline int free_pages_check(struct page *page)
|
|
|
|
{
|
2016-05-19 18:14:21 -06:00
|
|
|
if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
|
2016-05-19 18:14:18 -06:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* Something has gone sideways, find it */
|
|
|
|
free_pages_check_bad(page);
|
2016-05-19 18:14:15 -06:00
|
|
|
return 1;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:32 -06:00
|
|
|
static int free_tail_pages_check(struct page *head_page, struct page *page)
|
|
|
|
{
|
|
|
|
int ret = 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We rely page->lru.next never has bit 0 set, unless the page
|
|
|
|
* is PageTail(). Let's make sure that's true even for poisoned ->lru.
|
|
|
|
*/
|
|
|
|
BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
|
|
|
|
|
|
|
|
if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
|
|
|
|
ret = 0;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
switch (page - head_page) {
|
|
|
|
case 1:
|
|
|
|
/* the first tail page: ->mapping is compound_mapcount() */
|
|
|
|
if (unlikely(compound_mapcount(page))) {
|
|
|
|
bad_page(page, "nonzero compound_mapcount", 0);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
/*
|
|
|
|
* the second tail page: ->mapping is
|
|
|
|
* page_deferred_list().next -- ignore value.
|
|
|
|
*/
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
if (page->mapping != TAIL_MAPPING) {
|
|
|
|
bad_page(page, "corrupted mapping in tail page", 0);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (unlikely(!PageTail(page))) {
|
|
|
|
bad_page(page, "PageTail not set", 0);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (unlikely(compound_head(page) != head_page)) {
|
|
|
|
bad_page(page, "compound_head not consistent", 0);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
ret = 0;
|
|
|
|
out:
|
|
|
|
page->mapping = NULL;
|
|
|
|
clear_compound_head(page);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:38 -06:00
|
|
|
static __always_inline bool free_pages_prepare(struct page *page,
|
|
|
|
unsigned int order, bool check_free)
|
2016-05-19 18:14:32 -06:00
|
|
|
{
|
2016-05-19 18:14:38 -06:00
|
|
|
int bad = 0;
|
2016-05-19 18:14:32 -06:00
|
|
|
|
|
|
|
VM_BUG_ON_PAGE(PageTail(page), page);
|
|
|
|
|
2016-05-19 18:14:38 -06:00
|
|
|
trace_mm_page_free(page, order);
|
|
|
|
kmemcheck_free_shadow(page, order);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check tail pages before head page information is cleared to
|
|
|
|
* avoid checking PageCompound for order-0 pages.
|
|
|
|
*/
|
|
|
|
if (unlikely(order)) {
|
|
|
|
bool compound = PageCompound(page);
|
|
|
|
int i;
|
|
|
|
|
|
|
|
VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
|
2016-05-19 18:14:32 -06:00
|
|
|
|
2016-07-26 16:25:53 -06:00
|
|
|
if (compound)
|
|
|
|
ClearPageDoubleMap(page);
|
2016-05-19 18:14:38 -06:00
|
|
|
for (i = 1; i < (1 << order); i++) {
|
|
|
|
if (compound)
|
|
|
|
bad += free_tail_pages_check(page, page + i);
|
|
|
|
if (unlikely(free_pages_check(page + i))) {
|
|
|
|
bad++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
(page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
|
|
|
|
}
|
|
|
|
}
|
mm: migrate: support non-lru movable page migration
We have allowed migration for only LRU pages until now and it was enough
to make high-order pages. But recently, embedded system(e.g., webOS,
android) uses lots of non-movable pages(e.g., zram, GPU memory) so we
have seen several reports about troubles of small high-order allocation.
For fixing the problem, there were several efforts (e,g,. enhance
compaction algorithm, SLUB fallback to 0-order page, reserved memory,
vmalloc and so on) but if there are lots of non-movable pages in system,
their solutions are void in the long run.
So, this patch is to support facility to change non-movable pages with
movable. For the feature, this patch introduces functions related to
migration to address_space_operations as well as some page flags.
If a driver want to make own pages movable, it should define three
functions which are function pointers of struct
address_space_operations.
1. bool (*isolate_page) (struct page *page, isolate_mode_t mode);
What VM expects on isolate_page function of driver is to return *true*
if driver isolates page successfully. On returing true, VM marks the
page as PG_isolated so concurrent isolation in several CPUs skip the
page for isolation. If a driver cannot isolate the page, it should
return *false*.
Once page is successfully isolated, VM uses page.lru fields so driver
shouldn't expect to preserve values in that fields.
2. int (*migratepage) (struct address_space *mapping,
struct page *newpage, struct page *oldpage, enum migrate_mode);
After isolation, VM calls migratepage of driver with isolated page. The
function of migratepage is to move content of the old page to new page
and set up fields of struct page newpage. Keep in mind that you should
indicate to the VM the oldpage is no longer movable via
__ClearPageMovable() under page_lock if you migrated the oldpage
successfully and returns 0. If driver cannot migrate the page at the
moment, driver can return -EAGAIN. On -EAGAIN, VM will retry page
migration in a short time because VM interprets -EAGAIN as "temporal
migration failure". On returning any error except -EAGAIN, VM will give
up the page migration without retrying in this time.
Driver shouldn't touch page.lru field VM using in the functions.
3. void (*putback_page)(struct page *);
If migration fails on isolated page, VM should return the isolated page
to the driver so VM calls driver's putback_page with migration failed
page. In this function, driver should put the isolated page back to the
own data structure.
4. non-lru movable page flags
There are two page flags for supporting non-lru movable page.
* PG_movable
Driver should use the below function to make page movable under
page_lock.
void __SetPageMovable(struct page *page, struct address_space *mapping)
It needs argument of address_space for registering migration family
functions which will be called by VM. Exactly speaking, PG_movable is
not a real flag of struct page. Rather than, VM reuses page->mapping's
lower bits to represent it.
#define PAGE_MAPPING_MOVABLE 0x2
page->mapping = page->mapping | PAGE_MAPPING_MOVABLE;
so driver shouldn't access page->mapping directly. Instead, driver
should use page_mapping which mask off the low two bits of page->mapping
so it can get right struct address_space.
For testing of non-lru movable page, VM supports __PageMovable function.
However, it doesn't guarantee to identify non-lru movable page because
page->mapping field is unified with other variables in struct page. As
well, if driver releases the page after isolation by VM, page->mapping
doesn't have stable value although it has PAGE_MAPPING_MOVABLE (Look at
__ClearPageMovable). But __PageMovable is cheap to catch whether page
is LRU or non-lru movable once the page has been isolated. Because LRU
pages never can have PAGE_MAPPING_MOVABLE in page->mapping. It is also
good for just peeking to test non-lru movable pages before more
expensive checking with lock_page in pfn scanning to select victim.
For guaranteeing non-lru movable page, VM provides PageMovable function.
Unlike __PageMovable, PageMovable functions validates page->mapping and
mapping->a_ops->isolate_page under lock_page. The lock_page prevents
sudden destroying of page->mapping.
Driver using __SetPageMovable should clear the flag via
__ClearMovablePage under page_lock before the releasing the page.
* PG_isolated
To prevent concurrent isolation among several CPUs, VM marks isolated
page as PG_isolated under lock_page. So if a CPU encounters PG_isolated
non-lru movable page, it can skip it. Driver doesn't need to manipulate
the flag because VM will set/clear it automatically. Keep in mind that
if driver sees PG_isolated page, it means the page have been isolated by
VM so it shouldn't touch page.lru field. PG_isolated is alias with
PG_reclaim flag so driver shouldn't use the flag for own purpose.
[opensource.ganesh@gmail.com: mm/compaction: remove local variable is_lru]
Link: http://lkml.kernel.org/r/20160618014841.GA7422@leo-test
Link: http://lkml.kernel.org/r/1464736881-24886-3-git-send-email-minchan@kernel.org
Signed-off-by: Gioh Kim <gi-oh.kim@profitbricks.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Ganesh Mahendran <opensource.ganesh@gmail.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rafael Aquini <aquini@redhat.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: John Einar Reitan <john.reitan@foss.arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-26 16:23:05 -06:00
|
|
|
if (PageMappingFlags(page))
|
2016-05-19 18:14:32 -06:00
|
|
|
page->mapping = NULL;
|
mm: memcontrol: only mark charged pages with PageKmemcg
To distinguish non-slab pages charged to kmemcg we mark them PageKmemcg,
which sets page->_mapcount to -512. Currently, we set/clear PageKmemcg
in __alloc_pages_nodemask()/free_pages_prepare() for any page allocated
with __GFP_ACCOUNT, including those that aren't actually charged to any
cgroup, i.e. allocated from the root cgroup context. To avoid overhead
in case cgroups are not used, we only do that if memcg_kmem_enabled() is
true. The latter is set iff there are kmem-enabled memory cgroups
(online or offline). The root cgroup is not considered kmem-enabled.
As a result, if a page is allocated with __GFP_ACCOUNT for the root
cgroup when there are kmem-enabled memory cgroups and is freed after all
kmem-enabled memory cgroups were removed, e.g.
# no memory cgroups has been created yet, create one
mkdir /sys/fs/cgroup/memory/test
# run something allocating pages with __GFP_ACCOUNT, e.g.
# a program using pipe
dmesg | tail
# remove the memory cgroup
rmdir /sys/fs/cgroup/memory/test
we'll get bad page state bug complaining about page->_mapcount != -1:
BUG: Bad page state in process swapper/0 pfn:1fd945c
page:ffffea007f651700 count:0 mapcount:-511 mapping: (null) index:0x0
flags: 0x1000000000000000()
To avoid that, let's mark with PageKmemcg only those pages that are
actually charged to and hence pin a non-root memory cgroup.
Fixes: 4949148ad433 ("mm: charge/uncharge kmemcg from generic page allocator paths")
Reported-and-tested-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-08-08 14:03:12 -06:00
|
|
|
if (memcg_kmem_enabled() && PageKmemcg(page))
|
mm: charge/uncharge kmemcg from generic page allocator paths
Currently, to charge a non-slab allocation to kmemcg one has to use
alloc_kmem_pages helper with __GFP_ACCOUNT flag. A page allocated with
this helper should finally be freed using free_kmem_pages, otherwise it
won't be uncharged.
This API suits its current users fine, but it turns out to be impossible
to use along with page reference counting, i.e. when an allocation is
supposed to be freed with put_page, as it is the case with pipe or unix
socket buffers.
To overcome this limitation, this patch moves charging/uncharging to
generic page allocator paths, i.e. to __alloc_pages_nodemask and
free_pages_prepare, and zaps alloc/free_kmem_pages helpers. This way,
one can use any of the available page allocation functions to get the
allocated page charged to kmemcg - it's enough to pass __GFP_ACCOUNT,
just like in case of kmalloc and friends. A charged page will be
automatically uncharged on free.
To make it possible, we need to mark pages charged to kmemcg somehow.
To avoid introducing a new page flag, we make use of page->_mapcount for
marking such pages. Since pages charged to kmemcg are not supposed to
be mapped to userspace, it should work just fine. There are other
(ab)users of page->_mapcount - buddy and balloon pages - but we don't
conflict with them.
In case kmemcg is compiled out or not used at runtime, this patch
introduces no overhead to generic page allocator paths. If kmemcg is
used, it will be plus one gfp flags check on alloc and plus one
page->_mapcount check on free, which shouldn't hurt performance, because
the data accessed are hot.
Link: http://lkml.kernel.org/r/a9736d856f895bcb465d9f257b54efe32eda6f99.1464079538.git.vdavydov@virtuozzo.com
Signed-off-by: Vladimir Davydov <vdavydov@virtuozzo.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-26 16:24:24 -06:00
|
|
|
memcg_kmem_uncharge(page, order);
|
2016-05-19 18:14:38 -06:00
|
|
|
if (check_free)
|
|
|
|
bad += free_pages_check(page);
|
|
|
|
if (bad)
|
|
|
|
return false;
|
2016-05-19 18:14:32 -06:00
|
|
|
|
2016-05-19 18:14:38 -06:00
|
|
|
page_cpupid_reset_last(page);
|
|
|
|
page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
|
|
|
|
reset_page_owner(page, order);
|
2016-05-19 18:14:32 -06:00
|
|
|
|
|
|
|
if (!PageHighMem(page)) {
|
|
|
|
debug_check_no_locks_freed(page_address(page),
|
2016-05-19 18:14:38 -06:00
|
|
|
PAGE_SIZE << order);
|
2016-05-19 18:14:32 -06:00
|
|
|
debug_check_no_obj_freed(page_address(page),
|
2016-05-19 18:14:38 -06:00
|
|
|
PAGE_SIZE << order);
|
2016-05-19 18:14:32 -06:00
|
|
|
}
|
2016-05-19 18:14:38 -06:00
|
|
|
arch_free_page(page, order);
|
|
|
|
kernel_poison_pages(page, 1 << order, 0);
|
|
|
|
kernel_map_pages(page, 1 << order, 0);
|
2016-05-20 17:58:47 -06:00
|
|
|
kasan_free_pages(page, order);
|
2016-05-19 18:14:32 -06:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:38 -06:00
|
|
|
#ifdef CONFIG_DEBUG_VM
|
|
|
|
static inline bool free_pcp_prepare(struct page *page)
|
|
|
|
{
|
|
|
|
return free_pages_prepare(page, 0, true);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool bulkfree_pcp_prepare(struct page *page)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static bool free_pcp_prepare(struct page *page)
|
|
|
|
{
|
|
|
|
return free_pages_prepare(page, 0, false);
|
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:32 -06:00
|
|
|
static bool bulkfree_pcp_prepare(struct page *page)
|
|
|
|
{
|
|
|
|
return free_pages_check(page);
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_DEBUG_VM */
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
* Frees a number of pages from the PCP lists
|
2005-04-16 16:20:36 -06:00
|
|
|
* Assumes all pages on list are in same zone, and of same order.
|
2005-09-10 01:26:59 -06:00
|
|
|
* count is the number of pages to free.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* If the zone was previously in an "all pages pinned" state then look to
|
|
|
|
* see if this freeing clears that state.
|
|
|
|
*
|
|
|
|
* And clear the zone's pages_scanned counter, to hold off the "all pages are
|
|
|
|
* pinned" detection logic.
|
|
|
|
*/
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
static void free_pcppages_bulk(struct zone *zone, int count,
|
|
|
|
struct per_cpu_pages *pcp)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
int migratetype = 0;
|
2009-09-21 18:03:20 -06:00
|
|
|
int batch_free = 0;
|
2014-08-06 17:07:16 -06:00
|
|
|
unsigned long nr_scanned;
|
2016-05-19 18:13:58 -06:00
|
|
|
bool isolated_pageblocks;
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
|
2006-01-06 01:10:56 -07:00
|
|
|
spin_lock(&zone->lock);
|
2016-05-19 18:13:58 -06:00
|
|
|
isolated_pageblocks = has_isolate_pageblock(zone);
|
2016-07-28 16:45:31 -06:00
|
|
|
nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
|
2014-08-06 17:07:16 -06:00
|
|
|
if (nr_scanned)
|
2016-07-28 16:45:31 -06:00
|
|
|
__mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);
|
2009-06-16 16:32:13 -06:00
|
|
|
|
2016-05-19 18:14:24 -06:00
|
|
|
while (count) {
|
2006-01-08 02:00:42 -07:00
|
|
|
struct page *page;
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
struct list_head *list;
|
|
|
|
|
|
|
|
/*
|
2009-09-21 18:03:20 -06:00
|
|
|
* Remove pages from lists in a round-robin fashion. A
|
|
|
|
* batch_free count is maintained that is incremented when an
|
|
|
|
* empty list is encountered. This is so more pages are freed
|
|
|
|
* off fuller lists instead of spinning excessively around empty
|
|
|
|
* lists
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
*/
|
|
|
|
do {
|
2009-09-21 18:03:20 -06:00
|
|
|
batch_free++;
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
if (++migratetype == MIGRATE_PCPTYPES)
|
|
|
|
migratetype = 0;
|
|
|
|
list = &pcp->lists[migratetype];
|
|
|
|
} while (list_empty(list));
|
2006-01-08 02:00:42 -07:00
|
|
|
|
2011-03-22 17:32:45 -06:00
|
|
|
/* This is the only non-empty list. Free them all. */
|
|
|
|
if (batch_free == MIGRATE_PCPTYPES)
|
2016-05-19 18:14:24 -06:00
|
|
|
batch_free = count;
|
2011-03-22 17:32:45 -06:00
|
|
|
|
2009-09-21 18:03:20 -06:00
|
|
|
do {
|
2012-10-08 17:31:57 -06:00
|
|
|
int mt; /* migratetype of the to-be-freed page */
|
|
|
|
|
2016-01-14 16:20:30 -07:00
|
|
|
page = list_last_entry(list, struct page, lru);
|
2009-09-21 18:03:20 -06:00
|
|
|
/* must delete as __free_one_page list manipulates */
|
|
|
|
list_del(&page->lru);
|
mm, page_isolation: remove bogus tests for isolated pages
The __test_page_isolated_in_pageblock() is used to verify whether all
pages in pageblock were either successfully isolated, or are hwpoisoned.
Two of the possible state of pages, that are tested, are however bogus
and misleading.
Both tests rely on get_freepage_migratetype(page), which however has no
guarantees about pages on freelists. Specifically, it doesn't guarantee
that the migratetype returned by the function actually matches the
migratetype of the freelist that the page is on. Such guarantee is not
its purpose and would have negative impact on allocator performance.
The first test checks whether the freepage_migratetype equals
MIGRATE_ISOLATE, supposedly to catch races between page isolation and
allocator activity. These races should be fixed nowadays with
51bb1a4093 ("mm/page_alloc: add freepage on isolate pageblock to correct
buddy list") and related patches. As explained above, the check
wouldn't be able to catch them reliably anyway. For the same reason
false positives can happen, although they are harmless, as the
move_freepages() call would just move the page to the same freelist it's
already on. So removing the test is not a bug fix, just cleanup. After
this patch, we assume that all PageBuddy pages are on the correct
freelist and that the races were really fixed. A truly reliable
verification in the form of e.g. VM_BUG_ON() would be complicated and
is arguably not needed.
The second test (page_count(page) == 0 && get_freepage_migratetype(page)
== MIGRATE_ISOLATE) is probably supposed (the code comes from a big
memory isolation patch from 2007) to catch pages on MIGRATE_ISOLATE
pcplists. However, pcplists don't contain MIGRATE_ISOLATE freepages
nowadays, those are freed directly to free lists, so the check is
obsolete. Remove it as well.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Minchan Kim <minchan@kernel.org>
Acked-by: Michal Nazarewicz <mina86@mina86.com>
Cc: Laura Abbott <lauraa@codeaurora.org>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Seungho Park <seungho1.park@lge.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-09-08 16:01:22 -06:00
|
|
|
|
2015-09-08 16:01:25 -06:00
|
|
|
mt = get_pcppage_migratetype(page);
|
mm, page_isolation: remove bogus tests for isolated pages
The __test_page_isolated_in_pageblock() is used to verify whether all
pages in pageblock were either successfully isolated, or are hwpoisoned.
Two of the possible state of pages, that are tested, are however bogus
and misleading.
Both tests rely on get_freepage_migratetype(page), which however has no
guarantees about pages on freelists. Specifically, it doesn't guarantee
that the migratetype returned by the function actually matches the
migratetype of the freelist that the page is on. Such guarantee is not
its purpose and would have negative impact on allocator performance.
The first test checks whether the freepage_migratetype equals
MIGRATE_ISOLATE, supposedly to catch races between page isolation and
allocator activity. These races should be fixed nowadays with
51bb1a4093 ("mm/page_alloc: add freepage on isolate pageblock to correct
buddy list") and related patches. As explained above, the check
wouldn't be able to catch them reliably anyway. For the same reason
false positives can happen, although they are harmless, as the
move_freepages() call would just move the page to the same freelist it's
already on. So removing the test is not a bug fix, just cleanup. After
this patch, we assume that all PageBuddy pages are on the correct
freelist and that the races were really fixed. A truly reliable
verification in the form of e.g. VM_BUG_ON() would be complicated and
is arguably not needed.
The second test (page_count(page) == 0 && get_freepage_migratetype(page)
== MIGRATE_ISOLATE) is probably supposed (the code comes from a big
memory isolation patch from 2007) to catch pages on MIGRATE_ISOLATE
pcplists. However, pcplists don't contain MIGRATE_ISOLATE freepages
nowadays, those are freed directly to free lists, so the check is
obsolete. Remove it as well.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Minchan Kim <minchan@kernel.org>
Acked-by: Michal Nazarewicz <mina86@mina86.com>
Cc: Laura Abbott <lauraa@codeaurora.org>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Seungho Park <seungho1.park@lge.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-09-08 16:01:22 -06:00
|
|
|
/* MIGRATE_ISOLATE page should not go to pcplists */
|
|
|
|
VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
|
|
|
|
/* Pageblock could have been isolated meanwhile */
|
2016-05-19 18:13:58 -06:00
|
|
|
if (unlikely(isolated_pageblocks))
|
mm/page_alloc: add freepage on isolate pageblock to correct buddy list
In free_pcppages_bulk(), we use cached migratetype of freepage to
determine type of buddy list where freepage will be added. This
information is stored when freepage is added to pcp list, so if
isolation of pageblock of this freepage begins after storing, this
cached information could be stale. In other words, it has original
migratetype rather than MIGRATE_ISOLATE.
There are two problems caused by this stale information.
One is that we can't keep these freepages from being allocated.
Although this pageblock is isolated, freepage will be added to normal
buddy list so that it could be allocated without any restriction. And
the other problem is incorrect freepage accounting. Freepages on
isolate pageblock should not be counted for number of freepage.
Following is the code snippet in free_pcppages_bulk().
/* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
__free_one_page(page, page_to_pfn(page), zone, 0, mt);
trace_mm_page_pcpu_drain(page, 0, mt);
if (likely(!is_migrate_isolate_page(page))) {
__mod_zone_page_state(zone, NR_FREE_PAGES, 1);
if (is_migrate_cma(mt))
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1);
}
As you can see above snippet, current code already handle second
problem, incorrect freepage accounting, by re-fetching pageblock
migratetype through is_migrate_isolate_page(page).
But, because this re-fetched information isn't used for
__free_one_page(), first problem would not be solved. This patch try to
solve this situation to re-fetch pageblock migratetype before
__free_one_page() and to use it for __free_one_page().
In addition to move up position of this re-fetch, this patch use
optimization technique, re-fetching migratetype only if there is isolate
pageblock. Pageblock isolation is rare event, so we can avoid
re-fetching in common case with this optimization.
This patch also correct migratetype of the tracepoint output.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Minchan Kim <minchan@kernel.org>
Acked-by: Michal Nazarewicz <mina86@mina86.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Laura Abbott <lauraa@codeaurora.org>
Cc: Heesub Shin <heesub.shin@samsung.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: Ritesh Harjani <ritesh.list@gmail.com>
Cc: Gioh Kim <gioh.kim@lge.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-11-13 16:19:14 -07:00
|
|
|
mt = get_pageblock_migratetype(page);
|
|
|
|
|
2016-05-19 18:14:32 -06:00
|
|
|
if (bulkfree_pcp_prepare(page))
|
|
|
|
continue;
|
|
|
|
|
2014-06-04 17:10:17 -06:00
|
|
|
__free_one_page(page, page_to_pfn(page), zone, 0, mt);
|
2012-10-08 17:31:57 -06:00
|
|
|
trace_mm_page_pcpu_drain(page, 0, mt);
|
2016-05-19 18:14:24 -06:00
|
|
|
} while (--count && --batch_free && !list_empty(list));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2006-01-06 01:10:56 -07:00
|
|
|
spin_unlock(&zone->lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2014-06-04 17:10:17 -06:00
|
|
|
static void free_one_page(struct zone *zone,
|
|
|
|
struct page *page, unsigned long pfn,
|
2014-06-04 17:10:21 -06:00
|
|
|
unsigned int order,
|
2009-06-16 16:32:07 -06:00
|
|
|
int migratetype)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2014-08-06 17:07:16 -06:00
|
|
|
unsigned long nr_scanned;
|
2006-09-26 00:31:48 -06:00
|
|
|
spin_lock(&zone->lock);
|
2016-07-28 16:45:31 -06:00
|
|
|
nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
|
2014-08-06 17:07:16 -06:00
|
|
|
if (nr_scanned)
|
2016-07-28 16:45:31 -06:00
|
|
|
__mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);
|
2009-06-16 16:32:13 -06:00
|
|
|
|
mm/page_alloc: fix incorrect isolation behavior by rechecking migratetype
Before describing bugs itself, I first explain definition of freepage.
1. pages on buddy list are counted as freepage.
2. pages on isolate migratetype buddy list are *not* counted as freepage.
3. pages on cma buddy list are counted as CMA freepage, too.
Now, I describe problems and related patch.
Patch 1: There is race conditions on getting pageblock migratetype that
it results in misplacement of freepages on buddy list, incorrect
freepage count and un-availability of freepage.
Patch 2: Freepages on pcp list could have stale cached information to
determine migratetype of buddy list to go. This causes misplacement of
freepages on buddy list and incorrect freepage count.
Patch 4: Merging between freepages on different migratetype of
pageblocks will cause freepages accouting problem. This patch fixes it.
Without patchset [3], above problem doesn't happens on my CMA allocation
test, because CMA reserved pages aren't used at all. So there is no
chance for above race.
With patchset [3], I did simple CMA allocation test and get below
result:
- Virtual machine, 4 cpus, 1024 MB memory, 256 MB CMA reservation
- run kernel build (make -j16) on background
- 30 times CMA allocation(8MB * 30 = 240MB) attempts in 5 sec interval
- Result: more than 5000 freepage count are missed
With patchset [3] and this patchset, I found that no freepage count are
missed so that I conclude that problems are solved.
On my simple memory offlining test, these problems also occur on that
environment, too.
This patch (of 4):
There are two paths to reach core free function of buddy allocator,
__free_one_page(), one is free_one_page()->__free_one_page() and the
other is free_hot_cold_page()->free_pcppages_bulk()->__free_one_page().
Each paths has race condition causing serious problems. At first, this
patch is focused on first type of freepath. And then, following patch
will solve the problem in second type of freepath.
In the first type of freepath, we got migratetype of freeing page
without holding the zone lock, so it could be racy. There are two cases
of this race.
1. pages are added to isolate buddy list after restoring orignal
migratetype
CPU1 CPU2
get migratetype => return MIGRATE_ISOLATE
call free_one_page() with MIGRATE_ISOLATE
grab the zone lock
unisolate pageblock
release the zone lock
grab the zone lock
call __free_one_page() with MIGRATE_ISOLATE
freepage go into isolate buddy list,
although pageblock is already unisolated
This may cause two problems. One is that we can't use this page anymore
until next isolation attempt of this pageblock, because freepage is on
isolate buddy list. The other is that freepage accouting could be wrong
due to merging between different buddy list. Freepages on isolate buddy
list aren't counted as freepage, but ones on normal buddy list are
counted as freepage. If merge happens, buddy freepage on normal buddy
list is inevitably moved to isolate buddy list without any consideration
of freepage accouting so it could be incorrect.
2. pages are added to normal buddy list while pageblock is isolated.
It is similar with above case.
This also may cause two problems. One is that we can't keep these
freepages from being allocated. Although this pageblock is isolated,
freepage would be added to normal buddy list so that it could be
allocated without any restriction. And the other problem is same as
case 1, that it, incorrect freepage accouting.
This race condition would be prevented by checking migratetype again
with holding the zone lock. Because it is somewhat heavy operation and
it isn't needed in common case, we want to avoid rechecking as much as
possible. So this patch introduce new variable, nr_isolate_pageblock in
struct zone to check if there is isolated pageblock. With this, we can
avoid to re-check migratetype in common case and do it only if there is
isolated pageblock or migratetype is MIGRATE_ISOLATE. This solve above
mentioned problems.
Changes from v3:
Add one more check in free_one_page() that checks whether migratetype is
MIGRATE_ISOLATE or not. Without this, abovementioned case 1 could happens.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Minchan Kim <minchan@kernel.org>
Acked-by: Michal Nazarewicz <mina86@mina86.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Laura Abbott <lauraa@codeaurora.org>
Cc: Heesub Shin <heesub.shin@samsung.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: Ritesh Harjani <ritesh.list@gmail.com>
Cc: Gioh Kim <gioh.kim@lge.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-11-13 16:19:11 -07:00
|
|
|
if (unlikely(has_isolate_pageblock(zone) ||
|
|
|
|
is_migrate_isolate(migratetype))) {
|
|
|
|
migratetype = get_pfnblock_migratetype(page, pfn);
|
|
|
|
}
|
2014-06-04 17:10:17 -06:00
|
|
|
__free_one_page(page, pfn, zone, order, migratetype);
|
2006-09-26 00:31:48 -06:00
|
|
|
spin_unlock(&zone->lock);
|
2006-01-08 02:00:42 -07:00
|
|
|
}
|
|
|
|
|
2015-06-30 15:56:45 -06:00
|
|
|
static void __meminit __init_single_page(struct page *page, unsigned long pfn,
|
|
|
|
unsigned long zone, int nid)
|
|
|
|
{
|
|
|
|
set_page_links(page, zone, nid, pfn);
|
|
|
|
init_page_count(page);
|
|
|
|
page_mapcount_reset(page);
|
|
|
|
page_cpupid_reset_last(page);
|
|
|
|
|
|
|
|
INIT_LIST_HEAD(&page->lru);
|
|
|
|
#ifdef WANT_PAGE_VIRTUAL
|
|
|
|
/* The shift won't overflow because ZONE_NORMAL is below 4G. */
|
|
|
|
if (!is_highmem_idx(zone))
|
|
|
|
set_page_address(page, __va(pfn << PAGE_SHIFT));
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __meminit __init_single_pfn(unsigned long pfn, unsigned long zone,
|
|
|
|
int nid)
|
|
|
|
{
|
|
|
|
return __init_single_page(pfn_to_page(pfn), pfn, zone, nid);
|
|
|
|
}
|
|
|
|
|
2015-06-30 15:57:05 -06:00
|
|
|
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
|
|
|
|
static void init_reserved_page(unsigned long pfn)
|
|
|
|
{
|
|
|
|
pg_data_t *pgdat;
|
|
|
|
int nid, zid;
|
|
|
|
|
|
|
|
if (!early_page_uninitialised(pfn))
|
|
|
|
return;
|
|
|
|
|
|
|
|
nid = early_pfn_to_nid(pfn);
|
|
|
|
pgdat = NODE_DATA(nid);
|
|
|
|
|
|
|
|
for (zid = 0; zid < MAX_NR_ZONES; zid++) {
|
|
|
|
struct zone *zone = &pgdat->node_zones[zid];
|
|
|
|
|
|
|
|
if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
__init_single_pfn(pfn, zid, nid);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline void init_reserved_page(unsigned long pfn)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
|
|
|
|
|
2015-06-30 15:56:48 -06:00
|
|
|
/*
|
|
|
|
* Initialised pages do not have PageReserved set. This function is
|
|
|
|
* called for each range allocated by the bootmem allocator and
|
|
|
|
* marks the pages PageReserved. The remaining valid pages are later
|
|
|
|
* sent to the buddy page allocator.
|
|
|
|
*/
|
2016-05-20 17:58:38 -06:00
|
|
|
void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
|
2015-06-30 15:56:48 -06:00
|
|
|
{
|
|
|
|
unsigned long start_pfn = PFN_DOWN(start);
|
|
|
|
unsigned long end_pfn = PFN_UP(end);
|
|
|
|
|
2015-06-30 15:57:05 -06:00
|
|
|
for (; start_pfn < end_pfn; start_pfn++) {
|
|
|
|
if (pfn_valid(start_pfn)) {
|
|
|
|
struct page *page = pfn_to_page(start_pfn);
|
|
|
|
|
|
|
|
init_reserved_page(start_pfn);
|
2015-11-06 17:29:54 -07:00
|
|
|
|
|
|
|
/* Avoid false-positive PageTail() */
|
|
|
|
INIT_LIST_HEAD(&page->lru);
|
|
|
|
|
2015-06-30 15:57:05 -06:00
|
|
|
SetPageReserved(page);
|
|
|
|
}
|
|
|
|
}
|
2015-06-30 15:56:48 -06:00
|
|
|
}
|
|
|
|
|
2010-05-24 15:32:38 -06:00
|
|
|
static void __free_pages_ok(struct page *page, unsigned int order)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
2012-10-08 17:32:11 -06:00
|
|
|
int migratetype;
|
2014-06-04 17:10:17 -06:00
|
|
|
unsigned long pfn = page_to_pfn(page);
|
2010-05-24 15:32:38 -06:00
|
|
|
|
2016-05-19 18:14:38 -06:00
|
|
|
if (!free_pages_prepare(page, order, true))
|
2010-05-24 15:32:38 -06:00
|
|
|
return;
|
|
|
|
|
2014-06-04 17:10:19 -06:00
|
|
|
migratetype = get_pfnblock_migratetype(page, pfn);
|
2006-01-06 01:10:56 -07:00
|
|
|
local_irq_save(flags);
|
2006-06-30 02:55:45 -06:00
|
|
|
__count_vm_events(PGFREE, 1 << order);
|
2014-06-04 17:10:17 -06:00
|
|
|
free_one_page(page_zone(page), page, pfn, order, migratetype);
|
2006-01-06 01:10:56 -07:00
|
|
|
local_irq_restore(flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2016-05-19 18:11:37 -06:00
|
|
|
static void __init __free_pages_boot_core(struct page *page, unsigned int order)
|
2006-01-06 01:11:08 -07:00
|
|
|
{
|
2012-01-10 16:08:10 -07:00
|
|
|
unsigned int nr_pages = 1 << order;
|
2013-09-11 15:20:37 -06:00
|
|
|
struct page *p = page;
|
2012-01-10 16:08:10 -07:00
|
|
|
unsigned int loop;
|
2006-01-06 01:11:08 -07:00
|
|
|
|
2013-09-11 15:20:37 -06:00
|
|
|
prefetchw(p);
|
|
|
|
for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
|
|
|
|
prefetchw(p + 1);
|
2012-01-10 16:08:10 -07:00
|
|
|
__ClearPageReserved(p);
|
|
|
|
set_page_count(p, 0);
|
2006-01-06 01:11:08 -07:00
|
|
|
}
|
2013-09-11 15:20:37 -06:00
|
|
|
__ClearPageReserved(p);
|
|
|
|
set_page_count(p, 0);
|
2012-01-10 16:08:10 -07:00
|
|
|
|
2013-09-11 15:20:37 -06:00
|
|
|
page_zone(page)->managed_pages += nr_pages;
|
2012-01-10 16:08:10 -07:00
|
|
|
set_page_refcounted(page);
|
|
|
|
__free_pages(page, order);
|
2006-01-06 01:11:08 -07:00
|
|
|
}
|
|
|
|
|
2015-06-30 15:56:59 -06:00
|
|
|
#if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \
|
|
|
|
defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
|
2015-08-06 16:46:13 -06:00
|
|
|
|
2015-06-30 15:56:59 -06:00
|
|
|
static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
|
|
|
|
|
|
|
|
int __meminit early_pfn_to_nid(unsigned long pfn)
|
|
|
|
{
|
2015-08-06 16:46:13 -06:00
|
|
|
static DEFINE_SPINLOCK(early_pfn_lock);
|
2015-06-30 15:56:59 -06:00
|
|
|
int nid;
|
|
|
|
|
2015-08-06 16:46:13 -06:00
|
|
|
spin_lock(&early_pfn_lock);
|
2015-06-30 15:56:59 -06:00
|
|
|
nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
|
2015-08-06 16:46:13 -06:00
|
|
|
if (nid < 0)
|
2016-07-14 13:07:20 -06:00
|
|
|
nid = first_online_node;
|
2015-08-06 16:46:13 -06:00
|
|
|
spin_unlock(&early_pfn_lock);
|
|
|
|
|
|
|
|
return nid;
|
2015-06-30 15:56:59 -06:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_NODES_SPAN_OTHER_NODES
|
|
|
|
static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node,
|
|
|
|
struct mminit_pfnnid_cache *state)
|
|
|
|
{
|
|
|
|
int nid;
|
|
|
|
|
|
|
|
nid = __early_pfn_to_nid(pfn, state);
|
|
|
|
if (nid >= 0 && nid != node)
|
|
|
|
return false;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Only safe to use early in boot when initialisation is single-threaded */
|
|
|
|
static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
|
|
|
|
{
|
|
|
|
return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache);
|
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node,
|
|
|
|
struct mminit_pfnnid_cache *state)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
2015-06-30 15:57:27 -06:00
|
|
|
void __init __free_pages_bootmem(struct page *page, unsigned long pfn,
|
2015-06-30 15:57:02 -06:00
|
|
|
unsigned int order)
|
|
|
|
{
|
|
|
|
if (early_page_uninitialised(pfn))
|
|
|
|
return;
|
2016-05-19 18:11:37 -06:00
|
|
|
return __free_pages_boot_core(page, order);
|
2015-06-30 15:57:02 -06:00
|
|
|
}
|
|
|
|
|
2016-03-15 15:57:51 -06:00
|
|
|
/*
|
|
|
|
* Check that the whole (or subset of) a pageblock given by the interval of
|
|
|
|
* [start_pfn, end_pfn) is valid and within the same zone, before scanning it
|
|
|
|
* with the migration of free compaction scanner. The scanners then need to
|
|
|
|
* use only pfn_valid_within() check for arches that allow holes within
|
|
|
|
* pageblocks.
|
|
|
|
*
|
|
|
|
* Return struct page pointer of start_pfn, or NULL if checks were not passed.
|
|
|
|
*
|
|
|
|
* It's possible on some configurations to have a setup like node0 node1 node0
|
|
|
|
* i.e. it's possible that all pages within a zones range of pages do not
|
|
|
|
* belong to a single zone. We assume that a border between node0 and node1
|
|
|
|
* can occur within a single pageblock, but not a node0 node1 node0
|
|
|
|
* interleaving within a single pageblock. It is therefore sufficient to check
|
|
|
|
* the first and last page of a pageblock and avoid checking each individual
|
|
|
|
* page in a pageblock.
|
|
|
|
*/
|
|
|
|
struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
|
|
|
|
unsigned long end_pfn, struct zone *zone)
|
|
|
|
{
|
|
|
|
struct page *start_page;
|
|
|
|
struct page *end_page;
|
|
|
|
|
|
|
|
/* end_pfn is one past the range we are checking */
|
|
|
|
end_pfn--;
|
|
|
|
|
|
|
|
if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
start_page = pfn_to_page(start_pfn);
|
|
|
|
|
|
|
|
if (page_zone(start_page) != zone)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
end_page = pfn_to_page(end_pfn);
|
|
|
|
|
|
|
|
/* This gives a shorter code than deriving page_zone(end_page) */
|
|
|
|
if (page_zone_id(start_page) != page_zone_id(end_page))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return start_page;
|
|
|
|
}
|
|
|
|
|
|
|
|
void set_zone_contiguous(struct zone *zone)
|
|
|
|
{
|
|
|
|
unsigned long block_start_pfn = zone->zone_start_pfn;
|
|
|
|
unsigned long block_end_pfn;
|
|
|
|
|
|
|
|
block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
|
|
|
|
for (; block_start_pfn < zone_end_pfn(zone);
|
|
|
|
block_start_pfn = block_end_pfn,
|
|
|
|
block_end_pfn += pageblock_nr_pages) {
|
|
|
|
|
|
|
|
block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
|
|
|
|
|
|
|
|
if (!__pageblock_pfn_to_page(block_start_pfn,
|
|
|
|
block_end_pfn, zone))
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* We confirm that there is no hole */
|
|
|
|
zone->contiguous = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
void clear_zone_contiguous(struct zone *zone)
|
|
|
|
{
|
|
|
|
zone->contiguous = false;
|
|
|
|
}
|
|
|
|
|
2015-06-30 15:57:05 -06:00
|
|
|
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
|
2015-06-30 15:57:27 -06:00
|
|
|
static void __init deferred_free_range(struct page *page,
|
2015-06-30 15:57:16 -06:00
|
|
|
unsigned long pfn, int nr_pages)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!page)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Free a large naturally-aligned chunk if possible */
|
|
|
|
if (nr_pages == MAX_ORDER_NR_PAGES &&
|
|
|
|
(pfn & (MAX_ORDER_NR_PAGES-1)) == 0) {
|
2015-06-30 15:57:20 -06:00
|
|
|
set_pageblock_migratetype(page, MIGRATE_MOVABLE);
|
2016-05-19 18:11:37 -06:00
|
|
|
__free_pages_boot_core(page, MAX_ORDER-1);
|
2015-06-30 15:57:16 -06:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2016-05-19 18:11:37 -06:00
|
|
|
for (i = 0; i < nr_pages; i++, page++)
|
|
|
|
__free_pages_boot_core(page, 0);
|
2015-06-30 15:57:16 -06:00
|
|
|
}
|
|
|
|
|
2015-08-06 16:46:16 -06:00
|
|
|
/* Completion tracking for deferred_init_memmap() threads */
|
|
|
|
static atomic_t pgdat_init_n_undone __initdata;
|
|
|
|
static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
|
|
|
|
|
|
|
|
static inline void __init pgdat_init_report_one_done(void)
|
|
|
|
{
|
|
|
|
if (atomic_dec_and_test(&pgdat_init_n_undone))
|
|
|
|
complete(&pgdat_init_all_done_comp);
|
|
|
|
}
|
2015-06-30 15:57:27 -06:00
|
|
|
|
2015-06-30 15:57:05 -06:00
|
|
|
/* Initialise remaining memory on a node */
|
2015-06-30 15:57:27 -06:00
|
|
|
static int __init deferred_init_memmap(void *data)
|
2015-06-30 15:57:05 -06:00
|
|
|
{
|
2015-06-30 15:57:27 -06:00
|
|
|
pg_data_t *pgdat = data;
|
|
|
|
int nid = pgdat->node_id;
|
2015-06-30 15:57:05 -06:00
|
|
|
struct mminit_pfnnid_cache nid_init_state = { };
|
|
|
|
unsigned long start = jiffies;
|
|
|
|
unsigned long nr_pages = 0;
|
|
|
|
unsigned long walk_start, walk_end;
|
|
|
|
int i, zid;
|
|
|
|
struct zone *zone;
|
|
|
|
unsigned long first_init_pfn = pgdat->first_deferred_pfn;
|
2015-06-30 15:57:27 -06:00
|
|
|
const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
|
2015-06-30 15:57:05 -06:00
|
|
|
|
2015-06-30 15:57:27 -06:00
|
|
|
if (first_init_pfn == ULONG_MAX) {
|
2015-08-06 16:46:16 -06:00
|
|
|
pgdat_init_report_one_done();
|
2015-06-30 15:57:27 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Bind memory initialisation thread to a local node if possible */
|
|
|
|
if (!cpumask_empty(cpumask))
|
|
|
|
set_cpus_allowed_ptr(current, cpumask);
|
2015-06-30 15:57:05 -06:00
|
|
|
|
|
|
|
/* Sanity check boundaries */
|
|
|
|
BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
|
|
|
|
BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
|
|
|
|
pgdat->first_deferred_pfn = ULONG_MAX;
|
|
|
|
|
|
|
|
/* Only the highest zone is deferred so find it */
|
|
|
|
for (zid = 0; zid < MAX_NR_ZONES; zid++) {
|
|
|
|
zone = pgdat->node_zones + zid;
|
|
|
|
if (first_init_pfn < zone_end_pfn(zone))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
for_each_mem_pfn_range(i, nid, &walk_start, &walk_end, NULL) {
|
|
|
|
unsigned long pfn, end_pfn;
|
2015-06-30 15:57:09 -06:00
|
|
|
struct page *page = NULL;
|
2015-06-30 15:57:16 -06:00
|
|
|
struct page *free_base_page = NULL;
|
|
|
|
unsigned long free_base_pfn = 0;
|
|
|
|
int nr_to_free = 0;
|
2015-06-30 15:57:05 -06:00
|
|
|
|
|
|
|
end_pfn = min(walk_end, zone_end_pfn(zone));
|
|
|
|
pfn = first_init_pfn;
|
|
|
|
if (pfn < walk_start)
|
|
|
|
pfn = walk_start;
|
|
|
|
if (pfn < zone->zone_start_pfn)
|
|
|
|
pfn = zone->zone_start_pfn;
|
|
|
|
|
|
|
|
for (; pfn < end_pfn; pfn++) {
|
2015-06-30 15:57:09 -06:00
|
|
|
if (!pfn_valid_within(pfn))
|
2015-06-30 15:57:16 -06:00
|
|
|
goto free_range;
|
2015-06-30 15:57:05 -06:00
|
|
|
|
2015-06-30 15:57:09 -06:00
|
|
|
/*
|
|
|
|
* Ensure pfn_valid is checked every
|
|
|
|
* MAX_ORDER_NR_PAGES for memory holes
|
|
|
|
*/
|
|
|
|
if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) {
|
|
|
|
if (!pfn_valid(pfn)) {
|
|
|
|
page = NULL;
|
2015-06-30 15:57:16 -06:00
|
|
|
goto free_range;
|
2015-06-30 15:57:09 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!meminit_pfn_in_nid(pfn, nid, &nid_init_state)) {
|
|
|
|
page = NULL;
|
2015-06-30 15:57:16 -06:00
|
|
|
goto free_range;
|
2015-06-30 15:57:09 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Minimise pfn page lookups and scheduler checks */
|
|
|
|
if (page && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) {
|
|
|
|
page++;
|
|
|
|
} else {
|
2015-06-30 15:57:16 -06:00
|
|
|
nr_pages += nr_to_free;
|
|
|
|
deferred_free_range(free_base_page,
|
|
|
|
free_base_pfn, nr_to_free);
|
|
|
|
free_base_page = NULL;
|
|
|
|
free_base_pfn = nr_to_free = 0;
|
|
|
|
|
2015-06-30 15:57:09 -06:00
|
|
|
page = pfn_to_page(pfn);
|
|
|
|
cond_resched();
|
|
|
|
}
|
2015-06-30 15:57:05 -06:00
|
|
|
|
|
|
|
if (page->flags) {
|
|
|
|
VM_BUG_ON(page_zone(page) != zone);
|
2015-06-30 15:57:16 -06:00
|
|
|
goto free_range;
|
2015-06-30 15:57:05 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
__init_single_page(page, pfn, zid, nid);
|
2015-06-30 15:57:16 -06:00
|
|
|
if (!free_base_page) {
|
|
|
|
free_base_page = page;
|
|
|
|
free_base_pfn = pfn;
|
|
|
|
nr_to_free = 0;
|
|
|
|
}
|
|
|
|
nr_to_free++;
|
|
|
|
|
|
|
|
/* Where possible, batch up pages for a single free */
|
|
|
|
continue;
|
|
|
|
free_range:
|
|
|
|
/* Free the current block of pages to allocator */
|
|
|
|
nr_pages += nr_to_free;
|
|
|
|
deferred_free_range(free_base_page, free_base_pfn,
|
|
|
|
nr_to_free);
|
|
|
|
free_base_page = NULL;
|
|
|
|
free_base_pfn = nr_to_free = 0;
|
2015-06-30 15:57:05 -06:00
|
|
|
}
|
2015-06-30 15:57:16 -06:00
|
|
|
|
2015-06-30 15:57:05 -06:00
|
|
|
first_init_pfn = max(end_pfn, first_init_pfn);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Sanity check that the next zone really is unpopulated */
|
|
|
|
WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
|
|
|
|
|
2015-06-30 15:57:27 -06:00
|
|
|
pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages,
|
2015-06-30 15:57:05 -06:00
|
|
|
jiffies_to_msecs(jiffies - start));
|
2015-08-06 16:46:16 -06:00
|
|
|
|
|
|
|
pgdat_init_report_one_done();
|
2015-06-30 15:57:27 -06:00
|
|
|
return 0;
|
|
|
|
}
|
2016-03-15 15:57:51 -06:00
|
|
|
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
|
2015-06-30 15:57:27 -06:00
|
|
|
|
|
|
|
void __init page_alloc_init_late(void)
|
|
|
|
{
|
2016-03-15 15:57:51 -06:00
|
|
|
struct zone *zone;
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
|
2015-06-30 15:57:27 -06:00
|
|
|
int nid;
|
|
|
|
|
2015-08-06 16:46:16 -06:00
|
|
|
/* There will be num_node_state(N_MEMORY) threads */
|
|
|
|
atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
|
2015-06-30 15:57:27 -06:00
|
|
|
for_each_node_state(nid, N_MEMORY) {
|
|
|
|
kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Block until all are initialised */
|
2015-08-06 16:46:16 -06:00
|
|
|
wait_for_completion(&pgdat_init_all_done_comp);
|
2015-08-06 16:46:20 -06:00
|
|
|
|
|
|
|
/* Reinit limits that are based on free pages after the kernel is up */
|
|
|
|
files_maxfiles_init();
|
2016-03-15 15:57:51 -06:00
|
|
|
#endif
|
|
|
|
|
|
|
|
for_each_populated_zone(zone)
|
|
|
|
set_zone_contiguous(zone);
|
2015-06-30 15:57:05 -06:00
|
|
|
}
|
|
|
|
|
2011-12-29 05:09:50 -07:00
|
|
|
#ifdef CONFIG_CMA
|
2013-08-22 23:52:52 -06:00
|
|
|
/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
|
2011-12-29 05:09:50 -07:00
|
|
|
void __init init_cma_reserved_pageblock(struct page *page)
|
|
|
|
{
|
|
|
|
unsigned i = pageblock_nr_pages;
|
|
|
|
struct page *p = page;
|
|
|
|
|
|
|
|
do {
|
|
|
|
__ClearPageReserved(p);
|
|
|
|
set_page_count(p, 0);
|
|
|
|
} while (++p, --i);
|
|
|
|
|
|
|
|
set_pageblock_migratetype(page, MIGRATE_CMA);
|
2014-07-02 16:22:35 -06:00
|
|
|
|
|
|
|
if (pageblock_order >= MAX_ORDER) {
|
|
|
|
i = pageblock_nr_pages;
|
|
|
|
p = page;
|
|
|
|
do {
|
|
|
|
set_page_refcounted(p);
|
|
|
|
__free_pages(p, MAX_ORDER - 1);
|
|
|
|
p += MAX_ORDER_NR_PAGES;
|
|
|
|
} while (i -= MAX_ORDER_NR_PAGES);
|
|
|
|
} else {
|
|
|
|
set_page_refcounted(page);
|
|
|
|
__free_pages(page, pageblock_order);
|
|
|
|
}
|
|
|
|
|
2013-07-03 16:03:21 -06:00
|
|
|
adjust_managed_page_count(page, pageblock_nr_pages);
|
2011-12-29 05:09:50 -07:00
|
|
|
}
|
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The order of subdivision here is critical for the IO subsystem.
|
|
|
|
* Please do not alter this order without good reasons and regression
|
|
|
|
* testing. Specifically, as large blocks of memory are subdivided,
|
|
|
|
* the order in which smaller blocks are delivered depends on the order
|
|
|
|
* they're subdivided in this function. This is the primary factor
|
|
|
|
* influencing the order in which pages are delivered to the IO
|
|
|
|
* subsystem according to empirical testing, and this is also justified
|
|
|
|
* by considering the behavior of a buddy system containing a single
|
|
|
|
* large block of memory acted on by a series of small allocations.
|
|
|
|
* This behavior is a critical factor in sglist merging's success.
|
|
|
|
*
|
2012-12-06 02:39:54 -07:00
|
|
|
* -- nyc
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2006-01-06 01:11:01 -07:00
|
|
|
static inline void expand(struct zone *zone, struct page *page,
|
2007-10-16 02:25:48 -06:00
|
|
|
int low, int high, struct free_area *area,
|
|
|
|
int migratetype)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
unsigned long size = 1 << high;
|
|
|
|
|
|
|
|
while (high > low) {
|
|
|
|
area--;
|
|
|
|
high--;
|
|
|
|
size >>= 1;
|
2014-01-23 16:52:54 -07:00
|
|
|
VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
|
2012-01-10 16:07:28 -07:00
|
|
|
|
2014-12-12 17:55:01 -07:00
|
|
|
if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) &&
|
mm/debug-pagealloc: prepare boottime configurable on/off
Until now, debug-pagealloc needs extra flags in struct page, so we need to
recompile whole source code when we decide to use it. This is really
painful, because it takes some time to recompile and sometimes rebuild is
not possible due to third party module depending on struct page. So, we
can't use this good feature in many cases.
Now, we have the page extension feature that allows us to insert extra
flags to outside of struct page. This gets rid of third party module
issue mentioned above. And, this allows us to determine if we need extra
memory for this page extension in boottime. With these property, we can
avoid using debug-pagealloc in boottime with low computational overhead in
the kernel built with CONFIG_DEBUG_PAGEALLOC. This will help our
development process greatly.
This patch is the preparation step to achive above goal. debug-pagealloc
originally uses extra field of struct page, but, after this patch, it will
use field of struct page_ext. Because memory for page_ext is allocated
later than initialization of page allocator in CONFIG_SPARSEMEM, we should
disable debug-pagealloc feature temporarily until initialization of
page_ext. This patch implements this.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:49 -07:00
|
|
|
debug_guardpage_enabled() &&
|
2014-12-12 17:55:01 -07:00
|
|
|
high < debug_guardpage_minorder()) {
|
2012-01-10 16:07:28 -07:00
|
|
|
/*
|
|
|
|
* Mark as guard pages (or page), that will allow to
|
|
|
|
* merge back to allocator when buddy will be freed.
|
|
|
|
* Corresponding page table entries will not be touched,
|
|
|
|
* pages will stay not present in virtual address space
|
|
|
|
*/
|
2014-12-12 17:55:01 -07:00
|
|
|
set_page_guard(zone, &page[size], high, migratetype);
|
2012-01-10 16:07:28 -07:00
|
|
|
continue;
|
|
|
|
}
|
2007-10-16 02:25:48 -06:00
|
|
|
list_add(&page[size].lru, &area->free_list[migratetype]);
|
2005-04-16 16:20:36 -06:00
|
|
|
area->nr_free++;
|
|
|
|
set_page_order(&page[size], high);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:41 -06:00
|
|
|
static void check_new_page_bad(struct page *page)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2016-05-19 18:14:41 -06:00
|
|
|
const char *bad_reason = NULL;
|
|
|
|
unsigned long bad_flags = 0;
|
2016-05-19 18:14:15 -06:00
|
|
|
|
2016-01-15 17:53:42 -07:00
|
|
|
if (unlikely(atomic_read(&page->_mapcount) != -1))
|
2014-01-23 16:52:49 -07:00
|
|
|
bad_reason = "nonzero mapcount";
|
|
|
|
if (unlikely(page->mapping != NULL))
|
|
|
|
bad_reason = "non-NULL mapping";
|
2016-03-17 15:19:26 -06:00
|
|
|
if (unlikely(page_ref_count(page) != 0))
|
2014-01-23 16:52:49 -07:00
|
|
|
bad_reason = "nonzero _count";
|
mm: check __PG_HWPOISON separately from PAGE_FLAGS_CHECK_AT_*
The race condition addressed in commit add05cecef80 ("mm: soft-offline:
don't free target page in successful page migration") was not closed
completely, because that can happen not only for soft-offline, but also
for hard-offline. Consider that a slab page is about to be freed into
buddy pool, and then an uncorrected memory error hits the page just
after entering __free_one_page(), then VM_BUG_ON_PAGE(page->flags &
PAGE_FLAGS_CHECK_AT_PREP) is triggered, despite the fact that it's not
necessary because the data on the affected page is not consumed.
To solve it, this patch drops __PG_HWPOISON from page flag checks at
allocation/free time. I think it's justified because __PG_HWPOISON
flags is defined to prevent the page from being reused, and setting it
outside the page's alloc-free cycle is a designed behavior (not a bug.)
For recent months, I was annoyed about BUG_ON when soft-offlined page
remains on lru cache list for a while, which is avoided by calling
put_page() instead of putback_lru_page() in page migration's success
path. This means that this patch reverts a major change from commit
add05cecef80 about the new refcounting rule of soft-offlined pages, so
"reuse window" revives. This will be closed by a subsequent patch.
Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Dean Nelson <dnelson@redhat.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Hugh Dickins <hughd@google.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-08-06 16:47:08 -06:00
|
|
|
if (unlikely(page->flags & __PG_HWPOISON)) {
|
|
|
|
bad_reason = "HWPoisoned (hardware-corrupted)";
|
|
|
|
bad_flags = __PG_HWPOISON;
|
2016-05-20 17:58:50 -06:00
|
|
|
/* Don't complain about hwpoisoned pages */
|
|
|
|
page_mapcount_reset(page); /* remove PageBuddy */
|
|
|
|
return;
|
mm: check __PG_HWPOISON separately from PAGE_FLAGS_CHECK_AT_*
The race condition addressed in commit add05cecef80 ("mm: soft-offline:
don't free target page in successful page migration") was not closed
completely, because that can happen not only for soft-offline, but also
for hard-offline. Consider that a slab page is about to be freed into
buddy pool, and then an uncorrected memory error hits the page just
after entering __free_one_page(), then VM_BUG_ON_PAGE(page->flags &
PAGE_FLAGS_CHECK_AT_PREP) is triggered, despite the fact that it's not
necessary because the data on the affected page is not consumed.
To solve it, this patch drops __PG_HWPOISON from page flag checks at
allocation/free time. I think it's justified because __PG_HWPOISON
flags is defined to prevent the page from being reused, and setting it
outside the page's alloc-free cycle is a designed behavior (not a bug.)
For recent months, I was annoyed about BUG_ON when soft-offlined page
remains on lru cache list for a while, which is avoided by calling
put_page() instead of putback_lru_page() in page migration's success
path. This means that this patch reverts a major change from commit
add05cecef80 about the new refcounting rule of soft-offlined pages, so
"reuse window" revives. This will be closed by a subsequent patch.
Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Dean Nelson <dnelson@redhat.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Hugh Dickins <hughd@google.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-08-06 16:47:08 -06:00
|
|
|
}
|
2014-01-23 16:52:49 -07:00
|
|
|
if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
|
|
|
|
bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
|
|
|
|
bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
|
|
|
|
}
|
2014-12-10 16:44:58 -07:00
|
|
|
#ifdef CONFIG_MEMCG
|
|
|
|
if (unlikely(page->mem_cgroup))
|
|
|
|
bad_reason = "page still charged to cgroup";
|
|
|
|
#endif
|
2016-05-19 18:14:41 -06:00
|
|
|
bad_page(page, bad_reason, bad_flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This page is about to be returned from the page allocator
|
|
|
|
*/
|
|
|
|
static inline int check_new_page(struct page *page)
|
|
|
|
{
|
|
|
|
if (likely(page_expected_state(page,
|
|
|
|
PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
check_new_page_bad(page);
|
|
|
|
return 1;
|
2009-09-16 03:50:12 -06:00
|
|
|
}
|
|
|
|
|
2016-03-15 15:56:30 -06:00
|
|
|
static inline bool free_pages_prezeroed(bool poisoned)
|
|
|
|
{
|
|
|
|
return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
|
|
|
|
page_poisoning_enabled() && poisoned;
|
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:35 -06:00
|
|
|
#ifdef CONFIG_DEBUG_VM
|
|
|
|
static bool check_pcp_refill(struct page *page)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool check_new_pcp(struct page *page)
|
|
|
|
{
|
|
|
|
return check_new_page(page);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static bool check_pcp_refill(struct page *page)
|
|
|
|
{
|
|
|
|
return check_new_page(page);
|
|
|
|
}
|
|
|
|
static bool check_new_pcp(struct page *page)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_DEBUG_VM */
|
|
|
|
|
|
|
|
static bool check_new_pages(struct page *page, unsigned int order)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
for (i = 0; i < (1 << order); i++) {
|
|
|
|
struct page *p = page + i;
|
|
|
|
|
|
|
|
if (unlikely(check_new_page(p)))
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2016-07-26 16:23:58 -06:00
|
|
|
inline void post_alloc_hook(struct page *page, unsigned int order,
|
|
|
|
gfp_t gfp_flags)
|
|
|
|
{
|
|
|
|
set_page_private(page, 0);
|
|
|
|
set_page_refcounted(page);
|
|
|
|
|
|
|
|
arch_alloc_page(page, order);
|
|
|
|
kernel_map_pages(page, 1 << order, 1);
|
|
|
|
kernel_poison_pages(page, 1 << order, 1);
|
|
|
|
kasan_alloc_pages(page, order);
|
|
|
|
set_page_owner(page, order, gfp_flags);
|
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:35 -06:00
|
|
|
static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
|
2016-05-19 18:13:38 -06:00
|
|
|
unsigned int alloc_flags)
|
2009-09-16 03:50:12 -06:00
|
|
|
{
|
|
|
|
int i;
|
2016-03-15 15:56:30 -06:00
|
|
|
bool poisoned = true;
|
2009-09-16 03:50:12 -06:00
|
|
|
|
|
|
|
for (i = 0; i < (1 << order); i++) {
|
|
|
|
struct page *p = page + i;
|
2016-03-15 15:56:30 -06:00
|
|
|
if (poisoned)
|
|
|
|
poisoned &= page_is_poisoned(p);
|
2009-09-16 03:50:12 -06:00
|
|
|
}
|
2005-11-21 22:32:20 -07:00
|
|
|
|
2016-07-26 16:23:58 -06:00
|
|
|
post_alloc_hook(page, order, gfp_flags);
|
2006-03-22 01:08:41 -07:00
|
|
|
|
2016-03-15 15:56:30 -06:00
|
|
|
if (!free_pages_prezeroed(poisoned) && (gfp_flags & __GFP_ZERO))
|
2015-06-24 17:56:36 -06:00
|
|
|
for (i = 0; i < (1 << order); i++)
|
|
|
|
clear_highpage(page + i);
|
2006-03-22 01:08:41 -07:00
|
|
|
|
|
|
|
if (order && (gfp_flags & __GFP_COMP))
|
|
|
|
prep_compound_page(page, order);
|
|
|
|
|
mm: set page->pfmemalloc in prep_new_page()
The possibility of replacing the numerous parameters of alloc_pages*
functions with a single structure has been discussed when Minchan proposed
to expand the x86 kernel stack [1]. This series implements the change,
along with few more cleanups/microoptimizations.
The series is based on next-20150108 and I used gcc 4.8.3 20140627 on
openSUSE 13.2 for compiling. Config includess NUMA and COMPACTION.
The core change is the introduction of a new struct alloc_context, which looks
like this:
struct alloc_context {
struct zonelist *zonelist;
nodemask_t *nodemask;
struct zone *preferred_zone;
int classzone_idx;
int migratetype;
enum zone_type high_zoneidx;
};
All the contents is mostly constant, except that __alloc_pages_slowpath()
changes preferred_zone, classzone_idx and potentially zonelist. But
that's not a problem in case control returns to retry_cpuset: in
__alloc_pages_nodemask(), those will be reset to initial values again
(although it's a bit subtle). On the other hand, gfp_flags and alloc_info
mutate so much that it doesn't make sense to put them into alloc_context.
Still, the result is one parameter instead of up to 7. This is all in
Patch 2.
Patch 3 is a step to expand alloc_context usage out of page_alloc.c
itself. The function try_to_compact_pages() can also much benefit from
the parameter reduction, but it means the struct definition has to be
moved to a shared header.
Patch 1 should IMHO be included even if the rest is deemed not useful
enough. It improves maintainability and also has some code/stack
reduction. Patch 4 is OTOH a tiny optimization.
Overall bloat-o-meter results:
add/remove: 0/0 grow/shrink: 0/4 up/down: 0/-460 (-460)
function old new delta
nr_free_zone_pages 129 115 -14
__alloc_pages_direct_compact 329 256 -73
get_page_from_freelist 2670 2576 -94
__alloc_pages_nodemask 2564 2285 -279
try_to_compact_pages 582 579 -3
Overall stack sizes per ./scripts/checkstack.pl:
old new delta
get_page_from_freelist: 184 184 0
__alloc_pages_nodemask 248 200 -48
__alloc_pages_direct_c 40 - -40
try_to_compact_pages 72 72 0
-88
[1] http://marc.info/?l=linux-mm&m=140142462528257&w=2
This patch (of 4):
prep_new_page() sets almost everything in the struct page of the page
being allocated, except page->pfmemalloc. This is not obvious and has at
least once led to a bug where page->pfmemalloc was forgotten to be set
correctly, see commit 8fb74b9fb2b1 ("mm: compaction: partially revert
capture of suitable high-order page").
This patch moves the pfmemalloc setting to prep_new_page(), which means it
needs to gain alloc_flags parameter. The call to prep_new_page is moved
from buffered_rmqueue() to get_page_from_freelist(), which also leads to
simpler code. An obsolete comment for buffered_rmqueue() is replaced.
In addition to better maintainability there is a small reduction of code
and stack usage for get_page_from_freelist(), which inlines the other
functions involved.
add/remove: 0/0 grow/shrink: 0/1 up/down: 0/-145 (-145)
function old new delta
get_page_from_freelist 2670 2525 -145
Stack usage is reduced from 184 to 168 bytes.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.cz>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-11 16:25:38 -07:00
|
|
|
/*
|
2015-08-21 15:11:51 -06:00
|
|
|
* page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
|
mm: set page->pfmemalloc in prep_new_page()
The possibility of replacing the numerous parameters of alloc_pages*
functions with a single structure has been discussed when Minchan proposed
to expand the x86 kernel stack [1]. This series implements the change,
along with few more cleanups/microoptimizations.
The series is based on next-20150108 and I used gcc 4.8.3 20140627 on
openSUSE 13.2 for compiling. Config includess NUMA and COMPACTION.
The core change is the introduction of a new struct alloc_context, which looks
like this:
struct alloc_context {
struct zonelist *zonelist;
nodemask_t *nodemask;
struct zone *preferred_zone;
int classzone_idx;
int migratetype;
enum zone_type high_zoneidx;
};
All the contents is mostly constant, except that __alloc_pages_slowpath()
changes preferred_zone, classzone_idx and potentially zonelist. But
that's not a problem in case control returns to retry_cpuset: in
__alloc_pages_nodemask(), those will be reset to initial values again
(although it's a bit subtle). On the other hand, gfp_flags and alloc_info
mutate so much that it doesn't make sense to put them into alloc_context.
Still, the result is one parameter instead of up to 7. This is all in
Patch 2.
Patch 3 is a step to expand alloc_context usage out of page_alloc.c
itself. The function try_to_compact_pages() can also much benefit from
the parameter reduction, but it means the struct definition has to be
moved to a shared header.
Patch 1 should IMHO be included even if the rest is deemed not useful
enough. It improves maintainability and also has some code/stack
reduction. Patch 4 is OTOH a tiny optimization.
Overall bloat-o-meter results:
add/remove: 0/0 grow/shrink: 0/4 up/down: 0/-460 (-460)
function old new delta
nr_free_zone_pages 129 115 -14
__alloc_pages_direct_compact 329 256 -73
get_page_from_freelist 2670 2576 -94
__alloc_pages_nodemask 2564 2285 -279
try_to_compact_pages 582 579 -3
Overall stack sizes per ./scripts/checkstack.pl:
old new delta
get_page_from_freelist: 184 184 0
__alloc_pages_nodemask 248 200 -48
__alloc_pages_direct_c 40 - -40
try_to_compact_pages 72 72 0
-88
[1] http://marc.info/?l=linux-mm&m=140142462528257&w=2
This patch (of 4):
prep_new_page() sets almost everything in the struct page of the page
being allocated, except page->pfmemalloc. This is not obvious and has at
least once led to a bug where page->pfmemalloc was forgotten to be set
correctly, see commit 8fb74b9fb2b1 ("mm: compaction: partially revert
capture of suitable high-order page").
This patch moves the pfmemalloc setting to prep_new_page(), which means it
needs to gain alloc_flags parameter. The call to prep_new_page is moved
from buffered_rmqueue() to get_page_from_freelist(), which also leads to
simpler code. An obsolete comment for buffered_rmqueue() is replaced.
In addition to better maintainability there is a small reduction of code
and stack usage for get_page_from_freelist(), which inlines the other
functions involved.
add/remove: 0/0 grow/shrink: 0/1 up/down: 0/-145 (-145)
function old new delta
get_page_from_freelist 2670 2525 -145
Stack usage is reduced from 184 to 168 bytes.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.cz>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-11 16:25:38 -07:00
|
|
|
* allocate the page. The expectation is that the caller is taking
|
|
|
|
* steps that will free more memory. The caller should avoid the page
|
|
|
|
* being used for !PFMEMALLOC purposes.
|
|
|
|
*/
|
2015-08-21 15:11:51 -06:00
|
|
|
if (alloc_flags & ALLOC_NO_WATERMARKS)
|
|
|
|
set_page_pfmemalloc(page);
|
|
|
|
else
|
|
|
|
clear_page_pfmemalloc(page);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
Bias the location of pages freed for min_free_kbytes in the same MAX_ORDER_NR_PAGES blocks
The standard buddy allocator always favours the smallest block of pages.
The effect of this is that the pages free to satisfy min_free_kbytes tends
to be preserved since boot time at the same location of memory ffor a very
long time and as a contiguous block. When an administrator sets the
reserve at 16384 at boot time, it tends to be the same MAX_ORDER blocks
that remain free. This allows the occasional high atomic allocation to
succeed up until the point the blocks are split. In practice, it is
difficult to split these blocks but when they do split, the benefit of
having min_free_kbytes for contiguous blocks disappears. Additionally,
increasing min_free_kbytes once the system has been running for some time
has no guarantee of creating contiguous blocks.
On the other hand, CONFIG_PAGE_GROUP_BY_MOBILITY favours splitting large
blocks when there are no free pages of the appropriate type available. A
side-effect of this is that all blocks in memory tends to be used up and
the contiguous free blocks from boot time are not preserved like in the
vanilla allocator. This can cause a problem if a new caller is unwilling
to reclaim or does not reclaim for long enough.
A failure scenario was found for a wireless network device allocating
order-1 atomic allocations but the allocations were not intense or frequent
enough for a whole block of pages to be preserved for MIGRATE_HIGHALLOC.
This was reproduced on a desktop by booting with mem=256mb, forcing the
driver to allocate at order-1, running a bittorrent client (downloading a
debian ISO) and building a kernel with -j2.
This patch addresses the problem on the desktop machine booted with
mem=256mb. It works by setting aside a reserve of MAX_ORDER_NR_PAGES
blocks, the number of which depends on the value of min_free_kbytes. These
blocks are only fallen back to when there is no other free pages. Then the
smallest possible page is used just like the normal buddy allocator instead
of the largest possible page to preserve contiguous pages The pages in free
lists in the reserve blocks are never taken for another migrate type. The
results is that even if min_free_kbytes is set to a low value, contiguous
blocks will be preserved in the MIGRATE_RESERVE blocks.
This works better than the vanilla allocator because if min_free_kbytes is
increased, a new reserve block will be chosen based on the location of
reclaimable pages and the block will free up as contiguous pages. In the
vanilla allocator, no effort is made to target a block of pages to free as
contiguous pages and min_free_kbytes pages are scattered randomly.
This effect has been observed on the test machine. min_free_kbytes was set
initially low but it was kept as a contiguous free block within
MIGRATE_RESERVE. min_free_kbytes was then set to a higher value and over a
period of time, the free blocks were within the reserve and coalescing.
How long it takes to free up depends on how quickly LRU is rotating.
Amusingly, this means that more activity will free the blocks faster.
This mechanism potentially replaces MIGRATE_HIGHALLOC as it may be more
effective than grouping contiguous free pages together. It all depends on
whether the number of active atomic high allocations exceeds
min_free_kbytes or not. If the number of active allocations exceeds
min_free_kbytes, it's worth it but maybe in that situation, min_free_kbytes
should be set higher. Once there are no more reports of allocation
failures, a patch will be submitted that backs out MIGRATE_HIGHALLOC and
see if the reports stay missing.
Credit to Mariusz Kozlowski for discovering the problem, describing the
failure scenario and testing patches and scenarios.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:58 -06:00
|
|
|
/*
|
|
|
|
* Go through the free lists for the given migratetype and remove
|
|
|
|
* the smallest available page from the freelists
|
|
|
|
*/
|
2009-06-16 16:32:04 -06:00
|
|
|
static inline
|
|
|
|
struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
|
Bias the location of pages freed for min_free_kbytes in the same MAX_ORDER_NR_PAGES blocks
The standard buddy allocator always favours the smallest block of pages.
The effect of this is that the pages free to satisfy min_free_kbytes tends
to be preserved since boot time at the same location of memory ffor a very
long time and as a contiguous block. When an administrator sets the
reserve at 16384 at boot time, it tends to be the same MAX_ORDER blocks
that remain free. This allows the occasional high atomic allocation to
succeed up until the point the blocks are split. In practice, it is
difficult to split these blocks but when they do split, the benefit of
having min_free_kbytes for contiguous blocks disappears. Additionally,
increasing min_free_kbytes once the system has been running for some time
has no guarantee of creating contiguous blocks.
On the other hand, CONFIG_PAGE_GROUP_BY_MOBILITY favours splitting large
blocks when there are no free pages of the appropriate type available. A
side-effect of this is that all blocks in memory tends to be used up and
the contiguous free blocks from boot time are not preserved like in the
vanilla allocator. This can cause a problem if a new caller is unwilling
to reclaim or does not reclaim for long enough.
A failure scenario was found for a wireless network device allocating
order-1 atomic allocations but the allocations were not intense or frequent
enough for a whole block of pages to be preserved for MIGRATE_HIGHALLOC.
This was reproduced on a desktop by booting with mem=256mb, forcing the
driver to allocate at order-1, running a bittorrent client (downloading a
debian ISO) and building a kernel with -j2.
This patch addresses the problem on the desktop machine booted with
mem=256mb. It works by setting aside a reserve of MAX_ORDER_NR_PAGES
blocks, the number of which depends on the value of min_free_kbytes. These
blocks are only fallen back to when there is no other free pages. Then the
smallest possible page is used just like the normal buddy allocator instead
of the largest possible page to preserve contiguous pages The pages in free
lists in the reserve blocks are never taken for another migrate type. The
results is that even if min_free_kbytes is set to a low value, contiguous
blocks will be preserved in the MIGRATE_RESERVE blocks.
This works better than the vanilla allocator because if min_free_kbytes is
increased, a new reserve block will be chosen based on the location of
reclaimable pages and the block will free up as contiguous pages. In the
vanilla allocator, no effort is made to target a block of pages to free as
contiguous pages and min_free_kbytes pages are scattered randomly.
This effect has been observed on the test machine. min_free_kbytes was set
initially low but it was kept as a contiguous free block within
MIGRATE_RESERVE. min_free_kbytes was then set to a higher value and over a
period of time, the free blocks were within the reserve and coalescing.
How long it takes to free up depends on how quickly LRU is rotating.
Amusingly, this means that more activity will free the blocks faster.
This mechanism potentially replaces MIGRATE_HIGHALLOC as it may be more
effective than grouping contiguous free pages together. It all depends on
whether the number of active atomic high allocations exceeds
min_free_kbytes or not. If the number of active allocations exceeds
min_free_kbytes, it's worth it but maybe in that situation, min_free_kbytes
should be set higher. Once there are no more reports of allocation
failures, a patch will be submitted that backs out MIGRATE_HIGHALLOC and
see if the reports stay missing.
Credit to Mariusz Kozlowski for discovering the problem, describing the
failure scenario and testing patches and scenarios.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:58 -06:00
|
|
|
int migratetype)
|
|
|
|
{
|
|
|
|
unsigned int current_order;
|
2013-09-11 15:20:34 -06:00
|
|
|
struct free_area *area;
|
Bias the location of pages freed for min_free_kbytes in the same MAX_ORDER_NR_PAGES blocks
The standard buddy allocator always favours the smallest block of pages.
The effect of this is that the pages free to satisfy min_free_kbytes tends
to be preserved since boot time at the same location of memory ffor a very
long time and as a contiguous block. When an administrator sets the
reserve at 16384 at boot time, it tends to be the same MAX_ORDER blocks
that remain free. This allows the occasional high atomic allocation to
succeed up until the point the blocks are split. In practice, it is
difficult to split these blocks but when they do split, the benefit of
having min_free_kbytes for contiguous blocks disappears. Additionally,
increasing min_free_kbytes once the system has been running for some time
has no guarantee of creating contiguous blocks.
On the other hand, CONFIG_PAGE_GROUP_BY_MOBILITY favours splitting large
blocks when there are no free pages of the appropriate type available. A
side-effect of this is that all blocks in memory tends to be used up and
the contiguous free blocks from boot time are not preserved like in the
vanilla allocator. This can cause a problem if a new caller is unwilling
to reclaim or does not reclaim for long enough.
A failure scenario was found for a wireless network device allocating
order-1 atomic allocations but the allocations were not intense or frequent
enough for a whole block of pages to be preserved for MIGRATE_HIGHALLOC.
This was reproduced on a desktop by booting with mem=256mb, forcing the
driver to allocate at order-1, running a bittorrent client (downloading a
debian ISO) and building a kernel with -j2.
This patch addresses the problem on the desktop machine booted with
mem=256mb. It works by setting aside a reserve of MAX_ORDER_NR_PAGES
blocks, the number of which depends on the value of min_free_kbytes. These
blocks are only fallen back to when there is no other free pages. Then the
smallest possible page is used just like the normal buddy allocator instead
of the largest possible page to preserve contiguous pages The pages in free
lists in the reserve blocks are never taken for another migrate type. The
results is that even if min_free_kbytes is set to a low value, contiguous
blocks will be preserved in the MIGRATE_RESERVE blocks.
This works better than the vanilla allocator because if min_free_kbytes is
increased, a new reserve block will be chosen based on the location of
reclaimable pages and the block will free up as contiguous pages. In the
vanilla allocator, no effort is made to target a block of pages to free as
contiguous pages and min_free_kbytes pages are scattered randomly.
This effect has been observed on the test machine. min_free_kbytes was set
initially low but it was kept as a contiguous free block within
MIGRATE_RESERVE. min_free_kbytes was then set to a higher value and over a
period of time, the free blocks were within the reserve and coalescing.
How long it takes to free up depends on how quickly LRU is rotating.
Amusingly, this means that more activity will free the blocks faster.
This mechanism potentially replaces MIGRATE_HIGHALLOC as it may be more
effective than grouping contiguous free pages together. It all depends on
whether the number of active atomic high allocations exceeds
min_free_kbytes or not. If the number of active allocations exceeds
min_free_kbytes, it's worth it but maybe in that situation, min_free_kbytes
should be set higher. Once there are no more reports of allocation
failures, a patch will be submitted that backs out MIGRATE_HIGHALLOC and
see if the reports stay missing.
Credit to Mariusz Kozlowski for discovering the problem, describing the
failure scenario and testing patches and scenarios.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:58 -06:00
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
/* Find a page of the appropriate size in the preferred list */
|
|
|
|
for (current_order = order; current_order < MAX_ORDER; ++current_order) {
|
|
|
|
area = &(zone->free_area[current_order]);
|
2016-01-14 16:20:30 -07:00
|
|
|
page = list_first_entry_or_null(&area->free_list[migratetype],
|
Bias the location of pages freed for min_free_kbytes in the same MAX_ORDER_NR_PAGES blocks
The standard buddy allocator always favours the smallest block of pages.
The effect of this is that the pages free to satisfy min_free_kbytes tends
to be preserved since boot time at the same location of memory ffor a very
long time and as a contiguous block. When an administrator sets the
reserve at 16384 at boot time, it tends to be the same MAX_ORDER blocks
that remain free. This allows the occasional high atomic allocation to
succeed up until the point the blocks are split. In practice, it is
difficult to split these blocks but when they do split, the benefit of
having min_free_kbytes for contiguous blocks disappears. Additionally,
increasing min_free_kbytes once the system has been running for some time
has no guarantee of creating contiguous blocks.
On the other hand, CONFIG_PAGE_GROUP_BY_MOBILITY favours splitting large
blocks when there are no free pages of the appropriate type available. A
side-effect of this is that all blocks in memory tends to be used up and
the contiguous free blocks from boot time are not preserved like in the
vanilla allocator. This can cause a problem if a new caller is unwilling
to reclaim or does not reclaim for long enough.
A failure scenario was found for a wireless network device allocating
order-1 atomic allocations but the allocations were not intense or frequent
enough for a whole block of pages to be preserved for MIGRATE_HIGHALLOC.
This was reproduced on a desktop by booting with mem=256mb, forcing the
driver to allocate at order-1, running a bittorrent client (downloading a
debian ISO) and building a kernel with -j2.
This patch addresses the problem on the desktop machine booted with
mem=256mb. It works by setting aside a reserve of MAX_ORDER_NR_PAGES
blocks, the number of which depends on the value of min_free_kbytes. These
blocks are only fallen back to when there is no other free pages. Then the
smallest possible page is used just like the normal buddy allocator instead
of the largest possible page to preserve contiguous pages The pages in free
lists in the reserve blocks are never taken for another migrate type. The
results is that even if min_free_kbytes is set to a low value, contiguous
blocks will be preserved in the MIGRATE_RESERVE blocks.
This works better than the vanilla allocator because if min_free_kbytes is
increased, a new reserve block will be chosen based on the location of
reclaimable pages and the block will free up as contiguous pages. In the
vanilla allocator, no effort is made to target a block of pages to free as
contiguous pages and min_free_kbytes pages are scattered randomly.
This effect has been observed on the test machine. min_free_kbytes was set
initially low but it was kept as a contiguous free block within
MIGRATE_RESERVE. min_free_kbytes was then set to a higher value and over a
period of time, the free blocks were within the reserve and coalescing.
How long it takes to free up depends on how quickly LRU is rotating.
Amusingly, this means that more activity will free the blocks faster.
This mechanism potentially replaces MIGRATE_HIGHALLOC as it may be more
effective than grouping contiguous free pages together. It all depends on
whether the number of active atomic high allocations exceeds
min_free_kbytes or not. If the number of active allocations exceeds
min_free_kbytes, it's worth it but maybe in that situation, min_free_kbytes
should be set higher. Once there are no more reports of allocation
failures, a patch will be submitted that backs out MIGRATE_HIGHALLOC and
see if the reports stay missing.
Credit to Mariusz Kozlowski for discovering the problem, describing the
failure scenario and testing patches and scenarios.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:58 -06:00
|
|
|
struct page, lru);
|
2016-01-14 16:20:30 -07:00
|
|
|
if (!page)
|
|
|
|
continue;
|
Bias the location of pages freed for min_free_kbytes in the same MAX_ORDER_NR_PAGES blocks
The standard buddy allocator always favours the smallest block of pages.
The effect of this is that the pages free to satisfy min_free_kbytes tends
to be preserved since boot time at the same location of memory ffor a very
long time and as a contiguous block. When an administrator sets the
reserve at 16384 at boot time, it tends to be the same MAX_ORDER blocks
that remain free. This allows the occasional high atomic allocation to
succeed up until the point the blocks are split. In practice, it is
difficult to split these blocks but when they do split, the benefit of
having min_free_kbytes for contiguous blocks disappears. Additionally,
increasing min_free_kbytes once the system has been running for some time
has no guarantee of creating contiguous blocks.
On the other hand, CONFIG_PAGE_GROUP_BY_MOBILITY favours splitting large
blocks when there are no free pages of the appropriate type available. A
side-effect of this is that all blocks in memory tends to be used up and
the contiguous free blocks from boot time are not preserved like in the
vanilla allocator. This can cause a problem if a new caller is unwilling
to reclaim or does not reclaim for long enough.
A failure scenario was found for a wireless network device allocating
order-1 atomic allocations but the allocations were not intense or frequent
enough for a whole block of pages to be preserved for MIGRATE_HIGHALLOC.
This was reproduced on a desktop by booting with mem=256mb, forcing the
driver to allocate at order-1, running a bittorrent client (downloading a
debian ISO) and building a kernel with -j2.
This patch addresses the problem on the desktop machine booted with
mem=256mb. It works by setting aside a reserve of MAX_ORDER_NR_PAGES
blocks, the number of which depends on the value of min_free_kbytes. These
blocks are only fallen back to when there is no other free pages. Then the
smallest possible page is used just like the normal buddy allocator instead
of the largest possible page to preserve contiguous pages The pages in free
lists in the reserve blocks are never taken for another migrate type. The
results is that even if min_free_kbytes is set to a low value, contiguous
blocks will be preserved in the MIGRATE_RESERVE blocks.
This works better than the vanilla allocator because if min_free_kbytes is
increased, a new reserve block will be chosen based on the location of
reclaimable pages and the block will free up as contiguous pages. In the
vanilla allocator, no effort is made to target a block of pages to free as
contiguous pages and min_free_kbytes pages are scattered randomly.
This effect has been observed on the test machine. min_free_kbytes was set
initially low but it was kept as a contiguous free block within
MIGRATE_RESERVE. min_free_kbytes was then set to a higher value and over a
period of time, the free blocks were within the reserve and coalescing.
How long it takes to free up depends on how quickly LRU is rotating.
Amusingly, this means that more activity will free the blocks faster.
This mechanism potentially replaces MIGRATE_HIGHALLOC as it may be more
effective than grouping contiguous free pages together. It all depends on
whether the number of active atomic high allocations exceeds
min_free_kbytes or not. If the number of active allocations exceeds
min_free_kbytes, it's worth it but maybe in that situation, min_free_kbytes
should be set higher. Once there are no more reports of allocation
failures, a patch will be submitted that backs out MIGRATE_HIGHALLOC and
see if the reports stay missing.
Credit to Mariusz Kozlowski for discovering the problem, describing the
failure scenario and testing patches and scenarios.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:58 -06:00
|
|
|
list_del(&page->lru);
|
|
|
|
rmv_page_order(page);
|
|
|
|
area->nr_free--;
|
|
|
|
expand(zone, page, order, current_order, area, migratetype);
|
2015-09-08 16:01:25 -06:00
|
|
|
set_pcppage_migratetype(page, migratetype);
|
Bias the location of pages freed for min_free_kbytes in the same MAX_ORDER_NR_PAGES blocks
The standard buddy allocator always favours the smallest block of pages.
The effect of this is that the pages free to satisfy min_free_kbytes tends
to be preserved since boot time at the same location of memory ffor a very
long time and as a contiguous block. When an administrator sets the
reserve at 16384 at boot time, it tends to be the same MAX_ORDER blocks
that remain free. This allows the occasional high atomic allocation to
succeed up until the point the blocks are split. In practice, it is
difficult to split these blocks but when they do split, the benefit of
having min_free_kbytes for contiguous blocks disappears. Additionally,
increasing min_free_kbytes once the system has been running for some time
has no guarantee of creating contiguous blocks.
On the other hand, CONFIG_PAGE_GROUP_BY_MOBILITY favours splitting large
blocks when there are no free pages of the appropriate type available. A
side-effect of this is that all blocks in memory tends to be used up and
the contiguous free blocks from boot time are not preserved like in the
vanilla allocator. This can cause a problem if a new caller is unwilling
to reclaim or does not reclaim for long enough.
A failure scenario was found for a wireless network device allocating
order-1 atomic allocations but the allocations were not intense or frequent
enough for a whole block of pages to be preserved for MIGRATE_HIGHALLOC.
This was reproduced on a desktop by booting with mem=256mb, forcing the
driver to allocate at order-1, running a bittorrent client (downloading a
debian ISO) and building a kernel with -j2.
This patch addresses the problem on the desktop machine booted with
mem=256mb. It works by setting aside a reserve of MAX_ORDER_NR_PAGES
blocks, the number of which depends on the value of min_free_kbytes. These
blocks are only fallen back to when there is no other free pages. Then the
smallest possible page is used just like the normal buddy allocator instead
of the largest possible page to preserve contiguous pages The pages in free
lists in the reserve blocks are never taken for another migrate type. The
results is that even if min_free_kbytes is set to a low value, contiguous
blocks will be preserved in the MIGRATE_RESERVE blocks.
This works better than the vanilla allocator because if min_free_kbytes is
increased, a new reserve block will be chosen based on the location of
reclaimable pages and the block will free up as contiguous pages. In the
vanilla allocator, no effort is made to target a block of pages to free as
contiguous pages and min_free_kbytes pages are scattered randomly.
This effect has been observed on the test machine. min_free_kbytes was set
initially low but it was kept as a contiguous free block within
MIGRATE_RESERVE. min_free_kbytes was then set to a higher value and over a
period of time, the free blocks were within the reserve and coalescing.
How long it takes to free up depends on how quickly LRU is rotating.
Amusingly, this means that more activity will free the blocks faster.
This mechanism potentially replaces MIGRATE_HIGHALLOC as it may be more
effective than grouping contiguous free pages together. It all depends on
whether the number of active atomic high allocations exceeds
min_free_kbytes or not. If the number of active allocations exceeds
min_free_kbytes, it's worth it but maybe in that situation, min_free_kbytes
should be set higher. Once there are no more reports of allocation
failures, a patch will be submitted that backs out MIGRATE_HIGHALLOC and
see if the reports stay missing.
Credit to Mariusz Kozlowski for discovering the problem, describing the
failure scenario and testing patches and scenarios.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:58 -06:00
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2007-10-16 02:25:48 -06:00
|
|
|
/*
|
|
|
|
* This array describes the order lists are fallen back to when
|
|
|
|
* the free lists for the desirable migrate type are depleted
|
|
|
|
*/
|
2011-12-29 05:09:50 -07:00
|
|
|
static int fallbacks[MIGRATE_TYPES][4] = {
|
2015-11-06 17:28:34 -07:00
|
|
|
[MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
|
|
|
|
[MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
|
|
|
|
[MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
|
2011-12-29 05:09:50 -07:00
|
|
|
#ifdef CONFIG_CMA
|
2015-11-06 17:28:34 -07:00
|
|
|
[MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */
|
2011-12-29 05:09:50 -07:00
|
|
|
#endif
|
2013-02-22 17:33:58 -07:00
|
|
|
#ifdef CONFIG_MEMORY_ISOLATION
|
2015-11-06 17:28:34 -07:00
|
|
|
[MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */
|
2013-02-22 17:33:58 -07:00
|
|
|
#endif
|
2007-10-16 02:25:48 -06:00
|
|
|
};
|
|
|
|
|
2015-04-14 16:45:15 -06:00
|
|
|
#ifdef CONFIG_CMA
|
|
|
|
static struct page *__rmqueue_cma_fallback(struct zone *zone,
|
|
|
|
unsigned int order)
|
|
|
|
{
|
|
|
|
return __rmqueue_smallest(zone, order, MIGRATE_CMA);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
|
|
|
|
unsigned int order) { return NULL; }
|
|
|
|
#endif
|
|
|
|
|
2007-10-16 02:25:51 -06:00
|
|
|
/*
|
|
|
|
* Move the free pages in a range to the free lists of the requested type.
|
2007-10-16 02:26:01 -06:00
|
|
|
* Note that start_page and end_pages are not aligned on a pageblock
|
2007-10-16 02:25:51 -06:00
|
|
|
* boundary. If alignment is required, use move_freepages_block()
|
|
|
|
*/
|
2012-10-08 17:32:16 -06:00
|
|
|
int move_freepages(struct zone *zone,
|
2008-07-23 22:28:12 -06:00
|
|
|
struct page *start_page, struct page *end_page,
|
|
|
|
int migratetype)
|
2007-10-16 02:25:51 -06:00
|
|
|
{
|
|
|
|
struct page *page;
|
2015-11-06 17:29:57 -07:00
|
|
|
unsigned int order;
|
2007-10-16 02:26:00 -06:00
|
|
|
int pages_moved = 0;
|
2007-10-16 02:25:51 -06:00
|
|
|
|
|
|
|
#ifndef CONFIG_HOLES_IN_ZONE
|
|
|
|
/*
|
|
|
|
* page_zone is not safe to call in this context when
|
|
|
|
* CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
|
|
|
|
* anyway as we check zone boundaries in move_freepages_block().
|
|
|
|
* Remove at a later date when no bug reports exist related to
|
2007-10-16 02:25:58 -06:00
|
|
|
* grouping pages by mobility
|
2007-10-16 02:25:51 -06:00
|
|
|
*/
|
2014-10-09 16:28:28 -06:00
|
|
|
VM_BUG_ON(page_zone(start_page) != page_zone(end_page));
|
2007-10-16 02:25:51 -06:00
|
|
|
#endif
|
|
|
|
|
|
|
|
for (page = start_page; page <= end_page;) {
|
2008-09-02 15:35:38 -06:00
|
|
|
/* Make sure we are not inadvertently changing nodes */
|
2014-01-23 16:52:54 -07:00
|
|
|
VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
|
2008-09-02 15:35:38 -06:00
|
|
|
|
2007-10-16 02:25:51 -06:00
|
|
|
if (!pfn_valid_within(page_to_pfn(page))) {
|
|
|
|
page++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!PageBuddy(page)) {
|
|
|
|
page++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
order = page_order(page);
|
2011-03-22 17:33:41 -06:00
|
|
|
list_move(&page->lru,
|
|
|
|
&zone->free_area[order].free_list[migratetype]);
|
2007-10-16 02:25:51 -06:00
|
|
|
page += 1 << order;
|
2007-10-16 02:26:00 -06:00
|
|
|
pages_moved += 1 << order;
|
2007-10-16 02:25:51 -06:00
|
|
|
}
|
|
|
|
|
2007-10-16 02:26:00 -06:00
|
|
|
return pages_moved;
|
2007-10-16 02:25:51 -06:00
|
|
|
}
|
|
|
|
|
2012-07-31 17:43:50 -06:00
|
|
|
int move_freepages_block(struct zone *zone, struct page *page,
|
2012-06-03 21:05:57 -06:00
|
|
|
int migratetype)
|
2007-10-16 02:25:51 -06:00
|
|
|
{
|
|
|
|
unsigned long start_pfn, end_pfn;
|
|
|
|
struct page *start_page, *end_page;
|
|
|
|
|
|
|
|
start_pfn = page_to_pfn(page);
|
2007-10-16 02:26:01 -06:00
|
|
|
start_pfn = start_pfn & ~(pageblock_nr_pages-1);
|
2007-10-16 02:25:51 -06:00
|
|
|
start_page = pfn_to_page(start_pfn);
|
2007-10-16 02:26:01 -06:00
|
|
|
end_page = start_page + pageblock_nr_pages - 1;
|
|
|
|
end_pfn = start_pfn + pageblock_nr_pages - 1;
|
2007-10-16 02:25:51 -06:00
|
|
|
|
|
|
|
/* Do not cross zone boundaries */
|
2013-02-22 17:35:23 -07:00
|
|
|
if (!zone_spans_pfn(zone, start_pfn))
|
2007-10-16 02:25:51 -06:00
|
|
|
start_page = page;
|
2013-02-22 17:35:23 -07:00
|
|
|
if (!zone_spans_pfn(zone, end_pfn))
|
2007-10-16 02:25:51 -06:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
return move_freepages(zone, start_page, end_page, migratetype);
|
|
|
|
}
|
|
|
|
|
2009-09-21 18:02:31 -06:00
|
|
|
static void change_pageblock_range(struct page *pageblock_page,
|
|
|
|
int start_order, int migratetype)
|
|
|
|
{
|
|
|
|
int nr_pageblocks = 1 << (start_order - pageblock_order);
|
|
|
|
|
|
|
|
while (nr_pageblocks--) {
|
|
|
|
set_pageblock_migratetype(pageblock_page, migratetype);
|
|
|
|
pageblock_page += pageblock_nr_pages;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
mm/page_allo.c: restructure free-page stealing code and fix a bug
The free-page stealing code in __rmqueue_fallback() is somewhat hard to
follow, and has an incredible amount of subtlety hidden inside!
First off, there is a minor bug in the reporting of change-of-ownership of
pageblocks. Under some conditions, we try to move upto
'pageblock_nr_pages' no. of pages to the preferred allocation list. But
we change the ownership of that pageblock to the preferred type only if we
manage to successfully move atleast half of that pageblock (or if
page_group_by_mobility_disabled is set).
However, the current code ignores the latter part and sets the
'migratetype' variable to the preferred type, irrespective of whether we
actually changed the pageblock migratetype of that block or not. So, the
page_alloc_extfrag tracepoint can end up printing incorrect info (i.e.,
'change_ownership' might be shown as 1 when it must have been 0).
So fixing this involves moving the update of the 'migratetype' variable to
the right place. But looking closer, we observe that the 'migratetype'
variable is used subsequently for checks such as "is_migrate_cma()".
Obviously the intent there is to check if the *fallback* type is
MIGRATE_CMA, but since we already set the 'migratetype' variable to
start_migratetype, we end up checking if the *preferred* type is
MIGRATE_CMA!!
To make things more interesting, this actually doesn't cause a bug in
practice, because we never change *anything* if the fallback type is CMA.
So, restructure the code in such a way that it is trivial to understand
what is going on, and also fix the above mentioned bug. And while at it,
also add a comment explaining the subtlety behind the migratetype used in
the call to expand().
[akpm@linux-foundation.org: remove unneeded `inline', small coding-style fix]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Cody P Schafer <cody@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-09-11 15:20:35 -06:00
|
|
|
/*
|
mm: more aggressive page stealing for UNMOVABLE allocations
When allocation falls back to stealing free pages of another migratetype,
it can decide to steal extra pages, or even the whole pageblock in order
to reduce fragmentation, which could happen if further allocation
fallbacks pick a different pageblock. In try_to_steal_freepages(), one of
the situations where extra pages are stolen happens when we are trying to
allocate a MIGRATE_RECLAIMABLE page.
However, MIGRATE_UNMOVABLE allocations are not treated the same way,
although spreading such allocation over multiple fallback pageblocks is
arguably even worse than it is for RECLAIMABLE allocations. To minimize
fragmentation, we should minimize the number of such fallbacks, and thus
steal as much as is possible from each fallback pageblock.
Note that in theory this might put more pressure on movable pageblocks and
cause movable allocations to steal back from unmovable pageblocks.
However, movable allocations are not as aggressive with stealing, and do
not cause permanent fragmentation, so the tradeoff is reasonable, and
evaluation seems to support the change.
This patch thus adds a check for MIGRATE_UNMOVABLE to the decision to
steal extra free pages. When evaluating with stress-highalloc from
mmtests, this has reduced the number of MIGRATE_UNMOVABLE fallbacks to
roughly 1/6. The number of these fallbacks stealing from MIGRATE_MOVABLE
block is reduced to 1/3. There was no observation of growing number of
unmovable pageblocks over time, and also not of increased movable
allocation fallbacks.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-11 16:28:21 -07:00
|
|
|
* When we are falling back to another migratetype during allocation, try to
|
|
|
|
* steal extra free pages from the same pageblocks to satisfy further
|
|
|
|
* allocations, instead of polluting multiple pageblocks.
|
|
|
|
*
|
|
|
|
* If we are stealing a relatively large buddy page, it is likely there will
|
|
|
|
* be more free pages in the pageblock, so try to steal them all. For
|
|
|
|
* reclaimable and unmovable allocations, we steal regardless of page size,
|
|
|
|
* as fragmentation caused by those allocations polluting movable pageblocks
|
|
|
|
* is worse than movable allocations stealing from unmovable and reclaimable
|
|
|
|
* pageblocks.
|
mm/page_allo.c: restructure free-page stealing code and fix a bug
The free-page stealing code in __rmqueue_fallback() is somewhat hard to
follow, and has an incredible amount of subtlety hidden inside!
First off, there is a minor bug in the reporting of change-of-ownership of
pageblocks. Under some conditions, we try to move upto
'pageblock_nr_pages' no. of pages to the preferred allocation list. But
we change the ownership of that pageblock to the preferred type only if we
manage to successfully move atleast half of that pageblock (or if
page_group_by_mobility_disabled is set).
However, the current code ignores the latter part and sets the
'migratetype' variable to the preferred type, irrespective of whether we
actually changed the pageblock migratetype of that block or not. So, the
page_alloc_extfrag tracepoint can end up printing incorrect info (i.e.,
'change_ownership' might be shown as 1 when it must have been 0).
So fixing this involves moving the update of the 'migratetype' variable to
the right place. But looking closer, we observe that the 'migratetype'
variable is used subsequently for checks such as "is_migrate_cma()".
Obviously the intent there is to check if the *fallback* type is
MIGRATE_CMA, but since we already set the 'migratetype' variable to
start_migratetype, we end up checking if the *preferred* type is
MIGRATE_CMA!!
To make things more interesting, this actually doesn't cause a bug in
practice, because we never change *anything* if the fallback type is CMA.
So, restructure the code in such a way that it is trivial to understand
what is going on, and also fix the above mentioned bug. And while at it,
also add a comment explaining the subtlety behind the migratetype used in
the call to expand().
[akpm@linux-foundation.org: remove unneeded `inline', small coding-style fix]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Cody P Schafer <cody@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-09-11 15:20:35 -06:00
|
|
|
*/
|
2015-04-14 16:45:18 -06:00
|
|
|
static bool can_steal_fallback(unsigned int order, int start_mt)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Leaving this order check is intended, although there is
|
|
|
|
* relaxed order check in next check. The reason is that
|
|
|
|
* we can actually steal whole pageblock if this condition met,
|
|
|
|
* but, below check doesn't guarantee it and that is just heuristic
|
|
|
|
* so could be changed anytime.
|
|
|
|
*/
|
|
|
|
if (order >= pageblock_order)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (order >= pageblock_order / 2 ||
|
|
|
|
start_mt == MIGRATE_RECLAIMABLE ||
|
|
|
|
start_mt == MIGRATE_UNMOVABLE ||
|
|
|
|
page_group_by_mobility_disabled)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This function implements actual steal behaviour. If order is large enough,
|
|
|
|
* we can steal whole pageblock. If not, we first move freepages in this
|
|
|
|
* pageblock and check whether half of pages are moved or not. If half of
|
|
|
|
* pages are moved, we can change migratetype of pageblock and permanently
|
|
|
|
* use it's pages as requested migratetype in the future.
|
|
|
|
*/
|
|
|
|
static void steal_suitable_fallback(struct zone *zone, struct page *page,
|
|
|
|
int start_type)
|
mm/page_allo.c: restructure free-page stealing code and fix a bug
The free-page stealing code in __rmqueue_fallback() is somewhat hard to
follow, and has an incredible amount of subtlety hidden inside!
First off, there is a minor bug in the reporting of change-of-ownership of
pageblocks. Under some conditions, we try to move upto
'pageblock_nr_pages' no. of pages to the preferred allocation list. But
we change the ownership of that pageblock to the preferred type only if we
manage to successfully move atleast half of that pageblock (or if
page_group_by_mobility_disabled is set).
However, the current code ignores the latter part and sets the
'migratetype' variable to the preferred type, irrespective of whether we
actually changed the pageblock migratetype of that block or not. So, the
page_alloc_extfrag tracepoint can end up printing incorrect info (i.e.,
'change_ownership' might be shown as 1 when it must have been 0).
So fixing this involves moving the update of the 'migratetype' variable to
the right place. But looking closer, we observe that the 'migratetype'
variable is used subsequently for checks such as "is_migrate_cma()".
Obviously the intent there is to check if the *fallback* type is
MIGRATE_CMA, but since we already set the 'migratetype' variable to
start_migratetype, we end up checking if the *preferred* type is
MIGRATE_CMA!!
To make things more interesting, this actually doesn't cause a bug in
practice, because we never change *anything* if the fallback type is CMA.
So, restructure the code in such a way that it is trivial to understand
what is going on, and also fix the above mentioned bug. And while at it,
also add a comment explaining the subtlety behind the migratetype used in
the call to expand().
[akpm@linux-foundation.org: remove unneeded `inline', small coding-style fix]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Cody P Schafer <cody@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-09-11 15:20:35 -06:00
|
|
|
{
|
2015-11-06 17:29:57 -07:00
|
|
|
unsigned int current_order = page_order(page);
|
2015-04-14 16:45:18 -06:00
|
|
|
int pages;
|
mm/page_allo.c: restructure free-page stealing code and fix a bug
The free-page stealing code in __rmqueue_fallback() is somewhat hard to
follow, and has an incredible amount of subtlety hidden inside!
First off, there is a minor bug in the reporting of change-of-ownership of
pageblocks. Under some conditions, we try to move upto
'pageblock_nr_pages' no. of pages to the preferred allocation list. But
we change the ownership of that pageblock to the preferred type only if we
manage to successfully move atleast half of that pageblock (or if
page_group_by_mobility_disabled is set).
However, the current code ignores the latter part and sets the
'migratetype' variable to the preferred type, irrespective of whether we
actually changed the pageblock migratetype of that block or not. So, the
page_alloc_extfrag tracepoint can end up printing incorrect info (i.e.,
'change_ownership' might be shown as 1 when it must have been 0).
So fixing this involves moving the update of the 'migratetype' variable to
the right place. But looking closer, we observe that the 'migratetype'
variable is used subsequently for checks such as "is_migrate_cma()".
Obviously the intent there is to check if the *fallback* type is
MIGRATE_CMA, but since we already set the 'migratetype' variable to
start_migratetype, we end up checking if the *preferred* type is
MIGRATE_CMA!!
To make things more interesting, this actually doesn't cause a bug in
practice, because we never change *anything* if the fallback type is CMA.
So, restructure the code in such a way that it is trivial to understand
what is going on, and also fix the above mentioned bug. And while at it,
also add a comment explaining the subtlety behind the migratetype used in
the call to expand().
[akpm@linux-foundation.org: remove unneeded `inline', small coding-style fix]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Cody P Schafer <cody@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-09-11 15:20:35 -06:00
|
|
|
|
|
|
|
/* Take ownership for orders >= pageblock_order */
|
|
|
|
if (current_order >= pageblock_order) {
|
|
|
|
change_pageblock_range(page, current_order, start_type);
|
mm: always steal split buddies in fallback allocations
When allocation falls back to another migratetype, it will steal a page
with highest available order, and (depending on this order and desired
migratetype), it might also steal the rest of free pages from the same
pageblock.
Given the preference of highest available order, it is likely that it will
be higher than the desired order, and result in the stolen buddy page
being split. The remaining pages after split are currently stolen only
when the rest of the free pages are stolen. This can however lead to
situations where for MOVABLE allocations we split e.g. order-4 fallback
UNMOVABLE page, but steal only order-0 page. Then on the next MOVABLE
allocation (which may be batched to fill the pcplists) we split another
order-3 or higher page, etc. By stealing all pages that we have split, we
can avoid further stealing.
This patch therefore adjusts the page stealing so that buddy pages created
by split are always stolen. This has effect only on MOVABLE allocations,
as RECLAIMABLE and UNMOVABLE allocations already always do that in
addition to stealing the rest of free pages from the pageblock. The
change also allows to simplify try_to_steal_freepages() and factor out CMA
handling.
According to Mel, it has been intended since the beginning that buddy
pages after split would be stolen always, but it doesn't seem like it was
ever the case until commit 47118af076f6 ("mm: mmzone: MIGRATE_CMA
migration type added"). The commit has unintentionally introduced this
behavior, but was reverted by commit 0cbef29a7821 ("mm:
__rmqueue_fallback() should respect pageblock type"). Neither included
evaluation.
My evaluation with stress-highalloc from mmtests shows about 2.5x
reduction of page stealing events for MOVABLE allocations, without
affecting the page stealing events for other allocation migratetypes.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Acked-by: Minchan Kim <minchan@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-11 16:28:18 -07:00
|
|
|
return;
|
mm/page_allo.c: restructure free-page stealing code and fix a bug
The free-page stealing code in __rmqueue_fallback() is somewhat hard to
follow, and has an incredible amount of subtlety hidden inside!
First off, there is a minor bug in the reporting of change-of-ownership of
pageblocks. Under some conditions, we try to move upto
'pageblock_nr_pages' no. of pages to the preferred allocation list. But
we change the ownership of that pageblock to the preferred type only if we
manage to successfully move atleast half of that pageblock (or if
page_group_by_mobility_disabled is set).
However, the current code ignores the latter part and sets the
'migratetype' variable to the preferred type, irrespective of whether we
actually changed the pageblock migratetype of that block or not. So, the
page_alloc_extfrag tracepoint can end up printing incorrect info (i.e.,
'change_ownership' might be shown as 1 when it must have been 0).
So fixing this involves moving the update of the 'migratetype' variable to
the right place. But looking closer, we observe that the 'migratetype'
variable is used subsequently for checks such as "is_migrate_cma()".
Obviously the intent there is to check if the *fallback* type is
MIGRATE_CMA, but since we already set the 'migratetype' variable to
start_migratetype, we end up checking if the *preferred* type is
MIGRATE_CMA!!
To make things more interesting, this actually doesn't cause a bug in
practice, because we never change *anything* if the fallback type is CMA.
So, restructure the code in such a way that it is trivial to understand
what is going on, and also fix the above mentioned bug. And while at it,
also add a comment explaining the subtlety behind the migratetype used in
the call to expand().
[akpm@linux-foundation.org: remove unneeded `inline', small coding-style fix]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Cody P Schafer <cody@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-09-11 15:20:35 -06:00
|
|
|
}
|
|
|
|
|
2015-04-14 16:45:18 -06:00
|
|
|
pages = move_freepages_block(zone, page, start_type);
|
mm/page_allo.c: restructure free-page stealing code and fix a bug
The free-page stealing code in __rmqueue_fallback() is somewhat hard to
follow, and has an incredible amount of subtlety hidden inside!
First off, there is a minor bug in the reporting of change-of-ownership of
pageblocks. Under some conditions, we try to move upto
'pageblock_nr_pages' no. of pages to the preferred allocation list. But
we change the ownership of that pageblock to the preferred type only if we
manage to successfully move atleast half of that pageblock (or if
page_group_by_mobility_disabled is set).
However, the current code ignores the latter part and sets the
'migratetype' variable to the preferred type, irrespective of whether we
actually changed the pageblock migratetype of that block or not. So, the
page_alloc_extfrag tracepoint can end up printing incorrect info (i.e.,
'change_ownership' might be shown as 1 when it must have been 0).
So fixing this involves moving the update of the 'migratetype' variable to
the right place. But looking closer, we observe that the 'migratetype'
variable is used subsequently for checks such as "is_migrate_cma()".
Obviously the intent there is to check if the *fallback* type is
MIGRATE_CMA, but since we already set the 'migratetype' variable to
start_migratetype, we end up checking if the *preferred* type is
MIGRATE_CMA!!
To make things more interesting, this actually doesn't cause a bug in
practice, because we never change *anything* if the fallback type is CMA.
So, restructure the code in such a way that it is trivial to understand
what is going on, and also fix the above mentioned bug. And while at it,
also add a comment explaining the subtlety behind the migratetype used in
the call to expand().
[akpm@linux-foundation.org: remove unneeded `inline', small coding-style fix]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Cody P Schafer <cody@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-09-11 15:20:35 -06:00
|
|
|
|
2015-04-14 16:45:18 -06:00
|
|
|
/* Claim the whole block if over half of it is free */
|
|
|
|
if (pages >= (1 << (pageblock_order-1)) ||
|
|
|
|
page_group_by_mobility_disabled)
|
|
|
|
set_pageblock_migratetype(page, start_type);
|
|
|
|
}
|
|
|
|
|
mm/compaction: enhance compaction finish condition
Compaction has anti fragmentation algorithm. It is that freepage should
be more than pageblock order to finish the compaction if we don't find any
freepage in requested migratetype buddy list. This is for mitigating
fragmentation, but, there is a lack of migratetype consideration and it is
too excessive compared to page allocator's anti fragmentation algorithm.
Not considering migratetype would cause premature finish of compaction.
For example, if allocation request is for unmovable migratetype, freepage
with CMA migratetype doesn't help that allocation and compaction should
not be stopped. But, current logic regards this situation as compaction
is no longer needed, so finish the compaction.
Secondly, condition is too excessive compared to page allocator's logic.
We can steal freepage from other migratetype and change pageblock
migratetype on more relaxed conditions in page allocator. This is
designed to prevent fragmentation and we can use it here. Imposing hard
constraint only to the compaction doesn't help much in this case since
page allocator would cause fragmentation again.
To solve these problems, this patch borrows anti fragmentation logic from
page allocator. It will reduce premature compaction finish in some cases
and reduce excessive compaction work.
stress-highalloc test in mmtests with non movable order 7 allocation shows
considerable increase of compaction success rate.
Compaction success rate (Compaction success * 100 / Compaction stalls, %)
31.82 : 42.20
I tested it on non-reboot 5 runs stress-highalloc benchmark and found that
there is no more degradation on allocation success rate than before. That
roughly means that this patch doesn't result in more fragmentations.
Vlastimil suggests additional idea that we only test for fallbacks when
migration scanner has scanned a whole pageblock. It looked good for
fragmentation because chance of stealing increase due to making more free
pages in certain pageblock. So, I tested it, but, it results in decreased
compaction success rate, roughly 38.00. I guess the reason that if system
is low memory condition, watermark check could be failed due to not enough
order 0 free page and so, sometimes, we can't reach a fallback check
although migrate_pfn is aligned to pageblock_nr_pages. I can insert code
to cope with this situation but it makes code more complicated so I don't
include his idea at this patch.
[akpm@linux-foundation.org: fix CONFIG_CMA=n build]
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Mel Gorman <mgorman@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 16:45:21 -06:00
|
|
|
/*
|
|
|
|
* Check whether there is a suitable fallback freepage with requested order.
|
|
|
|
* If only_stealable is true, this function returns fallback_mt only if
|
|
|
|
* we can steal other freepages all together. This would help to reduce
|
|
|
|
* fragmentation due to mixed migratetype pages in one pageblock.
|
|
|
|
*/
|
|
|
|
int find_suitable_fallback(struct free_area *area, unsigned int order,
|
|
|
|
int migratetype, bool only_stealable, bool *can_steal)
|
2015-04-14 16:45:18 -06:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
int fallback_mt;
|
|
|
|
|
|
|
|
if (area->nr_free == 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
*can_steal = false;
|
|
|
|
for (i = 0;; i++) {
|
|
|
|
fallback_mt = fallbacks[migratetype][i];
|
2015-11-06 17:28:34 -07:00
|
|
|
if (fallback_mt == MIGRATE_TYPES)
|
2015-04-14 16:45:18 -06:00
|
|
|
break;
|
|
|
|
|
|
|
|
if (list_empty(&area->free_list[fallback_mt]))
|
|
|
|
continue;
|
mm/page_allo.c: restructure free-page stealing code and fix a bug
The free-page stealing code in __rmqueue_fallback() is somewhat hard to
follow, and has an incredible amount of subtlety hidden inside!
First off, there is a minor bug in the reporting of change-of-ownership of
pageblocks. Under some conditions, we try to move upto
'pageblock_nr_pages' no. of pages to the preferred allocation list. But
we change the ownership of that pageblock to the preferred type only if we
manage to successfully move atleast half of that pageblock (or if
page_group_by_mobility_disabled is set).
However, the current code ignores the latter part and sets the
'migratetype' variable to the preferred type, irrespective of whether we
actually changed the pageblock migratetype of that block or not. So, the
page_alloc_extfrag tracepoint can end up printing incorrect info (i.e.,
'change_ownership' might be shown as 1 when it must have been 0).
So fixing this involves moving the update of the 'migratetype' variable to
the right place. But looking closer, we observe that the 'migratetype'
variable is used subsequently for checks such as "is_migrate_cma()".
Obviously the intent there is to check if the *fallback* type is
MIGRATE_CMA, but since we already set the 'migratetype' variable to
start_migratetype, we end up checking if the *preferred* type is
MIGRATE_CMA!!
To make things more interesting, this actually doesn't cause a bug in
practice, because we never change *anything* if the fallback type is CMA.
So, restructure the code in such a way that it is trivial to understand
what is going on, and also fix the above mentioned bug. And while at it,
also add a comment explaining the subtlety behind the migratetype used in
the call to expand().
[akpm@linux-foundation.org: remove unneeded `inline', small coding-style fix]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Cody P Schafer <cody@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-09-11 15:20:35 -06:00
|
|
|
|
2015-04-14 16:45:18 -06:00
|
|
|
if (can_steal_fallback(order, migratetype))
|
|
|
|
*can_steal = true;
|
|
|
|
|
mm/compaction: enhance compaction finish condition
Compaction has anti fragmentation algorithm. It is that freepage should
be more than pageblock order to finish the compaction if we don't find any
freepage in requested migratetype buddy list. This is for mitigating
fragmentation, but, there is a lack of migratetype consideration and it is
too excessive compared to page allocator's anti fragmentation algorithm.
Not considering migratetype would cause premature finish of compaction.
For example, if allocation request is for unmovable migratetype, freepage
with CMA migratetype doesn't help that allocation and compaction should
not be stopped. But, current logic regards this situation as compaction
is no longer needed, so finish the compaction.
Secondly, condition is too excessive compared to page allocator's logic.
We can steal freepage from other migratetype and change pageblock
migratetype on more relaxed conditions in page allocator. This is
designed to prevent fragmentation and we can use it here. Imposing hard
constraint only to the compaction doesn't help much in this case since
page allocator would cause fragmentation again.
To solve these problems, this patch borrows anti fragmentation logic from
page allocator. It will reduce premature compaction finish in some cases
and reduce excessive compaction work.
stress-highalloc test in mmtests with non movable order 7 allocation shows
considerable increase of compaction success rate.
Compaction success rate (Compaction success * 100 / Compaction stalls, %)
31.82 : 42.20
I tested it on non-reboot 5 runs stress-highalloc benchmark and found that
there is no more degradation on allocation success rate than before. That
roughly means that this patch doesn't result in more fragmentations.
Vlastimil suggests additional idea that we only test for fallbacks when
migration scanner has scanned a whole pageblock. It looked good for
fragmentation because chance of stealing increase due to making more free
pages in certain pageblock. So, I tested it, but, it results in decreased
compaction success rate, roughly 38.00. I guess the reason that if system
is low memory condition, watermark check could be failed due to not enough
order 0 free page and so, sometimes, we can't reach a fallback check
although migrate_pfn is aligned to pageblock_nr_pages. I can insert code
to cope with this situation but it makes code more complicated so I don't
include his idea at this patch.
[akpm@linux-foundation.org: fix CONFIG_CMA=n build]
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Mel Gorman <mgorman@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 16:45:21 -06:00
|
|
|
if (!only_stealable)
|
|
|
|
return fallback_mt;
|
|
|
|
|
|
|
|
if (*can_steal)
|
|
|
|
return fallback_mt;
|
mm/page_allo.c: restructure free-page stealing code and fix a bug
The free-page stealing code in __rmqueue_fallback() is somewhat hard to
follow, and has an incredible amount of subtlety hidden inside!
First off, there is a minor bug in the reporting of change-of-ownership of
pageblocks. Under some conditions, we try to move upto
'pageblock_nr_pages' no. of pages to the preferred allocation list. But
we change the ownership of that pageblock to the preferred type only if we
manage to successfully move atleast half of that pageblock (or if
page_group_by_mobility_disabled is set).
However, the current code ignores the latter part and sets the
'migratetype' variable to the preferred type, irrespective of whether we
actually changed the pageblock migratetype of that block or not. So, the
page_alloc_extfrag tracepoint can end up printing incorrect info (i.e.,
'change_ownership' might be shown as 1 when it must have been 0).
So fixing this involves moving the update of the 'migratetype' variable to
the right place. But looking closer, we observe that the 'migratetype'
variable is used subsequently for checks such as "is_migrate_cma()".
Obviously the intent there is to check if the *fallback* type is
MIGRATE_CMA, but since we already set the 'migratetype' variable to
start_migratetype, we end up checking if the *preferred* type is
MIGRATE_CMA!!
To make things more interesting, this actually doesn't cause a bug in
practice, because we never change *anything* if the fallback type is CMA.
So, restructure the code in such a way that it is trivial to understand
what is going on, and also fix the above mentioned bug. And while at it,
also add a comment explaining the subtlety behind the migratetype used in
the call to expand().
[akpm@linux-foundation.org: remove unneeded `inline', small coding-style fix]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Cody P Schafer <cody@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-09-11 15:20:35 -06:00
|
|
|
}
|
2015-04-14 16:45:18 -06:00
|
|
|
|
|
|
|
return -1;
|
mm/page_allo.c: restructure free-page stealing code and fix a bug
The free-page stealing code in __rmqueue_fallback() is somewhat hard to
follow, and has an incredible amount of subtlety hidden inside!
First off, there is a minor bug in the reporting of change-of-ownership of
pageblocks. Under some conditions, we try to move upto
'pageblock_nr_pages' no. of pages to the preferred allocation list. But
we change the ownership of that pageblock to the preferred type only if we
manage to successfully move atleast half of that pageblock (or if
page_group_by_mobility_disabled is set).
However, the current code ignores the latter part and sets the
'migratetype' variable to the preferred type, irrespective of whether we
actually changed the pageblock migratetype of that block or not. So, the
page_alloc_extfrag tracepoint can end up printing incorrect info (i.e.,
'change_ownership' might be shown as 1 when it must have been 0).
So fixing this involves moving the update of the 'migratetype' variable to
the right place. But looking closer, we observe that the 'migratetype'
variable is used subsequently for checks such as "is_migrate_cma()".
Obviously the intent there is to check if the *fallback* type is
MIGRATE_CMA, but since we already set the 'migratetype' variable to
start_migratetype, we end up checking if the *preferred* type is
MIGRATE_CMA!!
To make things more interesting, this actually doesn't cause a bug in
practice, because we never change *anything* if the fallback type is CMA.
So, restructure the code in such a way that it is trivial to understand
what is going on, and also fix the above mentioned bug. And while at it,
also add a comment explaining the subtlety behind the migratetype used in
the call to expand().
[akpm@linux-foundation.org: remove unneeded `inline', small coding-style fix]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Cody P Schafer <cody@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-09-11 15:20:35 -06:00
|
|
|
}
|
|
|
|
|
2015-11-06 17:28:37 -07:00
|
|
|
/*
|
|
|
|
* Reserve a pageblock for exclusive use of high-order atomic allocations if
|
|
|
|
* there are no empty page blocks that contain a page with a suitable order
|
|
|
|
*/
|
|
|
|
static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
|
|
|
|
unsigned int alloc_order)
|
|
|
|
{
|
|
|
|
int mt;
|
|
|
|
unsigned long max_managed, flags;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Limit the number reserved to 1 pageblock or roughly 1% of a zone.
|
|
|
|
* Check is race-prone but harmless.
|
|
|
|
*/
|
|
|
|
max_managed = (zone->managed_pages / 100) + pageblock_nr_pages;
|
|
|
|
if (zone->nr_reserved_highatomic >= max_managed)
|
|
|
|
return;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&zone->lock, flags);
|
|
|
|
|
|
|
|
/* Recheck the nr_reserved_highatomic limit under the lock */
|
|
|
|
if (zone->nr_reserved_highatomic >= max_managed)
|
|
|
|
goto out_unlock;
|
|
|
|
|
|
|
|
/* Yoink! */
|
|
|
|
mt = get_pageblock_migratetype(page);
|
|
|
|
if (mt != MIGRATE_HIGHATOMIC &&
|
|
|
|
!is_migrate_isolate(mt) && !is_migrate_cma(mt)) {
|
|
|
|
zone->nr_reserved_highatomic += pageblock_nr_pages;
|
|
|
|
set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
|
|
|
|
move_freepages_block(zone, page, MIGRATE_HIGHATOMIC);
|
|
|
|
}
|
|
|
|
|
|
|
|
out_unlock:
|
|
|
|
spin_unlock_irqrestore(&zone->lock, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Used when an allocation is about to fail under memory pressure. This
|
|
|
|
* potentially hurts the reliability of high-order allocations when under
|
|
|
|
* intense memory pressure but failed atomic allocations should be easier
|
|
|
|
* to recover from than an OOM.
|
|
|
|
*/
|
|
|
|
static void unreserve_highatomic_pageblock(const struct alloc_context *ac)
|
|
|
|
{
|
|
|
|
struct zonelist *zonelist = ac->zonelist;
|
|
|
|
unsigned long flags;
|
|
|
|
struct zoneref *z;
|
|
|
|
struct zone *zone;
|
|
|
|
struct page *page;
|
|
|
|
int order;
|
|
|
|
|
|
|
|
for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
|
|
|
|
ac->nodemask) {
|
|
|
|
/* Preserve at least one pageblock */
|
|
|
|
if (zone->nr_reserved_highatomic <= pageblock_nr_pages)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&zone->lock, flags);
|
|
|
|
for (order = 0; order < MAX_ORDER; order++) {
|
|
|
|
struct free_area *area = &(zone->free_area[order]);
|
|
|
|
|
2016-01-14 16:20:30 -07:00
|
|
|
page = list_first_entry_or_null(
|
|
|
|
&area->free_list[MIGRATE_HIGHATOMIC],
|
|
|
|
struct page, lru);
|
|
|
|
if (!page)
|
2015-11-06 17:28:37 -07:00
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* It should never happen but changes to locking could
|
|
|
|
* inadvertently allow a per-cpu drain to add pages
|
|
|
|
* to MIGRATE_HIGHATOMIC while unreserving so be safe
|
|
|
|
* and watch for underflows.
|
|
|
|
*/
|
|
|
|
zone->nr_reserved_highatomic -= min(pageblock_nr_pages,
|
|
|
|
zone->nr_reserved_highatomic);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Convert to ac->migratetype and avoid the normal
|
|
|
|
* pageblock stealing heuristics. Minimally, the caller
|
|
|
|
* is doing the work and needs the pages. More
|
|
|
|
* importantly, if the block was always converted to
|
|
|
|
* MIGRATE_UNMOVABLE or another type then the number
|
|
|
|
* of pageblocks that cannot be completely freed
|
|
|
|
* may increase.
|
|
|
|
*/
|
|
|
|
set_pageblock_migratetype(page, ac->migratetype);
|
|
|
|
move_freepages_block(zone, page, ac->migratetype);
|
|
|
|
spin_unlock_irqrestore(&zone->lock, flags);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
spin_unlock_irqrestore(&zone->lock, flags);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-10-16 02:25:48 -06:00
|
|
|
/* Remove an element from the buddy allocator from the fallback list */
|
2009-06-16 16:32:06 -06:00
|
|
|
static inline struct page *
|
2014-06-04 17:10:21 -06:00
|
|
|
__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
|
2007-10-16 02:25:48 -06:00
|
|
|
{
|
2013-09-11 15:20:34 -06:00
|
|
|
struct free_area *area;
|
2014-06-04 17:10:21 -06:00
|
|
|
unsigned int current_order;
|
2007-10-16 02:25:48 -06:00
|
|
|
struct page *page;
|
2015-04-14 16:45:18 -06:00
|
|
|
int fallback_mt;
|
|
|
|
bool can_steal;
|
2007-10-16 02:25:48 -06:00
|
|
|
|
|
|
|
/* Find the largest possible block of pages in the other list */
|
2014-06-04 17:10:21 -06:00
|
|
|
for (current_order = MAX_ORDER-1;
|
|
|
|
current_order >= order && current_order <= MAX_ORDER-1;
|
|
|
|
--current_order) {
|
2015-04-14 16:45:18 -06:00
|
|
|
area = &(zone->free_area[current_order]);
|
|
|
|
fallback_mt = find_suitable_fallback(area, current_order,
|
mm/compaction: enhance compaction finish condition
Compaction has anti fragmentation algorithm. It is that freepage should
be more than pageblock order to finish the compaction if we don't find any
freepage in requested migratetype buddy list. This is for mitigating
fragmentation, but, there is a lack of migratetype consideration and it is
too excessive compared to page allocator's anti fragmentation algorithm.
Not considering migratetype would cause premature finish of compaction.
For example, if allocation request is for unmovable migratetype, freepage
with CMA migratetype doesn't help that allocation and compaction should
not be stopped. But, current logic regards this situation as compaction
is no longer needed, so finish the compaction.
Secondly, condition is too excessive compared to page allocator's logic.
We can steal freepage from other migratetype and change pageblock
migratetype on more relaxed conditions in page allocator. This is
designed to prevent fragmentation and we can use it here. Imposing hard
constraint only to the compaction doesn't help much in this case since
page allocator would cause fragmentation again.
To solve these problems, this patch borrows anti fragmentation logic from
page allocator. It will reduce premature compaction finish in some cases
and reduce excessive compaction work.
stress-highalloc test in mmtests with non movable order 7 allocation shows
considerable increase of compaction success rate.
Compaction success rate (Compaction success * 100 / Compaction stalls, %)
31.82 : 42.20
I tested it on non-reboot 5 runs stress-highalloc benchmark and found that
there is no more degradation on allocation success rate than before. That
roughly means that this patch doesn't result in more fragmentations.
Vlastimil suggests additional idea that we only test for fallbacks when
migration scanner has scanned a whole pageblock. It looked good for
fragmentation because chance of stealing increase due to making more free
pages in certain pageblock. So, I tested it, but, it results in decreased
compaction success rate, roughly 38.00. I guess the reason that if system
is low memory condition, watermark check could be failed due to not enough
order 0 free page and so, sometimes, we can't reach a fallback check
although migrate_pfn is aligned to pageblock_nr_pages. I can insert code
to cope with this situation but it makes code more complicated so I don't
include his idea at this patch.
[akpm@linux-foundation.org: fix CONFIG_CMA=n build]
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Mel Gorman <mgorman@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 16:45:21 -06:00
|
|
|
start_migratetype, false, &can_steal);
|
2015-04-14 16:45:18 -06:00
|
|
|
if (fallback_mt == -1)
|
|
|
|
continue;
|
2007-10-16 02:25:48 -06:00
|
|
|
|
2016-01-14 16:20:30 -07:00
|
|
|
page = list_first_entry(&area->free_list[fallback_mt],
|
2015-04-14 16:45:18 -06:00
|
|
|
struct page, lru);
|
|
|
|
if (can_steal)
|
|
|
|
steal_suitable_fallback(zone, page, start_migratetype);
|
2007-10-16 02:25:48 -06:00
|
|
|
|
2015-04-14 16:45:18 -06:00
|
|
|
/* Remove the page from the freelists */
|
|
|
|
area->nr_free--;
|
|
|
|
list_del(&page->lru);
|
|
|
|
rmv_page_order(page);
|
mm: always steal split buddies in fallback allocations
When allocation falls back to another migratetype, it will steal a page
with highest available order, and (depending on this order and desired
migratetype), it might also steal the rest of free pages from the same
pageblock.
Given the preference of highest available order, it is likely that it will
be higher than the desired order, and result in the stolen buddy page
being split. The remaining pages after split are currently stolen only
when the rest of the free pages are stolen. This can however lead to
situations where for MOVABLE allocations we split e.g. order-4 fallback
UNMOVABLE page, but steal only order-0 page. Then on the next MOVABLE
allocation (which may be batched to fill the pcplists) we split another
order-3 or higher page, etc. By stealing all pages that we have split, we
can avoid further stealing.
This patch therefore adjusts the page stealing so that buddy pages created
by split are always stolen. This has effect only on MOVABLE allocations,
as RECLAIMABLE and UNMOVABLE allocations already always do that in
addition to stealing the rest of free pages from the pageblock. The
change also allows to simplify try_to_steal_freepages() and factor out CMA
handling.
According to Mel, it has been intended since the beginning that buddy
pages after split would be stolen always, but it doesn't seem like it was
ever the case until commit 47118af076f6 ("mm: mmzone: MIGRATE_CMA
migration type added"). The commit has unintentionally introduced this
behavior, but was reverted by commit 0cbef29a7821 ("mm:
__rmqueue_fallback() should respect pageblock type"). Neither included
evaluation.
My evaluation with stress-highalloc from mmtests shows about 2.5x
reduction of page stealing events for MOVABLE allocations, without
affecting the page stealing events for other allocation migratetypes.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Acked-by: Minchan Kim <minchan@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-11 16:28:18 -07:00
|
|
|
|
2015-04-14 16:45:18 -06:00
|
|
|
expand(zone, page, order, current_order, area,
|
|
|
|
start_migratetype);
|
|
|
|
/*
|
2015-09-08 16:01:25 -06:00
|
|
|
* The pcppage_migratetype may differ from pageblock's
|
2015-04-14 16:45:18 -06:00
|
|
|
* migratetype depending on the decisions in
|
2015-09-08 16:01:25 -06:00
|
|
|
* find_suitable_fallback(). This is OK as long as it does not
|
|
|
|
* differ for MIGRATE_CMA pageblocks. Those can be used as
|
|
|
|
* fallback only via special __rmqueue_cma_fallback() function
|
2015-04-14 16:45:18 -06:00
|
|
|
*/
|
2015-09-08 16:01:25 -06:00
|
|
|
set_pcppage_migratetype(page, start_migratetype);
|
2009-09-21 18:02:42 -06:00
|
|
|
|
2015-04-14 16:45:18 -06:00
|
|
|
trace_mm_page_alloc_extfrag(page, order, current_order,
|
|
|
|
start_migratetype, fallback_mt);
|
2009-09-21 18:02:42 -06:00
|
|
|
|
2015-04-14 16:45:18 -06:00
|
|
|
return page;
|
2007-10-16 02:25:48 -06:00
|
|
|
}
|
|
|
|
|
2009-06-16 16:32:04 -06:00
|
|
|
return NULL;
|
2007-10-16 02:25:48 -06:00
|
|
|
}
|
|
|
|
|
Bias the location of pages freed for min_free_kbytes in the same MAX_ORDER_NR_PAGES blocks
The standard buddy allocator always favours the smallest block of pages.
The effect of this is that the pages free to satisfy min_free_kbytes tends
to be preserved since boot time at the same location of memory ffor a very
long time and as a contiguous block. When an administrator sets the
reserve at 16384 at boot time, it tends to be the same MAX_ORDER blocks
that remain free. This allows the occasional high atomic allocation to
succeed up until the point the blocks are split. In practice, it is
difficult to split these blocks but when they do split, the benefit of
having min_free_kbytes for contiguous blocks disappears. Additionally,
increasing min_free_kbytes once the system has been running for some time
has no guarantee of creating contiguous blocks.
On the other hand, CONFIG_PAGE_GROUP_BY_MOBILITY favours splitting large
blocks when there are no free pages of the appropriate type available. A
side-effect of this is that all blocks in memory tends to be used up and
the contiguous free blocks from boot time are not preserved like in the
vanilla allocator. This can cause a problem if a new caller is unwilling
to reclaim or does not reclaim for long enough.
A failure scenario was found for a wireless network device allocating
order-1 atomic allocations but the allocations were not intense or frequent
enough for a whole block of pages to be preserved for MIGRATE_HIGHALLOC.
This was reproduced on a desktop by booting with mem=256mb, forcing the
driver to allocate at order-1, running a bittorrent client (downloading a
debian ISO) and building a kernel with -j2.
This patch addresses the problem on the desktop machine booted with
mem=256mb. It works by setting aside a reserve of MAX_ORDER_NR_PAGES
blocks, the number of which depends on the value of min_free_kbytes. These
blocks are only fallen back to when there is no other free pages. Then the
smallest possible page is used just like the normal buddy allocator instead
of the largest possible page to preserve contiguous pages The pages in free
lists in the reserve blocks are never taken for another migrate type. The
results is that even if min_free_kbytes is set to a low value, contiguous
blocks will be preserved in the MIGRATE_RESERVE blocks.
This works better than the vanilla allocator because if min_free_kbytes is
increased, a new reserve block will be chosen based on the location of
reclaimable pages and the block will free up as contiguous pages. In the
vanilla allocator, no effort is made to target a block of pages to free as
contiguous pages and min_free_kbytes pages are scattered randomly.
This effect has been observed on the test machine. min_free_kbytes was set
initially low but it was kept as a contiguous free block within
MIGRATE_RESERVE. min_free_kbytes was then set to a higher value and over a
period of time, the free blocks were within the reserve and coalescing.
How long it takes to free up depends on how quickly LRU is rotating.
Amusingly, this means that more activity will free the blocks faster.
This mechanism potentially replaces MIGRATE_HIGHALLOC as it may be more
effective than grouping contiguous free pages together. It all depends on
whether the number of active atomic high allocations exceeds
min_free_kbytes or not. If the number of active allocations exceeds
min_free_kbytes, it's worth it but maybe in that situation, min_free_kbytes
should be set higher. Once there are no more reports of allocation
failures, a patch will be submitted that backs out MIGRATE_HIGHALLOC and
see if the reports stay missing.
Credit to Mariusz Kozlowski for discovering the problem, describing the
failure scenario and testing patches and scenarios.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:58 -06:00
|
|
|
/*
|
2005-04-16 16:20:36 -06:00
|
|
|
* Do the hard work of removing an element from the buddy allocator.
|
|
|
|
* Call me with the zone->lock already held.
|
|
|
|
*/
|
2007-10-16 02:25:48 -06:00
|
|
|
static struct page *__rmqueue(struct zone *zone, unsigned int order,
|
2016-01-14 16:20:28 -07:00
|
|
|
int migratetype)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
|
Bias the location of pages freed for min_free_kbytes in the same MAX_ORDER_NR_PAGES blocks
The standard buddy allocator always favours the smallest block of pages.
The effect of this is that the pages free to satisfy min_free_kbytes tends
to be preserved since boot time at the same location of memory ffor a very
long time and as a contiguous block. When an administrator sets the
reserve at 16384 at boot time, it tends to be the same MAX_ORDER blocks
that remain free. This allows the occasional high atomic allocation to
succeed up until the point the blocks are split. In practice, it is
difficult to split these blocks but when they do split, the benefit of
having min_free_kbytes for contiguous blocks disappears. Additionally,
increasing min_free_kbytes once the system has been running for some time
has no guarantee of creating contiguous blocks.
On the other hand, CONFIG_PAGE_GROUP_BY_MOBILITY favours splitting large
blocks when there are no free pages of the appropriate type available. A
side-effect of this is that all blocks in memory tends to be used up and
the contiguous free blocks from boot time are not preserved like in the
vanilla allocator. This can cause a problem if a new caller is unwilling
to reclaim or does not reclaim for long enough.
A failure scenario was found for a wireless network device allocating
order-1 atomic allocations but the allocations were not intense or frequent
enough for a whole block of pages to be preserved for MIGRATE_HIGHALLOC.
This was reproduced on a desktop by booting with mem=256mb, forcing the
driver to allocate at order-1, running a bittorrent client (downloading a
debian ISO) and building a kernel with -j2.
This patch addresses the problem on the desktop machine booted with
mem=256mb. It works by setting aside a reserve of MAX_ORDER_NR_PAGES
blocks, the number of which depends on the value of min_free_kbytes. These
blocks are only fallen back to when there is no other free pages. Then the
smallest possible page is used just like the normal buddy allocator instead
of the largest possible page to preserve contiguous pages The pages in free
lists in the reserve blocks are never taken for another migrate type. The
results is that even if min_free_kbytes is set to a low value, contiguous
blocks will be preserved in the MIGRATE_RESERVE blocks.
This works better than the vanilla allocator because if min_free_kbytes is
increased, a new reserve block will be chosen based on the location of
reclaimable pages and the block will free up as contiguous pages. In the
vanilla allocator, no effort is made to target a block of pages to free as
contiguous pages and min_free_kbytes pages are scattered randomly.
This effect has been observed on the test machine. min_free_kbytes was set
initially low but it was kept as a contiguous free block within
MIGRATE_RESERVE. min_free_kbytes was then set to a higher value and over a
period of time, the free blocks were within the reserve and coalescing.
How long it takes to free up depends on how quickly LRU is rotating.
Amusingly, this means that more activity will free the blocks faster.
This mechanism potentially replaces MIGRATE_HIGHALLOC as it may be more
effective than grouping contiguous free pages together. It all depends on
whether the number of active atomic high allocations exceeds
min_free_kbytes or not. If the number of active allocations exceeds
min_free_kbytes, it's worth it but maybe in that situation, min_free_kbytes
should be set higher. Once there are no more reports of allocation
failures, a patch will be submitted that backs out MIGRATE_HIGHALLOC and
see if the reports stay missing.
Credit to Mariusz Kozlowski for discovering the problem, describing the
failure scenario and testing patches and scenarios.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:58 -06:00
|
|
|
page = __rmqueue_smallest(zone, order, migratetype);
|
2015-11-06 17:28:34 -07:00
|
|
|
if (unlikely(!page)) {
|
2015-04-14 16:45:15 -06:00
|
|
|
if (migratetype == MIGRATE_MOVABLE)
|
|
|
|
page = __rmqueue_cma_fallback(zone, order);
|
|
|
|
|
|
|
|
if (!page)
|
|
|
|
page = __rmqueue_fallback(zone, order, migratetype);
|
2009-06-16 16:32:04 -06:00
|
|
|
}
|
|
|
|
|
2009-09-21 18:02:44 -06:00
|
|
|
trace_mm_page_alloc_zone_locked(page, order, migratetype);
|
2007-10-16 02:25:48 -06:00
|
|
|
return page;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2012-01-11 07:16:11 -07:00
|
|
|
/*
|
2005-04-16 16:20:36 -06:00
|
|
|
* Obtain a specified number of elements from the buddy allocator, all under
|
|
|
|
* a single hold of the lock, for efficiency. Add them to the supplied list.
|
|
|
|
* Returns the number of new pages which were placed at *list.
|
|
|
|
*/
|
2012-01-11 07:16:11 -07:00
|
|
|
static int rmqueue_bulk(struct zone *zone, unsigned int order,
|
2007-10-16 02:25:48 -06:00
|
|
|
unsigned long count, struct list_head *list,
|
2014-06-04 17:10:22 -06:00
|
|
|
int migratetype, bool cold)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
mm/page_alloc: prevent MIGRATE_RESERVE pages from being misplaced
For the MIGRATE_RESERVE pages, it is useful when they do not get
misplaced on free_list of other migratetype, otherwise they might get
allocated prematurely and e.g. fragment the MIGRATE_RESEVE pageblocks.
While this cannot be avoided completely when allocating new
MIGRATE_RESERVE pageblocks in min_free_kbytes sysctl handler, we should
prevent the misplacement where possible.
Currently, it is possible for the misplacement to happen when a
MIGRATE_RESERVE page is allocated on pcplist through rmqueue_bulk() as a
fallback for other desired migratetype, and then later freed back
through free_pcppages_bulk() without being actually used. This happens
because free_pcppages_bulk() uses get_freepage_migratetype() to choose
the free_list, and rmqueue_bulk() calls set_freepage_migratetype() with
the *desired* migratetype and not the page's original MIGRATE_RESERVE
migratetype.
This patch fixes the problem by moving the call to
set_freepage_migratetype() from rmqueue_bulk() down to
__rmqueue_smallest() and __rmqueue_fallback() where the actual page's
migratetype (e.g. from which free_list the page is taken from) is used.
Note that this migratetype might be different from the pageblock's
migratetype due to freepage stealing decisions. This is OK, as page
stealing never uses MIGRATE_RESERVE as a fallback, and also takes care
to leave all MIGRATE_CMA pages on the correct freelist.
Therefore, as an additional benefit, the call to
get_pageblock_migratetype() from rmqueue_bulk() when CMA is enabled, can
be removed completely. This relies on the fact that MIGRATE_CMA
pageblocks are created only during system init, and the above. The
related is_migrate_isolate() check is also unnecessary, as memory
isolation has other ways to move pages between freelists, and drain pcp
lists containing pages that should be isolated. The buffered_rmqueue()
can also benefit from calling get_freepage_migratetype() instead of
get_pageblock_migratetype().
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reported-by: Yong-Taek Lee <ytk.lee@samsung.com>
Reported-by: Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
Suggested-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Suggested-by: Mel Gorman <mgorman@suse.de>
Acked-by: Minchan Kim <minchan@kernel.org>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: "Wang, Yalin" <Yalin.Wang@sonymobile.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 17:07:22 -06:00
|
|
|
int i;
|
2012-01-11 07:16:11 -07:00
|
|
|
|
2006-01-06 01:10:56 -07:00
|
|
|
spin_lock(&zone->lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
for (i = 0; i < count; ++i) {
|
2016-01-14 16:20:28 -07:00
|
|
|
struct page *page = __rmqueue(zone, order, migratetype);
|
2006-01-06 01:11:01 -07:00
|
|
|
if (unlikely(page == NULL))
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
2007-12-17 17:20:05 -07:00
|
|
|
|
2016-05-19 18:14:35 -06:00
|
|
|
if (unlikely(check_pcp_refill(page)))
|
|
|
|
continue;
|
|
|
|
|
2007-12-17 17:20:05 -07:00
|
|
|
/*
|
|
|
|
* Split buddy pages returned by expand() are received here
|
|
|
|
* in physical page order. The page is added to the callers and
|
|
|
|
* list and the list head then moves forward. From the callers
|
|
|
|
* perspective, the linked list is ordered by page number in
|
|
|
|
* some conditions. This is useful for IO devices that can
|
|
|
|
* merge IO requests if the physical pages are ordered
|
|
|
|
* properly.
|
|
|
|
*/
|
2014-06-04 17:10:22 -06:00
|
|
|
if (likely(!cold))
|
2009-07-29 16:02:04 -06:00
|
|
|
list_add(&page->lru, list);
|
|
|
|
else
|
|
|
|
list_add_tail(&page->lru, list);
|
2007-12-17 17:20:05 -07:00
|
|
|
list = &page->lru;
|
2015-09-08 16:01:25 -06:00
|
|
|
if (is_migrate_cma(get_pcppage_migratetype(page)))
|
2012-10-08 17:32:02 -06:00
|
|
|
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
|
|
|
|
-(1 << order));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-06-16 16:32:13 -06:00
|
|
|
__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
|
2006-01-06 01:10:56 -07:00
|
|
|
spin_unlock(&zone->lock);
|
2006-01-06 01:11:01 -07:00
|
|
|
return i;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2005-06-21 18:14:57 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
2006-03-09 18:33:54 -07:00
|
|
|
/*
|
2007-05-09 03:35:14 -06:00
|
|
|
* Called from the vmstat counter updater to drain pagesets of this
|
|
|
|
* currently executing processor on remote nodes after they have
|
|
|
|
* expired.
|
|
|
|
*
|
2006-03-22 01:09:08 -07:00
|
|
|
* Note that this function must be called with the thread pinned to
|
|
|
|
* a single processor.
|
2006-03-09 18:33:54 -07:00
|
|
|
*/
|
2007-05-09 03:35:14 -06:00
|
|
|
void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
|
2005-06-21 18:14:57 -06:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
2014-08-06 17:05:15 -06:00
|
|
|
int to_drain, batch;
|
2005-06-21 18:14:57 -06:00
|
|
|
|
2007-05-09 03:35:14 -06:00
|
|
|
local_irq_save(flags);
|
2015-04-15 17:14:08 -06:00
|
|
|
batch = READ_ONCE(pcp->batch);
|
2014-08-06 17:05:15 -06:00
|
|
|
to_drain = min(pcp->count, batch);
|
2012-07-31 17:42:53 -06:00
|
|
|
if (to_drain > 0) {
|
|
|
|
free_pcppages_bulk(zone, to_drain, pcp);
|
|
|
|
pcp->count -= to_drain;
|
|
|
|
}
|
2007-05-09 03:35:14 -06:00
|
|
|
local_irq_restore(flags);
|
2005-06-21 18:14:57 -06:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2008-02-04 23:29:11 -07:00
|
|
|
/*
|
2014-12-10 16:43:01 -07:00
|
|
|
* Drain pcplists of the indicated processor and zone.
|
2008-02-04 23:29:11 -07:00
|
|
|
*
|
|
|
|
* The processor must either be the current processor and the
|
|
|
|
* thread pinned to the current processor or a processor that
|
|
|
|
* is not online.
|
|
|
|
*/
|
2014-12-10 16:43:01 -07:00
|
|
|
static void drain_pages_zone(unsigned int cpu, struct zone *zone)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-01-06 01:10:56 -07:00
|
|
|
unsigned long flags;
|
2014-12-10 16:43:01 -07:00
|
|
|
struct per_cpu_pageset *pset;
|
|
|
|
struct per_cpu_pages *pcp;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2014-12-10 16:43:01 -07:00
|
|
|
local_irq_save(flags);
|
|
|
|
pset = per_cpu_ptr(zone->pageset, cpu);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2014-12-10 16:43:01 -07:00
|
|
|
pcp = &pset->pcp;
|
|
|
|
if (pcp->count) {
|
|
|
|
free_pcppages_bulk(zone, pcp->count, pcp);
|
|
|
|
pcp->count = 0;
|
|
|
|
}
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
2008-02-04 23:29:19 -07:00
|
|
|
|
2014-12-10 16:43:01 -07:00
|
|
|
/*
|
|
|
|
* Drain pcplists of all zones on the indicated processor.
|
|
|
|
*
|
|
|
|
* The processor must either be the current processor and the
|
|
|
|
* thread pinned to the current processor or a processor that
|
|
|
|
* is not online.
|
|
|
|
*/
|
|
|
|
static void drain_pages(unsigned int cpu)
|
|
|
|
{
|
|
|
|
struct zone *zone;
|
|
|
|
|
|
|
|
for_each_populated_zone(zone) {
|
|
|
|
drain_pages_zone(cpu, zone);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-02-04 23:29:11 -07:00
|
|
|
/*
|
|
|
|
* Spill all of this CPU's per-cpu pages back into the buddy allocator.
|
2014-12-10 16:43:01 -07:00
|
|
|
*
|
|
|
|
* The CPU has to be pinned. When zone parameter is non-NULL, spill just
|
|
|
|
* the single zone's pages.
|
2008-02-04 23:29:11 -07:00
|
|
|
*/
|
2014-12-10 16:43:01 -07:00
|
|
|
void drain_local_pages(struct zone *zone)
|
2008-02-04 23:29:11 -07:00
|
|
|
{
|
2014-12-10 16:43:01 -07:00
|
|
|
int cpu = smp_processor_id();
|
|
|
|
|
|
|
|
if (zone)
|
|
|
|
drain_pages_zone(cpu, zone);
|
|
|
|
else
|
|
|
|
drain_pages(cpu);
|
2008-02-04 23:29:11 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2012-03-28 15:42:45 -06:00
|
|
|
* Spill all the per-cpu pages from all CPUs back into the buddy allocator.
|
|
|
|
*
|
2014-12-10 16:43:01 -07:00
|
|
|
* When zone parameter is non-NULL, spill just the single zone's pages.
|
|
|
|
*
|
2012-03-28 15:42:45 -06:00
|
|
|
* Note that this code is protected against sending an IPI to an offline
|
|
|
|
* CPU but does not guarantee sending an IPI to newly hotplugged CPUs:
|
|
|
|
* on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but
|
|
|
|
* nothing keeps CPUs from showing up after we populated the cpumask and
|
|
|
|
* before the call to on_each_cpu_mask().
|
2008-02-04 23:29:11 -07:00
|
|
|
*/
|
2014-12-10 16:43:01 -07:00
|
|
|
void drain_all_pages(struct zone *zone)
|
2008-02-04 23:29:11 -07:00
|
|
|
{
|
2012-03-28 15:42:45 -06:00
|
|
|
int cpu;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate in the BSS so we wont require allocation in
|
|
|
|
* direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
|
|
|
|
*/
|
|
|
|
static cpumask_t cpus_with_pcps;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't care about racing with CPU hotplug event
|
|
|
|
* as offline notification will cause the notified
|
|
|
|
* cpu to drain that CPU pcps and on_each_cpu_mask
|
|
|
|
* disables preemption as part of its processing
|
|
|
|
*/
|
|
|
|
for_each_online_cpu(cpu) {
|
2014-12-10 16:43:01 -07:00
|
|
|
struct per_cpu_pageset *pcp;
|
|
|
|
struct zone *z;
|
2012-03-28 15:42:45 -06:00
|
|
|
bool has_pcps = false;
|
2014-12-10 16:43:01 -07:00
|
|
|
|
|
|
|
if (zone) {
|
2012-03-28 15:42:45 -06:00
|
|
|
pcp = per_cpu_ptr(zone->pageset, cpu);
|
2014-12-10 16:43:01 -07:00
|
|
|
if (pcp->pcp.count)
|
2012-03-28 15:42:45 -06:00
|
|
|
has_pcps = true;
|
2014-12-10 16:43:01 -07:00
|
|
|
} else {
|
|
|
|
for_each_populated_zone(z) {
|
|
|
|
pcp = per_cpu_ptr(z->pageset, cpu);
|
|
|
|
if (pcp->pcp.count) {
|
|
|
|
has_pcps = true;
|
|
|
|
break;
|
|
|
|
}
|
2012-03-28 15:42:45 -06:00
|
|
|
}
|
|
|
|
}
|
2014-12-10 16:43:01 -07:00
|
|
|
|
2012-03-28 15:42:45 -06:00
|
|
|
if (has_pcps)
|
|
|
|
cpumask_set_cpu(cpu, &cpus_with_pcps);
|
|
|
|
else
|
|
|
|
cpumask_clear_cpu(cpu, &cpus_with_pcps);
|
|
|
|
}
|
2014-12-10 16:43:01 -07:00
|
|
|
on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages,
|
|
|
|
zone, 1);
|
2008-02-04 23:29:11 -07:00
|
|
|
}
|
|
|
|
|
2007-07-29 15:27:18 -06:00
|
|
|
#ifdef CONFIG_HIBERNATION
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
void mark_free_pages(struct zone *zone)
|
|
|
|
{
|
2006-09-26 00:32:49 -06:00
|
|
|
unsigned long pfn, max_zone_pfn;
|
|
|
|
unsigned long flags;
|
2014-06-04 17:10:21 -06:00
|
|
|
unsigned int order, t;
|
2016-01-14 16:20:33 -07:00
|
|
|
struct page *page;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-09-11 15:21:45 -06:00
|
|
|
if (zone_is_empty(zone))
|
2005-04-16 16:20:36 -06:00
|
|
|
return;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&zone->lock, flags);
|
2006-09-26 00:32:49 -06:00
|
|
|
|
2013-02-22 17:35:23 -07:00
|
|
|
max_zone_pfn = zone_end_pfn(zone);
|
2006-09-26 00:32:49 -06:00
|
|
|
for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
|
|
|
|
if (pfn_valid(pfn)) {
|
2016-01-14 16:20:33 -07:00
|
|
|
page = pfn_to_page(pfn);
|
2016-05-19 18:12:16 -06:00
|
|
|
|
|
|
|
if (page_zone(page) != zone)
|
|
|
|
continue;
|
|
|
|
|
2007-05-06 15:50:42 -06:00
|
|
|
if (!swsusp_page_is_forbidden(page))
|
|
|
|
swsusp_unset_page_free(page);
|
2006-09-26 00:32:49 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-16 02:25:48 -06:00
|
|
|
for_each_migratetype_order(order, t) {
|
2016-01-14 16:20:33 -07:00
|
|
|
list_for_each_entry(page,
|
|
|
|
&zone->free_area[order].free_list[t], lru) {
|
2006-09-26 00:32:49 -06:00
|
|
|
unsigned long i;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2016-01-14 16:20:33 -07:00
|
|
|
pfn = page_to_pfn(page);
|
2006-09-26 00:32:49 -06:00
|
|
|
for (i = 0; i < (1UL << order); i++)
|
2007-05-06 15:50:42 -06:00
|
|
|
swsusp_set_page_free(pfn_to_page(pfn + i));
|
2006-09-26 00:32:49 -06:00
|
|
|
}
|
2007-10-16 02:25:48 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_unlock_irqrestore(&zone->lock, flags);
|
|
|
|
}
|
2007-10-16 02:25:50 -06:00
|
|
|
#endif /* CONFIG_PM */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Free a 0-order page
|
2014-06-04 17:10:22 -06:00
|
|
|
* cold == true ? free a cold page : free a hot page
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2014-06-04 17:10:22 -06:00
|
|
|
void free_hot_cold_page(struct page *page, bool cold)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct zone *zone = page_zone(page);
|
|
|
|
struct per_cpu_pages *pcp;
|
|
|
|
unsigned long flags;
|
2014-06-04 17:10:17 -06:00
|
|
|
unsigned long pfn = page_to_pfn(page);
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
int migratetype;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2016-05-19 18:14:32 -06:00
|
|
|
if (!free_pcp_prepare(page))
|
2005-11-21 22:32:20 -07:00
|
|
|
return;
|
|
|
|
|
2014-06-04 17:10:17 -06:00
|
|
|
migratetype = get_pfnblock_migratetype(page, pfn);
|
2015-09-08 16:01:25 -06:00
|
|
|
set_pcppage_migratetype(page, migratetype);
|
2005-04-16 16:20:36 -06:00
|
|
|
local_irq_save(flags);
|
2006-06-30 02:55:45 -06:00
|
|
|
__count_vm_event(PGFREE);
|
2009-06-16 16:32:08 -06:00
|
|
|
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
/*
|
|
|
|
* We only track unmovable, reclaimable and movable on pcp lists.
|
|
|
|
* Free ISOLATE pages back to the allocator because they are being
|
|
|
|
* offlined but treat RESERVE as movable pages so we can get those
|
|
|
|
* areas back if necessary. Otherwise, we may have to free
|
|
|
|
* excessively into the page allocator
|
|
|
|
*/
|
|
|
|
if (migratetype >= MIGRATE_PCPTYPES) {
|
2013-02-22 17:33:58 -07:00
|
|
|
if (unlikely(is_migrate_isolate(migratetype))) {
|
2014-06-04 17:10:17 -06:00
|
|
|
free_one_page(zone, page, pfn, 0, migratetype);
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
migratetype = MIGRATE_MOVABLE;
|
|
|
|
}
|
|
|
|
|
2010-01-04 23:34:51 -07:00
|
|
|
pcp = &this_cpu_ptr(zone->pageset)->pcp;
|
2014-06-04 17:10:22 -06:00
|
|
|
if (!cold)
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
list_add(&page->lru, &pcp->lists[migratetype]);
|
2014-06-04 17:10:22 -06:00
|
|
|
else
|
|
|
|
list_add_tail(&page->lru, &pcp->lists[migratetype]);
|
2005-04-16 16:20:36 -06:00
|
|
|
pcp->count++;
|
2006-01-08 02:00:42 -07:00
|
|
|
if (pcp->count >= pcp->high) {
|
2015-04-15 17:14:08 -06:00
|
|
|
unsigned long batch = READ_ONCE(pcp->batch);
|
2013-07-03 16:01:32 -06:00
|
|
|
free_pcppages_bulk(zone, batch, pcp);
|
|
|
|
pcp->count -= batch;
|
2006-01-08 02:00:42 -07:00
|
|
|
}
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
|
|
|
|
out:
|
2005-04-16 16:20:36 -06:00
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
2012-01-10 16:07:04 -07:00
|
|
|
/*
|
|
|
|
* Free a list of 0-order pages
|
|
|
|
*/
|
2014-06-04 17:10:22 -06:00
|
|
|
void free_hot_cold_page_list(struct list_head *list, bool cold)
|
2012-01-10 16:07:04 -07:00
|
|
|
{
|
|
|
|
struct page *page, *next;
|
|
|
|
|
|
|
|
list_for_each_entry_safe(page, next, list, lru) {
|
2012-01-10 16:07:09 -07:00
|
|
|
trace_mm_page_free_batched(page, cold);
|
2012-01-10 16:07:04 -07:00
|
|
|
free_hot_cold_page(page, cold);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-03-22 01:08:05 -07:00
|
|
|
/*
|
|
|
|
* split_page takes a non-compound higher-order page, and splits it into
|
|
|
|
* n (1<<order) sub-pages: page[0..n]
|
|
|
|
* Each sub-page must be freed individually.
|
|
|
|
*
|
|
|
|
* Note: this is probably too low level an operation for use in drivers.
|
|
|
|
* Please consult with lkml before using this in your driver.
|
|
|
|
*/
|
|
|
|
void split_page(struct page *page, unsigned int order)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2014-01-23 16:52:54 -07:00
|
|
|
VM_BUG_ON_PAGE(PageCompound(page), page);
|
|
|
|
VM_BUG_ON_PAGE(!page_count(page), page);
|
2008-11-25 08:55:53 -07:00
|
|
|
|
|
|
|
#ifdef CONFIG_KMEMCHECK
|
|
|
|
/*
|
|
|
|
* Split shadow pages too, because free(page[0]) would
|
|
|
|
* otherwise free the whole shadow.
|
|
|
|
*/
|
|
|
|
if (kmemcheck_page_is_tracked(page))
|
|
|
|
split_page(virt_to_page(page[0].shadow), order);
|
|
|
|
#endif
|
|
|
|
|
2016-07-26 16:23:49 -06:00
|
|
|
for (i = 1; i < (1 << order); i++)
|
2006-03-22 01:08:40 -07:00
|
|
|
set_page_refcounted(page + i);
|
2016-07-26 16:23:49 -06:00
|
|
|
split_page_owner(page, order);
|
2006-03-22 01:08:05 -07:00
|
|
|
}
|
2013-03-25 16:47:38 -06:00
|
|
|
EXPORT_SYMBOL_GPL(split_page);
|
2006-03-22 01:08:05 -07:00
|
|
|
|
2014-11-13 16:19:21 -07:00
|
|
|
int __isolate_free_page(struct page *page, unsigned int order)
|
2010-05-24 15:32:27 -06:00
|
|
|
{
|
|
|
|
unsigned long watermark;
|
|
|
|
struct zone *zone;
|
2012-10-08 17:32:00 -06:00
|
|
|
int mt;
|
2010-05-24 15:32:27 -06:00
|
|
|
|
|
|
|
BUG_ON(!PageBuddy(page));
|
|
|
|
|
|
|
|
zone = page_zone(page);
|
2012-12-11 17:02:57 -07:00
|
|
|
mt = get_pageblock_migratetype(page);
|
2010-05-24 15:32:27 -06:00
|
|
|
|
2013-02-22 17:33:58 -07:00
|
|
|
if (!is_migrate_isolate(mt)) {
|
2012-12-11 17:02:57 -07:00
|
|
|
/* Obey watermarks as if the page was being allocated */
|
|
|
|
watermark = low_wmark_pages(zone) + (1 << order);
|
|
|
|
if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
|
|
|
|
return 0;
|
|
|
|
|
2013-01-11 15:32:16 -07:00
|
|
|
__mod_zone_freepage_state(zone, -(1UL << order), mt);
|
2012-12-11 17:02:57 -07:00
|
|
|
}
|
2010-05-24 15:32:27 -06:00
|
|
|
|
|
|
|
/* Remove page from free list */
|
|
|
|
list_del(&page->lru);
|
|
|
|
zone->free_area[order].nr_free--;
|
|
|
|
rmv_page_order(page);
|
2012-10-08 17:32:00 -06:00
|
|
|
|
2016-07-28 16:45:07 -06:00
|
|
|
/*
|
|
|
|
* Set the pageblock if the isolated page is at least half of a
|
|
|
|
* pageblock
|
|
|
|
*/
|
2010-05-24 15:32:27 -06:00
|
|
|
if (order >= pageblock_order - 1) {
|
|
|
|
struct page *endpage = page + (1 << order) - 1;
|
2011-12-29 05:09:50 -07:00
|
|
|
for (; page < endpage; page += pageblock_nr_pages) {
|
|
|
|
int mt = get_pageblock_migratetype(page);
|
2013-02-22 17:33:58 -07:00
|
|
|
if (!is_migrate_isolate(mt) && !is_migrate_cma(mt))
|
2011-12-29 05:09:50 -07:00
|
|
|
set_pageblock_migratetype(page,
|
|
|
|
MIGRATE_MOVABLE);
|
|
|
|
}
|
2010-05-24 15:32:27 -06:00
|
|
|
}
|
|
|
|
|
2015-07-17 17:24:15 -06:00
|
|
|
|
2013-01-11 15:32:16 -07:00
|
|
|
return 1UL << order;
|
2012-10-08 17:29:12 -06:00
|
|
|
}
|
|
|
|
|
2016-05-19 18:13:27 -06:00
|
|
|
/*
|
|
|
|
* Update NUMA hit/miss statistics
|
|
|
|
*
|
|
|
|
* Must be called with interrupts disabled.
|
|
|
|
*
|
|
|
|
* When __GFP_OTHER_NODE is set assume the node of the preferred
|
|
|
|
* zone is the local node. This is useful for daemons who allocate
|
|
|
|
* memory on behalf of other processes.
|
|
|
|
*/
|
|
|
|
static inline void zone_statistics(struct zone *preferred_zone, struct zone *z,
|
|
|
|
gfp_t flags)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
int local_nid = numa_node_id();
|
|
|
|
enum zone_stat_item local_stat = NUMA_LOCAL;
|
|
|
|
|
|
|
|
if (unlikely(flags & __GFP_OTHER_NODE)) {
|
|
|
|
local_stat = NUMA_OTHER;
|
|
|
|
local_nid = preferred_zone->node;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (z->node == local_nid) {
|
|
|
|
__inc_zone_state(z, NUMA_HIT);
|
|
|
|
__inc_zone_state(z, local_stat);
|
|
|
|
} else {
|
|
|
|
__inc_zone_state(z, NUMA_MISS);
|
|
|
|
__inc_zone_state(preferred_zone, NUMA_FOREIGN);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
mm: set page->pfmemalloc in prep_new_page()
The possibility of replacing the numerous parameters of alloc_pages*
functions with a single structure has been discussed when Minchan proposed
to expand the x86 kernel stack [1]. This series implements the change,
along with few more cleanups/microoptimizations.
The series is based on next-20150108 and I used gcc 4.8.3 20140627 on
openSUSE 13.2 for compiling. Config includess NUMA and COMPACTION.
The core change is the introduction of a new struct alloc_context, which looks
like this:
struct alloc_context {
struct zonelist *zonelist;
nodemask_t *nodemask;
struct zone *preferred_zone;
int classzone_idx;
int migratetype;
enum zone_type high_zoneidx;
};
All the contents is mostly constant, except that __alloc_pages_slowpath()
changes preferred_zone, classzone_idx and potentially zonelist. But
that's not a problem in case control returns to retry_cpuset: in
__alloc_pages_nodemask(), those will be reset to initial values again
(although it's a bit subtle). On the other hand, gfp_flags and alloc_info
mutate so much that it doesn't make sense to put them into alloc_context.
Still, the result is one parameter instead of up to 7. This is all in
Patch 2.
Patch 3 is a step to expand alloc_context usage out of page_alloc.c
itself. The function try_to_compact_pages() can also much benefit from
the parameter reduction, but it means the struct definition has to be
moved to a shared header.
Patch 1 should IMHO be included even if the rest is deemed not useful
enough. It improves maintainability and also has some code/stack
reduction. Patch 4 is OTOH a tiny optimization.
Overall bloat-o-meter results:
add/remove: 0/0 grow/shrink: 0/4 up/down: 0/-460 (-460)
function old new delta
nr_free_zone_pages 129 115 -14
__alloc_pages_direct_compact 329 256 -73
get_page_from_freelist 2670 2576 -94
__alloc_pages_nodemask 2564 2285 -279
try_to_compact_pages 582 579 -3
Overall stack sizes per ./scripts/checkstack.pl:
old new delta
get_page_from_freelist: 184 184 0
__alloc_pages_nodemask 248 200 -48
__alloc_pages_direct_c 40 - -40
try_to_compact_pages 72 72 0
-88
[1] http://marc.info/?l=linux-mm&m=140142462528257&w=2
This patch (of 4):
prep_new_page() sets almost everything in the struct page of the page
being allocated, except page->pfmemalloc. This is not obvious and has at
least once led to a bug where page->pfmemalloc was forgotten to be set
correctly, see commit 8fb74b9fb2b1 ("mm: compaction: partially revert
capture of suitable high-order page").
This patch moves the pfmemalloc setting to prep_new_page(), which means it
needs to gain alloc_flags parameter. The call to prep_new_page is moved
from buffered_rmqueue() to get_page_from_freelist(), which also leads to
simpler code. An obsolete comment for buffered_rmqueue() is replaced.
In addition to better maintainability there is a small reduction of code
and stack usage for get_page_from_freelist(), which inlines the other
functions involved.
add/remove: 0/0 grow/shrink: 0/1 up/down: 0/-145 (-145)
function old new delta
get_page_from_freelist 2670 2525 -145
Stack usage is reduced from 184 to 168 bytes.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.cz>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-11 16:25:38 -07:00
|
|
|
* Allocate a page from the given zone. Use pcplists for order-0 allocations.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2009-06-16 16:32:05 -06:00
|
|
|
static inline
|
|
|
|
struct page *buffered_rmqueue(struct zone *preferred_zone,
|
2014-06-04 17:10:21 -06:00
|
|
|
struct zone *zone, unsigned int order,
|
2016-05-19 18:13:38 -06:00
|
|
|
gfp_t gfp_flags, unsigned int alloc_flags,
|
|
|
|
int migratetype)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
2005-11-21 22:32:20 -07:00
|
|
|
struct page *page;
|
2014-06-04 17:10:22 -06:00
|
|
|
bool cold = ((gfp_flags & __GFP_COLD) != 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-01-08 02:00:42 -07:00
|
|
|
if (likely(order == 0)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
struct per_cpu_pages *pcp;
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
struct list_head *list;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
local_irq_save(flags);
|
2016-05-19 18:14:35 -06:00
|
|
|
do {
|
|
|
|
pcp = &this_cpu_ptr(zone->pageset)->pcp;
|
|
|
|
list = &pcp->lists[migratetype];
|
|
|
|
if (list_empty(list)) {
|
|
|
|
pcp->count += rmqueue_bulk(zone, 0,
|
|
|
|
pcp->batch, list,
|
|
|
|
migratetype, cold);
|
|
|
|
if (unlikely(list_empty(list)))
|
|
|
|
goto failed;
|
|
|
|
}
|
2007-10-16 02:25:50 -06:00
|
|
|
|
2016-05-19 18:14:35 -06:00
|
|
|
if (cold)
|
|
|
|
page = list_last_entry(list, struct page, lru);
|
|
|
|
else
|
|
|
|
page = list_first_entry(list, struct page, lru);
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
|
2016-06-03 15:55:52 -06:00
|
|
|
list_del(&page->lru);
|
|
|
|
pcp->count--;
|
|
|
|
|
|
|
|
} while (check_new_pcp(page));
|
2005-11-13 17:06:43 -07:00
|
|
|
} else {
|
2016-03-17 15:19:32 -06:00
|
|
|
/*
|
|
|
|
* We most definitely don't want callers attempting to
|
|
|
|
* allocate greater than order-1 page units with __GFP_NOFAIL.
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_lock_irqsave(&zone->lock, flags);
|
2015-11-06 17:28:37 -07:00
|
|
|
|
2016-05-19 18:14:35 -06:00
|
|
|
do {
|
|
|
|
page = NULL;
|
|
|
|
if (alloc_flags & ALLOC_HARDER) {
|
|
|
|
page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
|
|
|
|
if (page)
|
|
|
|
trace_mm_page_alloc_zone_locked(page, order, migratetype);
|
|
|
|
}
|
|
|
|
if (!page)
|
|
|
|
page = __rmqueue(zone, order, migratetype);
|
|
|
|
} while (page && check_new_pages(page, order));
|
2006-01-06 01:11:20 -07:00
|
|
|
spin_unlock(&zone->lock);
|
|
|
|
if (!page)
|
|
|
|
goto failed;
|
2012-10-08 17:32:02 -06:00
|
|
|
__mod_zone_freepage_state(zone, -(1 << order),
|
2015-09-08 16:01:25 -06:00
|
|
|
get_pcppage_migratetype(page));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2016-07-28 16:46:56 -06:00
|
|
|
__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
|
2011-03-22 17:33:12 -06:00
|
|
|
zone_statistics(preferred_zone, zone, gfp_flags);
|
2006-01-06 01:11:20 -07:00
|
|
|
local_irq_restore(flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2014-01-23 16:52:54 -07:00
|
|
|
VM_BUG_ON_PAGE(bad_range(zone, page), page);
|
2005-04-16 16:20:36 -06:00
|
|
|
return page;
|
2006-01-06 01:11:20 -07:00
|
|
|
|
|
|
|
failed:
|
|
|
|
local_irq_restore(flags);
|
|
|
|
return NULL;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-12-08 03:39:45 -07:00
|
|
|
#ifdef CONFIG_FAIL_PAGE_ALLOC
|
|
|
|
|
2011-07-26 17:09:03 -06:00
|
|
|
static struct {
|
2006-12-08 03:39:45 -07:00
|
|
|
struct fault_attr attr;
|
|
|
|
|
2015-09-26 16:04:07 -06:00
|
|
|
bool ignore_gfp_highmem;
|
2015-11-06 17:28:28 -07:00
|
|
|
bool ignore_gfp_reclaim;
|
2007-07-16 00:40:23 -06:00
|
|
|
u32 min_order;
|
2006-12-08 03:39:45 -07:00
|
|
|
} fail_page_alloc = {
|
|
|
|
.attr = FAULT_ATTR_INITIALIZER,
|
2015-11-06 17:28:28 -07:00
|
|
|
.ignore_gfp_reclaim = true,
|
2015-09-26 16:04:07 -06:00
|
|
|
.ignore_gfp_highmem = true,
|
2007-07-16 00:40:23 -06:00
|
|
|
.min_order = 1,
|
2006-12-08 03:39:45 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
static int __init setup_fail_page_alloc(char *str)
|
|
|
|
{
|
|
|
|
return setup_fault_attr(&fail_page_alloc.attr, str);
|
|
|
|
}
|
|
|
|
__setup("fail_page_alloc=", setup_fail_page_alloc);
|
|
|
|
|
2012-07-31 17:41:51 -06:00
|
|
|
static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
|
2006-12-08 03:39:45 -07:00
|
|
|
{
|
2007-07-16 00:40:23 -06:00
|
|
|
if (order < fail_page_alloc.min_order)
|
2012-07-31 17:41:51 -06:00
|
|
|
return false;
|
2006-12-08 03:39:45 -07:00
|
|
|
if (gfp_mask & __GFP_NOFAIL)
|
2012-07-31 17:41:51 -06:00
|
|
|
return false;
|
2006-12-08 03:39:45 -07:00
|
|
|
if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
|
2012-07-31 17:41:51 -06:00
|
|
|
return false;
|
2015-11-06 17:28:28 -07:00
|
|
|
if (fail_page_alloc.ignore_gfp_reclaim &&
|
|
|
|
(gfp_mask & __GFP_DIRECT_RECLAIM))
|
2012-07-31 17:41:51 -06:00
|
|
|
return false;
|
2006-12-08 03:39:45 -07:00
|
|
|
|
|
|
|
return should_fail(&fail_page_alloc.attr, 1 << order);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
|
|
|
|
|
|
|
|
static int __init fail_page_alloc_debugfs(void)
|
|
|
|
{
|
2011-07-24 02:33:43 -06:00
|
|
|
umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
|
2006-12-08 03:39:45 -07:00
|
|
|
struct dentry *dir;
|
|
|
|
|
2011-08-03 17:21:01 -06:00
|
|
|
dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
|
|
|
|
&fail_page_alloc.attr);
|
|
|
|
if (IS_ERR(dir))
|
|
|
|
return PTR_ERR(dir);
|
2006-12-08 03:39:45 -07:00
|
|
|
|
2011-07-26 17:09:03 -06:00
|
|
|
if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
|
2015-11-06 17:28:28 -07:00
|
|
|
&fail_page_alloc.ignore_gfp_reclaim))
|
2011-07-26 17:09:03 -06:00
|
|
|
goto fail;
|
|
|
|
if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
|
|
|
|
&fail_page_alloc.ignore_gfp_highmem))
|
|
|
|
goto fail;
|
|
|
|
if (!debugfs_create_u32("min-order", mode, dir,
|
|
|
|
&fail_page_alloc.min_order))
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
fail:
|
2011-08-03 17:21:01 -06:00
|
|
|
debugfs_remove_recursive(dir);
|
2006-12-08 03:39:45 -07:00
|
|
|
|
2011-07-26 17:09:03 -06:00
|
|
|
return -ENOMEM;
|
2006-12-08 03:39:45 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
late_initcall(fail_page_alloc_debugfs);
|
|
|
|
|
|
|
|
#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
|
|
|
|
|
|
|
|
#else /* CONFIG_FAIL_PAGE_ALLOC */
|
|
|
|
|
2012-07-31 17:41:51 -06:00
|
|
|
static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
|
2006-12-08 03:39:45 -07:00
|
|
|
{
|
2012-07-31 17:41:51 -06:00
|
|
|
return false;
|
2006-12-08 03:39:45 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#endif /* CONFIG_FAIL_PAGE_ALLOC */
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
2015-11-06 17:28:40 -07:00
|
|
|
* Return true if free base pages are above 'mark'. For high-order checks it
|
|
|
|
* will return true of the order-0 watermark is reached and there is at least
|
|
|
|
* one free page of a suitable size. Checking now avoids taking the zone lock
|
|
|
|
* to check in the allocation paths if no pages are free.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2016-05-20 17:57:12 -06:00
|
|
|
bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
|
|
|
|
int classzone_idx, unsigned int alloc_flags,
|
|
|
|
long free_pages)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-02-10 02:43:02 -07:00
|
|
|
long min = mark;
|
2005-04-16 16:20:36 -06:00
|
|
|
int o;
|
2016-05-19 18:13:38 -06:00
|
|
|
const bool alloc_harder = (alloc_flags & ALLOC_HARDER);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2015-11-06 17:28:37 -07:00
|
|
|
/* free_pages may go negative - that's OK */
|
2012-01-10 16:08:02 -07:00
|
|
|
free_pages -= (1 << order) - 1;
|
2015-11-06 17:28:37 -07:00
|
|
|
|
2005-11-13 17:06:43 -07:00
|
|
|
if (alloc_flags & ALLOC_HIGH)
|
2005-04-16 16:20:36 -06:00
|
|
|
min -= min / 2;
|
2015-11-06 17:28:37 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If the caller does not have rights to ALLOC_HARDER then subtract
|
|
|
|
* the high-atomic reserves. This will over-estimate the size of the
|
|
|
|
* atomic reserve but it avoids a search.
|
|
|
|
*/
|
2015-11-06 17:28:40 -07:00
|
|
|
if (likely(!alloc_harder))
|
2015-11-06 17:28:37 -07:00
|
|
|
free_pages -= z->nr_reserved_highatomic;
|
|
|
|
else
|
2005-04-16 16:20:36 -06:00
|
|
|
min -= min / 4;
|
2015-11-06 17:28:09 -07:00
|
|
|
|
2012-10-08 17:32:05 -06:00
|
|
|
#ifdef CONFIG_CMA
|
|
|
|
/* If allocation can't use CMA areas don't use free CMA pages */
|
|
|
|
if (!(alloc_flags & ALLOC_CMA))
|
2015-11-06 17:28:40 -07:00
|
|
|
free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
|
2012-10-08 17:32:05 -06:00
|
|
|
#endif
|
2013-06-12 15:05:02 -06:00
|
|
|
|
2015-11-06 17:28:40 -07:00
|
|
|
/*
|
|
|
|
* Check watermarks for an order-0 allocation request. If these
|
|
|
|
* are not met, then a high-order request also cannot go ahead
|
|
|
|
* even if a suitable page happened to be free.
|
|
|
|
*/
|
|
|
|
if (free_pages <= min + z->lowmem_reserve[classzone_idx])
|
mm: page allocator: adjust the per-cpu counter threshold when memory is low
Commit aa45484 ("calculate a better estimate of NR_FREE_PAGES when memory
is low") noted that watermarks were based on the vmstat NR_FREE_PAGES. To
avoid synchronization overhead, these counters are maintained on a per-cpu
basis and drained both periodically and when a threshold is above a
threshold. On large CPU systems, the difference between the estimate and
real value of NR_FREE_PAGES can be very high. The system can get into a
case where pages are allocated far below the min watermark potentially
causing livelock issues. The commit solved the problem by taking a better
reading of NR_FREE_PAGES when memory was low.
Unfortately, as reported by Shaohua Li this accurate reading can consume a
large amount of CPU time on systems with many sockets due to cache line
bouncing. This patch takes a different approach. For large machines
where counter drift might be unsafe and while kswapd is awake, the per-cpu
thresholds for the target pgdat are reduced to limit the level of drift to
what should be a safe level. This incurs a performance penalty in heavy
memory pressure by a factor that depends on the workload and the machine
but the machine should function correctly without accidentally exhausting
all memory on a node. There is an additional cost when kswapd wakes and
sleeps but the event is not expected to be frequent - in Shaohua's test
case, there was one recorded sleep and wake event at least.
To ensure that kswapd wakes up, a safe version of zone_watermark_ok() is
introduced that takes a more accurate reading of NR_FREE_PAGES when called
from wakeup_kswapd, when deciding whether it is really safe to go back to
sleep in sleeping_prematurely() and when deciding if a zone is really
balanced or not in balance_pgdat(). We are still using an expensive
function but limiting how often it is called.
When the test case is reproduced, the time spent in the watermark
functions is reduced. The following report is on the percentage of time
spent cumulatively spent in the functions zone_nr_free_pages(),
zone_watermark_ok(), __zone_watermark_ok(), zone_watermark_ok_safe(),
zone_page_state_snapshot(), zone_page_state().
vanilla 11.6615%
disable-threshold 0.2584%
David said:
: We had to pull aa454840 "mm: page allocator: calculate a better estimate
: of NR_FREE_PAGES when memory is low and kswapd is awake" from 2.6.36
: internally because tests showed that it would cause the machine to stall
: as the result of heavy kswapd activity. I merged it back with this fix as
: it is pending in the -mm tree and it solves the issue we were seeing, so I
: definitely think this should be pushed to -stable (and I would seriously
: consider it for 2.6.37 inclusion even at this late date).
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Reported-by: Shaohua Li <shaohua.li@intel.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Tested-by: Nicolas Bareil <nico@chdir.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: <stable@kernel.org> [2.6.37.1, 2.6.36.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 16:45:41 -07:00
|
|
|
return false;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2015-11-06 17:28:40 -07:00
|
|
|
/* If this is an order-0 request then the watermark is fine */
|
|
|
|
if (!order)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
/* For a high-order request, check at least one suitable page is free */
|
|
|
|
for (o = order; o < MAX_ORDER; o++) {
|
|
|
|
struct free_area *area = &z->free_area[o];
|
|
|
|
int mt;
|
|
|
|
|
|
|
|
if (!area->nr_free)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (alloc_harder)
|
|
|
|
return true;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2015-11-06 17:28:40 -07:00
|
|
|
for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
|
|
|
|
if (!list_empty(&area->free_list[mt]))
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_CMA
|
|
|
|
if ((alloc_flags & ALLOC_CMA) &&
|
|
|
|
!list_empty(&area->free_list[MIGRATE_CMA])) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2015-11-06 17:28:40 -07:00
|
|
|
return false;
|
mm: page allocator: adjust the per-cpu counter threshold when memory is low
Commit aa45484 ("calculate a better estimate of NR_FREE_PAGES when memory
is low") noted that watermarks were based on the vmstat NR_FREE_PAGES. To
avoid synchronization overhead, these counters are maintained on a per-cpu
basis and drained both periodically and when a threshold is above a
threshold. On large CPU systems, the difference between the estimate and
real value of NR_FREE_PAGES can be very high. The system can get into a
case where pages are allocated far below the min watermark potentially
causing livelock issues. The commit solved the problem by taking a better
reading of NR_FREE_PAGES when memory was low.
Unfortately, as reported by Shaohua Li this accurate reading can consume a
large amount of CPU time on systems with many sockets due to cache line
bouncing. This patch takes a different approach. For large machines
where counter drift might be unsafe and while kswapd is awake, the per-cpu
thresholds for the target pgdat are reduced to limit the level of drift to
what should be a safe level. This incurs a performance penalty in heavy
memory pressure by a factor that depends on the workload and the machine
but the machine should function correctly without accidentally exhausting
all memory on a node. There is an additional cost when kswapd wakes and
sleeps but the event is not expected to be frequent - in Shaohua's test
case, there was one recorded sleep and wake event at least.
To ensure that kswapd wakes up, a safe version of zone_watermark_ok() is
introduced that takes a more accurate reading of NR_FREE_PAGES when called
from wakeup_kswapd, when deciding whether it is really safe to go back to
sleep in sleeping_prematurely() and when deciding if a zone is really
balanced or not in balance_pgdat(). We are still using an expensive
function but limiting how often it is called.
When the test case is reproduced, the time spent in the watermark
functions is reduced. The following report is on the percentage of time
spent cumulatively spent in the functions zone_nr_free_pages(),
zone_watermark_ok(), __zone_watermark_ok(), zone_watermark_ok_safe(),
zone_page_state_snapshot(), zone_page_state().
vanilla 11.6615%
disable-threshold 0.2584%
David said:
: We had to pull aa454840 "mm: page allocator: calculate a better estimate
: of NR_FREE_PAGES when memory is low and kswapd is awake" from 2.6.36
: internally because tests showed that it would cause the machine to stall
: as the result of heavy kswapd activity. I merged it back with this fix as
: it is pending in the -mm tree and it solves the issue we were seeing, so I
: definitely think this should be pushed to -stable (and I would seriously
: consider it for 2.6.37 inclusion even at this late date).
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Reported-by: Shaohua Li <shaohua.li@intel.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Tested-by: Nicolas Bareil <nico@chdir.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: <stable@kernel.org> [2.6.37.1, 2.6.36.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 16:45:41 -07:00
|
|
|
}
|
|
|
|
|
2014-06-04 17:10:21 -06:00
|
|
|
bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
|
2016-05-19 18:13:38 -06:00
|
|
|
int classzone_idx, unsigned int alloc_flags)
|
mm: page allocator: adjust the per-cpu counter threshold when memory is low
Commit aa45484 ("calculate a better estimate of NR_FREE_PAGES when memory
is low") noted that watermarks were based on the vmstat NR_FREE_PAGES. To
avoid synchronization overhead, these counters are maintained on a per-cpu
basis and drained both periodically and when a threshold is above a
threshold. On large CPU systems, the difference between the estimate and
real value of NR_FREE_PAGES can be very high. The system can get into a
case where pages are allocated far below the min watermark potentially
causing livelock issues. The commit solved the problem by taking a better
reading of NR_FREE_PAGES when memory was low.
Unfortately, as reported by Shaohua Li this accurate reading can consume a
large amount of CPU time on systems with many sockets due to cache line
bouncing. This patch takes a different approach. For large machines
where counter drift might be unsafe and while kswapd is awake, the per-cpu
thresholds for the target pgdat are reduced to limit the level of drift to
what should be a safe level. This incurs a performance penalty in heavy
memory pressure by a factor that depends on the workload and the machine
but the machine should function correctly without accidentally exhausting
all memory on a node. There is an additional cost when kswapd wakes and
sleeps but the event is not expected to be frequent - in Shaohua's test
case, there was one recorded sleep and wake event at least.
To ensure that kswapd wakes up, a safe version of zone_watermark_ok() is
introduced that takes a more accurate reading of NR_FREE_PAGES when called
from wakeup_kswapd, when deciding whether it is really safe to go back to
sleep in sleeping_prematurely() and when deciding if a zone is really
balanced or not in balance_pgdat(). We are still using an expensive
function but limiting how often it is called.
When the test case is reproduced, the time spent in the watermark
functions is reduced. The following report is on the percentage of time
spent cumulatively spent in the functions zone_nr_free_pages(),
zone_watermark_ok(), __zone_watermark_ok(), zone_watermark_ok_safe(),
zone_page_state_snapshot(), zone_page_state().
vanilla 11.6615%
disable-threshold 0.2584%
David said:
: We had to pull aa454840 "mm: page allocator: calculate a better estimate
: of NR_FREE_PAGES when memory is low and kswapd is awake" from 2.6.36
: internally because tests showed that it would cause the machine to stall
: as the result of heavy kswapd activity. I merged it back with this fix as
: it is pending in the -mm tree and it solves the issue we were seeing, so I
: definitely think this should be pushed to -stable (and I would seriously
: consider it for 2.6.37 inclusion even at this late date).
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Reported-by: Shaohua Li <shaohua.li@intel.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Tested-by: Nicolas Bareil <nico@chdir.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: <stable@kernel.org> [2.6.37.1, 2.6.36.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 16:45:41 -07:00
|
|
|
{
|
|
|
|
return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
|
|
|
|
zone_page_state(z, NR_FREE_PAGES));
|
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:07 -06:00
|
|
|
static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
|
|
|
|
unsigned long mark, int classzone_idx, unsigned int alloc_flags)
|
|
|
|
{
|
|
|
|
long free_pages = zone_page_state(z, NR_FREE_PAGES);
|
|
|
|
long cma_pages = 0;
|
|
|
|
|
|
|
|
#ifdef CONFIG_CMA
|
|
|
|
/* If allocation can't use CMA areas don't use free CMA pages */
|
|
|
|
if (!(alloc_flags & ALLOC_CMA))
|
|
|
|
cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Fast check for order-0 only. If this fails then the reserves
|
|
|
|
* need to be calculated. There is a corner case where the check
|
|
|
|
* passes but only the high-order atomic reserve are free. If
|
|
|
|
* the caller is !atomic then it'll uselessly search the free
|
|
|
|
* list. That corner case is then slower but it is harmless.
|
|
|
|
*/
|
|
|
|
if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx])
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
|
|
|
|
free_pages);
|
|
|
|
}
|
|
|
|
|
2014-06-04 17:10:21 -06:00
|
|
|
bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
|
2015-11-06 17:28:09 -07:00
|
|
|
unsigned long mark, int classzone_idx)
|
mm: page allocator: adjust the per-cpu counter threshold when memory is low
Commit aa45484 ("calculate a better estimate of NR_FREE_PAGES when memory
is low") noted that watermarks were based on the vmstat NR_FREE_PAGES. To
avoid synchronization overhead, these counters are maintained on a per-cpu
basis and drained both periodically and when a threshold is above a
threshold. On large CPU systems, the difference between the estimate and
real value of NR_FREE_PAGES can be very high. The system can get into a
case where pages are allocated far below the min watermark potentially
causing livelock issues. The commit solved the problem by taking a better
reading of NR_FREE_PAGES when memory was low.
Unfortately, as reported by Shaohua Li this accurate reading can consume a
large amount of CPU time on systems with many sockets due to cache line
bouncing. This patch takes a different approach. For large machines
where counter drift might be unsafe and while kswapd is awake, the per-cpu
thresholds for the target pgdat are reduced to limit the level of drift to
what should be a safe level. This incurs a performance penalty in heavy
memory pressure by a factor that depends on the workload and the machine
but the machine should function correctly without accidentally exhausting
all memory on a node. There is an additional cost when kswapd wakes and
sleeps but the event is not expected to be frequent - in Shaohua's test
case, there was one recorded sleep and wake event at least.
To ensure that kswapd wakes up, a safe version of zone_watermark_ok() is
introduced that takes a more accurate reading of NR_FREE_PAGES when called
from wakeup_kswapd, when deciding whether it is really safe to go back to
sleep in sleeping_prematurely() and when deciding if a zone is really
balanced or not in balance_pgdat(). We are still using an expensive
function but limiting how often it is called.
When the test case is reproduced, the time spent in the watermark
functions is reduced. The following report is on the percentage of time
spent cumulatively spent in the functions zone_nr_free_pages(),
zone_watermark_ok(), __zone_watermark_ok(), zone_watermark_ok_safe(),
zone_page_state_snapshot(), zone_page_state().
vanilla 11.6615%
disable-threshold 0.2584%
David said:
: We had to pull aa454840 "mm: page allocator: calculate a better estimate
: of NR_FREE_PAGES when memory is low and kswapd is awake" from 2.6.36
: internally because tests showed that it would cause the machine to stall
: as the result of heavy kswapd activity. I merged it back with this fix as
: it is pending in the -mm tree and it solves the issue we were seeing, so I
: definitely think this should be pushed to -stable (and I would seriously
: consider it for 2.6.37 inclusion even at this late date).
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Reported-by: Shaohua Li <shaohua.li@intel.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Tested-by: Nicolas Bareil <nico@chdir.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: <stable@kernel.org> [2.6.37.1, 2.6.36.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 16:45:41 -07:00
|
|
|
{
|
|
|
|
long free_pages = zone_page_state(z, NR_FREE_PAGES);
|
|
|
|
|
|
|
|
if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
|
|
|
|
free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
|
|
|
|
|
2015-11-06 17:28:09 -07:00
|
|
|
return __zone_watermark_ok(z, order, mark, classzone_idx, 0,
|
mm: page allocator: adjust the per-cpu counter threshold when memory is low
Commit aa45484 ("calculate a better estimate of NR_FREE_PAGES when memory
is low") noted that watermarks were based on the vmstat NR_FREE_PAGES. To
avoid synchronization overhead, these counters are maintained on a per-cpu
basis and drained both periodically and when a threshold is above a
threshold. On large CPU systems, the difference between the estimate and
real value of NR_FREE_PAGES can be very high. The system can get into a
case where pages are allocated far below the min watermark potentially
causing livelock issues. The commit solved the problem by taking a better
reading of NR_FREE_PAGES when memory was low.
Unfortately, as reported by Shaohua Li this accurate reading can consume a
large amount of CPU time on systems with many sockets due to cache line
bouncing. This patch takes a different approach. For large machines
where counter drift might be unsafe and while kswapd is awake, the per-cpu
thresholds for the target pgdat are reduced to limit the level of drift to
what should be a safe level. This incurs a performance penalty in heavy
memory pressure by a factor that depends on the workload and the machine
but the machine should function correctly without accidentally exhausting
all memory on a node. There is an additional cost when kswapd wakes and
sleeps but the event is not expected to be frequent - in Shaohua's test
case, there was one recorded sleep and wake event at least.
To ensure that kswapd wakes up, a safe version of zone_watermark_ok() is
introduced that takes a more accurate reading of NR_FREE_PAGES when called
from wakeup_kswapd, when deciding whether it is really safe to go back to
sleep in sleeping_prematurely() and when deciding if a zone is really
balanced or not in balance_pgdat(). We are still using an expensive
function but limiting how often it is called.
When the test case is reproduced, the time spent in the watermark
functions is reduced. The following report is on the percentage of time
spent cumulatively spent in the functions zone_nr_free_pages(),
zone_watermark_ok(), __zone_watermark_ok(), zone_watermark_ok_safe(),
zone_page_state_snapshot(), zone_page_state().
vanilla 11.6615%
disable-threshold 0.2584%
David said:
: We had to pull aa454840 "mm: page allocator: calculate a better estimate
: of NR_FREE_PAGES when memory is low and kswapd is awake" from 2.6.36
: internally because tests showed that it would cause the machine to stall
: as the result of heavy kswapd activity. I merged it back with this fix as
: it is pending in the -mm tree and it solves the issue we were seeing, so I
: definitely think this should be pushed to -stable (and I would seriously
: consider it for 2.6.37 inclusion even at this late date).
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Reported-by: Shaohua Li <shaohua.li@intel.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Tested-by: Nicolas Bareil <nico@chdir.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: <stable@kernel.org> [2.6.37.1, 2.6.36.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 16:45:41 -07:00
|
|
|
free_pages);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
[PATCH] memory page_alloc zonelist caching speedup
Optimize the critical zonelist scanning for free pages in the kernel memory
allocator by caching the zones that were found to be full recently, and
skipping them.
Remembers the zones in a zonelist that were short of free memory in the
last second. And it stashes a zone-to-node table in the zonelist struct,
to optimize that conversion (minimize its cache footprint.)
Recent changes:
This differs in a significant way from a similar patch that I
posted a week ago. Now, instead of having a nodemask_t of
recently full nodes, I have a bitmask of recently full zones.
This solves a problem that last weeks patch had, which on
systems with multiple zones per node (such as DMA zone) would
take seeing any of these zones full as meaning that all zones
on that node were full.
Also I changed names - from "zonelist faster" to "zonelist cache",
as that seemed to better convey what we're doing here - caching
some of the key zonelist state (for faster access.)
See below for some performance benchmark results. After all that
discussion with David on why I didn't need them, I went and got
some ;). I wanted to verify that I had not hurt the normal case
of memory allocation noticeably. At least for my one little
microbenchmark, I found (1) the normal case wasn't affected, and
(2) workloads that forced scanning across multiple nodes for
memory improved up to 10% fewer System CPU cycles and lower
elapsed clock time ('sys' and 'real'). Good. See details, below.
I didn't have the logic in get_page_from_freelist() for various
full nodes and zone reclaim failures correct. That should be
fixed up now - notice the new goto labels zonelist_scan,
this_zone_full, and try_next_zone, in get_page_from_freelist().
There are two reasons I persued this alternative, over some earlier
proposals that would have focused on optimizing the fake numa
emulation case by caching the last useful zone:
1) Contrary to what I said before, we (SGI, on large ia64 sn2 systems)
have seen real customer loads where the cost to scan the zonelist
was a problem, due to many nodes being full of memory before
we got to a node we could use. Or at least, I think we have.
This was related to me by another engineer, based on experiences
from some time past. So this is not guaranteed. Most likely, though.
The following approach should help such real numa systems just as
much as it helps fake numa systems, or any combination thereof.
2) The effort to distinguish fake from real numa, using node_distance,
so that we could cache a fake numa node and optimize choosing
it over equivalent distance fake nodes, while continuing to
properly scan all real nodes in distance order, was going to
require a nasty blob of zonelist and node distance munging.
The following approach has no new dependency on node distances or
zone sorting.
See comment in the patch below for a description of what it actually does.
Technical details of note (or controversy):
- See the use of "zlc_active" and "did_zlc_setup" below, to delay
adding any work for this new mechanism until we've looked at the
first zone in zonelist. I figured the odds of the first zone
having the memory we needed were high enough that we should just
look there, first, then get fancy only if we need to keep looking.
- Some odd hackery was needed to add items to struct zonelist, while
not tripping up the custom zonelists built by the mm/mempolicy.c
code for MPOL_BIND. My usual wordy comments below explain this.
Search for "MPOL_BIND".
- Some per-node data in the struct zonelist is now modified frequently,
with no locking. Multiple CPU cores on a node could hit and mangle
this data. The theory is that this is just performance hint data,
and the memory allocator will work just fine despite any such mangling.
The fields at risk are the struct 'zonelist_cache' fields 'fullzones'
(a bitmask) and 'last_full_zap' (unsigned long jiffies). It should
all be self correcting after at most a one second delay.
- This still does a linear scan of the same lengths as before. All
I've optimized is making the scan faster, not algorithmically
shorter. It is now able to scan a compact array of 'unsigned
short' in the case of many full nodes, so one cache line should
cover quite a few nodes, rather than each node hitting another
one or two new and distinct cache lines.
- If both Andi and Nick don't find this too complicated, I will be
(pleasantly) flabbergasted.
- I removed the comment claiming we only use one cachline's worth of
zonelist. We seem, at least in the fake numa case, to have put the
lie to that claim.
- I pay no attention to the various watermarks and such in this performance
hint. A node could be marked full for one watermark, and then skipped
over when searching for a page using a different watermark. I think
that's actually quite ok, as it will tend to slightly increase the
spreading of memory over other nodes, away from a memory stressed node.
===============
Performance - some benchmark results and analysis:
This benchmark runs a memory hog program that uses multiple
threads to touch alot of memory as quickly as it can.
Multiple runs were made, touching 12, 38, 64 or 90 GBytes out of
the total 96 GBytes on the system, and using 1, 19, 37, or 55
threads (on a 56 CPU system.) System, user and real (elapsed)
timings were recorded for each run, shown in units of seconds,
in the table below.
Two kernels were tested - 2.6.18-mm3 and the same kernel with
this zonelist caching patch added. The table also shows the
percentage improvement the zonelist caching sys time is over
(lower than) the stock *-mm kernel.
number 2.6.18-mm3 zonelist-cache delta (< 0 good) percent
GBs N ------------ -------------- ---------------- systime
mem threads sys user real sys user real sys user real better
12 1 153 24 177 151 24 176 -2 0 -1 1%
12 19 99 22 8 99 22 8 0 0 0 0%
12 37 111 25 6 112 25 6 1 0 0 -0%
12 55 115 25 5 110 23 5 -5 -2 0 4%
38 1 502 74 576 497 73 570 -5 -1 -6 0%
38 19 426 78 48 373 76 39 -53 -2 -9 12%
38 37 544 83 36 547 82 36 3 -1 0 -0%
38 55 501 77 23 511 80 24 10 3 1 -1%
64 1 917 125 1042 890 124 1014 -27 -1 -28 2%
64 19 1118 138 119 965 141 103 -153 3 -16 13%
64 37 1202 151 94 1136 150 81 -66 -1 -13 5%
64 55 1118 141 61 1072 140 58 -46 -1 -3 4%
90 1 1342 177 1519 1275 174 1450 -67 -3 -69 4%
90 19 2392 199 192 2116 189 176 -276 -10 -16 11%
90 37 3313 238 175 2972 225 145 -341 -13 -30 10%
90 55 1948 210 104 1843 213 100 -105 3 -4 5%
Notes:
1) This test ran a memory hog program that started a specified number N of
threads, and had each thread allocate and touch 1/N'th of
the total memory to be used in the test run in a single loop,
writing a constant word to memory, one store every 4096 bytes.
Watching this test during some earlier trial runs, I would see
each of these threads sit down on one CPU and stay there, for
the remainder of the pass, a different CPU for each thread.
2) The 'real' column is not comparable to the 'sys' or 'user' columns.
The 'real' column is seconds wall clock time elapsed, from beginning
to end of that test pass. The 'sys' and 'user' columns are total
CPU seconds spent on that test pass. For a 19 thread test run,
for example, the sum of 'sys' and 'user' could be up to 19 times the
number of 'real' elapsed wall clock seconds.
3) Tests were run on a fresh, single-user boot, to minimize the amount
of memory already in use at the start of the test, and to minimize
the amount of background activity that might interfere.
4) Tests were done on a 56 CPU, 28 Node system with 96 GBytes of RAM.
5) Notice that the 'real' time gets large for the single thread runs, even
though the measured 'sys' and 'user' times are modest. I'm not sure what
that means - probably something to do with it being slow for one thread to
be accessing memory along ways away. Perhaps the fake numa system, running
ostensibly the same workload, would not show this substantial degradation
of 'real' time for one thread on many nodes -- lets hope not.
6) The high thread count passes (one thread per CPU - on 55 of 56 CPUs)
ran quite efficiently, as one might expect. Each pair of threads needed
to allocate and touch the memory on the node the two threads shared, a
pleasantly parallizable workload.
7) The intermediate thread count passes, when asking for alot of memory forcing
them to go to a few neighboring nodes, improved the most with this zonelist
caching patch.
Conclusions:
* This zonelist cache patch probably makes little difference one way or the
other for most workloads on real numa hardware, if those workloads avoid
heavy off node allocations.
* For memory intensive workloads requiring substantial off-node allocations
on real numa hardware, this patch improves both kernel and elapsed timings
up to ten per-cent.
* For fake numa systems, I'm optimistic, but will have to leave that up to
Rohit Seth to actually test (once I get him a 2.6.18 backport.)
Signed-off-by: Paul Jackson <pj@sgi.com>
Cc: Rohit Seth <rohitseth@google.com>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Cc: David Rientjes <rientjes@cs.washington.edu>
Cc: Paul Menage <menage@google.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-06 21:31:48 -07:00
|
|
|
#ifdef CONFIG_NUMA
|
2012-10-08 17:33:24 -06:00
|
|
|
static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
|
|
|
|
{
|
2014-06-04 17:07:15 -06:00
|
|
|
return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <
|
|
|
|
RECLAIM_DISTANCE;
|
2012-10-08 17:33:24 -06:00
|
|
|
}
|
[PATCH] memory page_alloc zonelist caching speedup
Optimize the critical zonelist scanning for free pages in the kernel memory
allocator by caching the zones that were found to be full recently, and
skipping them.
Remembers the zones in a zonelist that were short of free memory in the
last second. And it stashes a zone-to-node table in the zonelist struct,
to optimize that conversion (minimize its cache footprint.)
Recent changes:
This differs in a significant way from a similar patch that I
posted a week ago. Now, instead of having a nodemask_t of
recently full nodes, I have a bitmask of recently full zones.
This solves a problem that last weeks patch had, which on
systems with multiple zones per node (such as DMA zone) would
take seeing any of these zones full as meaning that all zones
on that node were full.
Also I changed names - from "zonelist faster" to "zonelist cache",
as that seemed to better convey what we're doing here - caching
some of the key zonelist state (for faster access.)
See below for some performance benchmark results. After all that
discussion with David on why I didn't need them, I went and got
some ;). I wanted to verify that I had not hurt the normal case
of memory allocation noticeably. At least for my one little
microbenchmark, I found (1) the normal case wasn't affected, and
(2) workloads that forced scanning across multiple nodes for
memory improved up to 10% fewer System CPU cycles and lower
elapsed clock time ('sys' and 'real'). Good. See details, below.
I didn't have the logic in get_page_from_freelist() for various
full nodes and zone reclaim failures correct. That should be
fixed up now - notice the new goto labels zonelist_scan,
this_zone_full, and try_next_zone, in get_page_from_freelist().
There are two reasons I persued this alternative, over some earlier
proposals that would have focused on optimizing the fake numa
emulation case by caching the last useful zone:
1) Contrary to what I said before, we (SGI, on large ia64 sn2 systems)
have seen real customer loads where the cost to scan the zonelist
was a problem, due to many nodes being full of memory before
we got to a node we could use. Or at least, I think we have.
This was related to me by another engineer, based on experiences
from some time past. So this is not guaranteed. Most likely, though.
The following approach should help such real numa systems just as
much as it helps fake numa systems, or any combination thereof.
2) The effort to distinguish fake from real numa, using node_distance,
so that we could cache a fake numa node and optimize choosing
it over equivalent distance fake nodes, while continuing to
properly scan all real nodes in distance order, was going to
require a nasty blob of zonelist and node distance munging.
The following approach has no new dependency on node distances or
zone sorting.
See comment in the patch below for a description of what it actually does.
Technical details of note (or controversy):
- See the use of "zlc_active" and "did_zlc_setup" below, to delay
adding any work for this new mechanism until we've looked at the
first zone in zonelist. I figured the odds of the first zone
having the memory we needed were high enough that we should just
look there, first, then get fancy only if we need to keep looking.
- Some odd hackery was needed to add items to struct zonelist, while
not tripping up the custom zonelists built by the mm/mempolicy.c
code for MPOL_BIND. My usual wordy comments below explain this.
Search for "MPOL_BIND".
- Some per-node data in the struct zonelist is now modified frequently,
with no locking. Multiple CPU cores on a node could hit and mangle
this data. The theory is that this is just performance hint data,
and the memory allocator will work just fine despite any such mangling.
The fields at risk are the struct 'zonelist_cache' fields 'fullzones'
(a bitmask) and 'last_full_zap' (unsigned long jiffies). It should
all be self correcting after at most a one second delay.
- This still does a linear scan of the same lengths as before. All
I've optimized is making the scan faster, not algorithmically
shorter. It is now able to scan a compact array of 'unsigned
short' in the case of many full nodes, so one cache line should
cover quite a few nodes, rather than each node hitting another
one or two new and distinct cache lines.
- If both Andi and Nick don't find this too complicated, I will be
(pleasantly) flabbergasted.
- I removed the comment claiming we only use one cachline's worth of
zonelist. We seem, at least in the fake numa case, to have put the
lie to that claim.
- I pay no attention to the various watermarks and such in this performance
hint. A node could be marked full for one watermark, and then skipped
over when searching for a page using a different watermark. I think
that's actually quite ok, as it will tend to slightly increase the
spreading of memory over other nodes, away from a memory stressed node.
===============
Performance - some benchmark results and analysis:
This benchmark runs a memory hog program that uses multiple
threads to touch alot of memory as quickly as it can.
Multiple runs were made, touching 12, 38, 64 or 90 GBytes out of
the total 96 GBytes on the system, and using 1, 19, 37, or 55
threads (on a 56 CPU system.) System, user and real (elapsed)
timings were recorded for each run, shown in units of seconds,
in the table below.
Two kernels were tested - 2.6.18-mm3 and the same kernel with
this zonelist caching patch added. The table also shows the
percentage improvement the zonelist caching sys time is over
(lower than) the stock *-mm kernel.
number 2.6.18-mm3 zonelist-cache delta (< 0 good) percent
GBs N ------------ -------------- ---------------- systime
mem threads sys user real sys user real sys user real better
12 1 153 24 177 151 24 176 -2 0 -1 1%
12 19 99 22 8 99 22 8 0 0 0 0%
12 37 111 25 6 112 25 6 1 0 0 -0%
12 55 115 25 5 110 23 5 -5 -2 0 4%
38 1 502 74 576 497 73 570 -5 -1 -6 0%
38 19 426 78 48 373 76 39 -53 -2 -9 12%
38 37 544 83 36 547 82 36 3 -1 0 -0%
38 55 501 77 23 511 80 24 10 3 1 -1%
64 1 917 125 1042 890 124 1014 -27 -1 -28 2%
64 19 1118 138 119 965 141 103 -153 3 -16 13%
64 37 1202 151 94 1136 150 81 -66 -1 -13 5%
64 55 1118 141 61 1072 140 58 -46 -1 -3 4%
90 1 1342 177 1519 1275 174 1450 -67 -3 -69 4%
90 19 2392 199 192 2116 189 176 -276 -10 -16 11%
90 37 3313 238 175 2972 225 145 -341 -13 -30 10%
90 55 1948 210 104 1843 213 100 -105 3 -4 5%
Notes:
1) This test ran a memory hog program that started a specified number N of
threads, and had each thread allocate and touch 1/N'th of
the total memory to be used in the test run in a single loop,
writing a constant word to memory, one store every 4096 bytes.
Watching this test during some earlier trial runs, I would see
each of these threads sit down on one CPU and stay there, for
the remainder of the pass, a different CPU for each thread.
2) The 'real' column is not comparable to the 'sys' or 'user' columns.
The 'real' column is seconds wall clock time elapsed, from beginning
to end of that test pass. The 'sys' and 'user' columns are total
CPU seconds spent on that test pass. For a 19 thread test run,
for example, the sum of 'sys' and 'user' could be up to 19 times the
number of 'real' elapsed wall clock seconds.
3) Tests were run on a fresh, single-user boot, to minimize the amount
of memory already in use at the start of the test, and to minimize
the amount of background activity that might interfere.
4) Tests were done on a 56 CPU, 28 Node system with 96 GBytes of RAM.
5) Notice that the 'real' time gets large for the single thread runs, even
though the measured 'sys' and 'user' times are modest. I'm not sure what
that means - probably something to do with it being slow for one thread to
be accessing memory along ways away. Perhaps the fake numa system, running
ostensibly the same workload, would not show this substantial degradation
of 'real' time for one thread on many nodes -- lets hope not.
6) The high thread count passes (one thread per CPU - on 55 of 56 CPUs)
ran quite efficiently, as one might expect. Each pair of threads needed
to allocate and touch the memory on the node the two threads shared, a
pleasantly parallizable workload.
7) The intermediate thread count passes, when asking for alot of memory forcing
them to go to a few neighboring nodes, improved the most with this zonelist
caching patch.
Conclusions:
* This zonelist cache patch probably makes little difference one way or the
other for most workloads on real numa hardware, if those workloads avoid
heavy off node allocations.
* For memory intensive workloads requiring substantial off-node allocations
on real numa hardware, this patch improves both kernel and elapsed timings
up to ten per-cent.
* For fake numa systems, I'm optimistic, but will have to leave that up to
Rohit Seth to actually test (once I get him a 2.6.18 backport.)
Signed-off-by: Paul Jackson <pj@sgi.com>
Cc: Rohit Seth <rohitseth@google.com>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Cc: David Rientjes <rientjes@cs.washington.edu>
Cc: Paul Menage <menage@google.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-06 21:31:48 -07:00
|
|
|
#else /* CONFIG_NUMA */
|
2012-10-08 17:33:24 -06:00
|
|
|
static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
[PATCH] memory page_alloc zonelist caching speedup
Optimize the critical zonelist scanning for free pages in the kernel memory
allocator by caching the zones that were found to be full recently, and
skipping them.
Remembers the zones in a zonelist that were short of free memory in the
last second. And it stashes a zone-to-node table in the zonelist struct,
to optimize that conversion (minimize its cache footprint.)
Recent changes:
This differs in a significant way from a similar patch that I
posted a week ago. Now, instead of having a nodemask_t of
recently full nodes, I have a bitmask of recently full zones.
This solves a problem that last weeks patch had, which on
systems with multiple zones per node (such as DMA zone) would
take seeing any of these zones full as meaning that all zones
on that node were full.
Also I changed names - from "zonelist faster" to "zonelist cache",
as that seemed to better convey what we're doing here - caching
some of the key zonelist state (for faster access.)
See below for some performance benchmark results. After all that
discussion with David on why I didn't need them, I went and got
some ;). I wanted to verify that I had not hurt the normal case
of memory allocation noticeably. At least for my one little
microbenchmark, I found (1) the normal case wasn't affected, and
(2) workloads that forced scanning across multiple nodes for
memory improved up to 10% fewer System CPU cycles and lower
elapsed clock time ('sys' and 'real'). Good. See details, below.
I didn't have the logic in get_page_from_freelist() for various
full nodes and zone reclaim failures correct. That should be
fixed up now - notice the new goto labels zonelist_scan,
this_zone_full, and try_next_zone, in get_page_from_freelist().
There are two reasons I persued this alternative, over some earlier
proposals that would have focused on optimizing the fake numa
emulation case by caching the last useful zone:
1) Contrary to what I said before, we (SGI, on large ia64 sn2 systems)
have seen real customer loads where the cost to scan the zonelist
was a problem, due to many nodes being full of memory before
we got to a node we could use. Or at least, I think we have.
This was related to me by another engineer, based on experiences
from some time past. So this is not guaranteed. Most likely, though.
The following approach should help such real numa systems just as
much as it helps fake numa systems, or any combination thereof.
2) The effort to distinguish fake from real numa, using node_distance,
so that we could cache a fake numa node and optimize choosing
it over equivalent distance fake nodes, while continuing to
properly scan all real nodes in distance order, was going to
require a nasty blob of zonelist and node distance munging.
The following approach has no new dependency on node distances or
zone sorting.
See comment in the patch below for a description of what it actually does.
Technical details of note (or controversy):
- See the use of "zlc_active" and "did_zlc_setup" below, to delay
adding any work for this new mechanism until we've looked at the
first zone in zonelist. I figured the odds of the first zone
having the memory we needed were high enough that we should just
look there, first, then get fancy only if we need to keep looking.
- Some odd hackery was needed to add items to struct zonelist, while
not tripping up the custom zonelists built by the mm/mempolicy.c
code for MPOL_BIND. My usual wordy comments below explain this.
Search for "MPOL_BIND".
- Some per-node data in the struct zonelist is now modified frequently,
with no locking. Multiple CPU cores on a node could hit and mangle
this data. The theory is that this is just performance hint data,
and the memory allocator will work just fine despite any such mangling.
The fields at risk are the struct 'zonelist_cache' fields 'fullzones'
(a bitmask) and 'last_full_zap' (unsigned long jiffies). It should
all be self correcting after at most a one second delay.
- This still does a linear scan of the same lengths as before. All
I've optimized is making the scan faster, not algorithmically
shorter. It is now able to scan a compact array of 'unsigned
short' in the case of many full nodes, so one cache line should
cover quite a few nodes, rather than each node hitting another
one or two new and distinct cache lines.
- If both Andi and Nick don't find this too complicated, I will be
(pleasantly) flabbergasted.
- I removed the comment claiming we only use one cachline's worth of
zonelist. We seem, at least in the fake numa case, to have put the
lie to that claim.
- I pay no attention to the various watermarks and such in this performance
hint. A node could be marked full for one watermark, and then skipped
over when searching for a page using a different watermark. I think
that's actually quite ok, as it will tend to slightly increase the
spreading of memory over other nodes, away from a memory stressed node.
===============
Performance - some benchmark results and analysis:
This benchmark runs a memory hog program that uses multiple
threads to touch alot of memory as quickly as it can.
Multiple runs were made, touching 12, 38, 64 or 90 GBytes out of
the total 96 GBytes on the system, and using 1, 19, 37, or 55
threads (on a 56 CPU system.) System, user and real (elapsed)
timings were recorded for each run, shown in units of seconds,
in the table below.
Two kernels were tested - 2.6.18-mm3 and the same kernel with
this zonelist caching patch added. The table also shows the
percentage improvement the zonelist caching sys time is over
(lower than) the stock *-mm kernel.
number 2.6.18-mm3 zonelist-cache delta (< 0 good) percent
GBs N ------------ -------------- ---------------- systime
mem threads sys user real sys user real sys user real better
12 1 153 24 177 151 24 176 -2 0 -1 1%
12 19 99 22 8 99 22 8 0 0 0 0%
12 37 111 25 6 112 25 6 1 0 0 -0%
12 55 115 25 5 110 23 5 -5 -2 0 4%
38 1 502 74 576 497 73 570 -5 -1 -6 0%
38 19 426 78 48 373 76 39 -53 -2 -9 12%
38 37 544 83 36 547 82 36 3 -1 0 -0%
38 55 501 77 23 511 80 24 10 3 1 -1%
64 1 917 125 1042 890 124 1014 -27 -1 -28 2%
64 19 1118 138 119 965 141 103 -153 3 -16 13%
64 37 1202 151 94 1136 150 81 -66 -1 -13 5%
64 55 1118 141 61 1072 140 58 -46 -1 -3 4%
90 1 1342 177 1519 1275 174 1450 -67 -3 -69 4%
90 19 2392 199 192 2116 189 176 -276 -10 -16 11%
90 37 3313 238 175 2972 225 145 -341 -13 -30 10%
90 55 1948 210 104 1843 213 100 -105 3 -4 5%
Notes:
1) This test ran a memory hog program that started a specified number N of
threads, and had each thread allocate and touch 1/N'th of
the total memory to be used in the test run in a single loop,
writing a constant word to memory, one store every 4096 bytes.
Watching this test during some earlier trial runs, I would see
each of these threads sit down on one CPU and stay there, for
the remainder of the pass, a different CPU for each thread.
2) The 'real' column is not comparable to the 'sys' or 'user' columns.
The 'real' column is seconds wall clock time elapsed, from beginning
to end of that test pass. The 'sys' and 'user' columns are total
CPU seconds spent on that test pass. For a 19 thread test run,
for example, the sum of 'sys' and 'user' could be up to 19 times the
number of 'real' elapsed wall clock seconds.
3) Tests were run on a fresh, single-user boot, to minimize the amount
of memory already in use at the start of the test, and to minimize
the amount of background activity that might interfere.
4) Tests were done on a 56 CPU, 28 Node system with 96 GBytes of RAM.
5) Notice that the 'real' time gets large for the single thread runs, even
though the measured 'sys' and 'user' times are modest. I'm not sure what
that means - probably something to do with it being slow for one thread to
be accessing memory along ways away. Perhaps the fake numa system, running
ostensibly the same workload, would not show this substantial degradation
of 'real' time for one thread on many nodes -- lets hope not.
6) The high thread count passes (one thread per CPU - on 55 of 56 CPUs)
ran quite efficiently, as one might expect. Each pair of threads needed
to allocate and touch the memory on the node the two threads shared, a
pleasantly parallizable workload.
7) The intermediate thread count passes, when asking for alot of memory forcing
them to go to a few neighboring nodes, improved the most with this zonelist
caching patch.
Conclusions:
* This zonelist cache patch probably makes little difference one way or the
other for most workloads on real numa hardware, if those workloads avoid
heavy off node allocations.
* For memory intensive workloads requiring substantial off-node allocations
on real numa hardware, this patch improves both kernel and elapsed timings
up to ten per-cent.
* For fake numa systems, I'm optimistic, but will have to leave that up to
Rohit Seth to actually test (once I get him a 2.6.18 backport.)
Signed-off-by: Paul Jackson <pj@sgi.com>
Cc: Rohit Seth <rohitseth@google.com>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Cc: David Rientjes <rientjes@cs.washington.edu>
Cc: Paul Menage <menage@google.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-06 21:31:48 -07:00
|
|
|
#endif /* CONFIG_NUMA */
|
|
|
|
|
2005-11-13 17:06:43 -07:00
|
|
|
/*
|
2006-12-06 21:31:38 -07:00
|
|
|
* get_page_from_freelist goes through the zonelist trying to allocate
|
2005-11-13 17:06:43 -07:00
|
|
|
* a page.
|
|
|
|
*/
|
|
|
|
static struct page *
|
2015-02-11 16:25:41 -07:00
|
|
|
get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
|
|
|
|
const struct alloc_context *ac)
|
2005-06-21 18:14:41 -06:00
|
|
|
{
|
2016-05-19 18:14:10 -06:00
|
|
|
struct zoneref *z = ac->preferred_zoneref;
|
2009-06-16 16:31:59 -06:00
|
|
|
struct zone *zone;
|
2016-07-28 16:46:53 -06:00
|
|
|
struct pglist_data *last_pgdat_dirty_limit = NULL;
|
|
|
|
|
2005-11-13 17:06:43 -07:00
|
|
|
/*
|
[PATCH] memory page_alloc zonelist caching speedup
Optimize the critical zonelist scanning for free pages in the kernel memory
allocator by caching the zones that were found to be full recently, and
skipping them.
Remembers the zones in a zonelist that were short of free memory in the
last second. And it stashes a zone-to-node table in the zonelist struct,
to optimize that conversion (minimize its cache footprint.)
Recent changes:
This differs in a significant way from a similar patch that I
posted a week ago. Now, instead of having a nodemask_t of
recently full nodes, I have a bitmask of recently full zones.
This solves a problem that last weeks patch had, which on
systems with multiple zones per node (such as DMA zone) would
take seeing any of these zones full as meaning that all zones
on that node were full.
Also I changed names - from "zonelist faster" to "zonelist cache",
as that seemed to better convey what we're doing here - caching
some of the key zonelist state (for faster access.)
See below for some performance benchmark results. After all that
discussion with David on why I didn't need them, I went and got
some ;). I wanted to verify that I had not hurt the normal case
of memory allocation noticeably. At least for my one little
microbenchmark, I found (1) the normal case wasn't affected, and
(2) workloads that forced scanning across multiple nodes for
memory improved up to 10% fewer System CPU cycles and lower
elapsed clock time ('sys' and 'real'). Good. See details, below.
I didn't have the logic in get_page_from_freelist() for various
full nodes and zone reclaim failures correct. That should be
fixed up now - notice the new goto labels zonelist_scan,
this_zone_full, and try_next_zone, in get_page_from_freelist().
There are two reasons I persued this alternative, over some earlier
proposals that would have focused on optimizing the fake numa
emulation case by caching the last useful zone:
1) Contrary to what I said before, we (SGI, on large ia64 sn2 systems)
have seen real customer loads where the cost to scan the zonelist
was a problem, due to many nodes being full of memory before
we got to a node we could use. Or at least, I think we have.
This was related to me by another engineer, based on experiences
from some time past. So this is not guaranteed. Most likely, though.
The following approach should help such real numa systems just as
much as it helps fake numa systems, or any combination thereof.
2) The effort to distinguish fake from real numa, using node_distance,
so that we could cache a fake numa node and optimize choosing
it over equivalent distance fake nodes, while continuing to
properly scan all real nodes in distance order, was going to
require a nasty blob of zonelist and node distance munging.
The following approach has no new dependency on node distances or
zone sorting.
See comment in the patch below for a description of what it actually does.
Technical details of note (or controversy):
- See the use of "zlc_active" and "did_zlc_setup" below, to delay
adding any work for this new mechanism until we've looked at the
first zone in zonelist. I figured the odds of the first zone
having the memory we needed were high enough that we should just
look there, first, then get fancy only if we need to keep looking.
- Some odd hackery was needed to add items to struct zonelist, while
not tripping up the custom zonelists built by the mm/mempolicy.c
code for MPOL_BIND. My usual wordy comments below explain this.
Search for "MPOL_BIND".
- Some per-node data in the struct zonelist is now modified frequently,
with no locking. Multiple CPU cores on a node could hit and mangle
this data. The theory is that this is just performance hint data,
and the memory allocator will work just fine despite any such mangling.
The fields at risk are the struct 'zonelist_cache' fields 'fullzones'
(a bitmask) and 'last_full_zap' (unsigned long jiffies). It should
all be self correcting after at most a one second delay.
- This still does a linear scan of the same lengths as before. All
I've optimized is making the scan faster, not algorithmically
shorter. It is now able to scan a compact array of 'unsigned
short' in the case of many full nodes, so one cache line should
cover quite a few nodes, rather than each node hitting another
one or two new and distinct cache lines.
- If both Andi and Nick don't find this too complicated, I will be
(pleasantly) flabbergasted.
- I removed the comment claiming we only use one cachline's worth of
zonelist. We seem, at least in the fake numa case, to have put the
lie to that claim.
- I pay no attention to the various watermarks and such in this performance
hint. A node could be marked full for one watermark, and then skipped
over when searching for a page using a different watermark. I think
that's actually quite ok, as it will tend to slightly increase the
spreading of memory over other nodes, away from a memory stressed node.
===============
Performance - some benchmark results and analysis:
This benchmark runs a memory hog program that uses multiple
threads to touch alot of memory as quickly as it can.
Multiple runs were made, touching 12, 38, 64 or 90 GBytes out of
the total 96 GBytes on the system, and using 1, 19, 37, or 55
threads (on a 56 CPU system.) System, user and real (elapsed)
timings were recorded for each run, shown in units of seconds,
in the table below.
Two kernels were tested - 2.6.18-mm3 and the same kernel with
this zonelist caching patch added. The table also shows the
percentage improvement the zonelist caching sys time is over
(lower than) the stock *-mm kernel.
number 2.6.18-mm3 zonelist-cache delta (< 0 good) percent
GBs N ------------ -------------- ---------------- systime
mem threads sys user real sys user real sys user real better
12 1 153 24 177 151 24 176 -2 0 -1 1%
12 19 99 22 8 99 22 8 0 0 0 0%
12 37 111 25 6 112 25 6 1 0 0 -0%
12 55 115 25 5 110 23 5 -5 -2 0 4%
38 1 502 74 576 497 73 570 -5 -1 -6 0%
38 19 426 78 48 373 76 39 -53 -2 -9 12%
38 37 544 83 36 547 82 36 3 -1 0 -0%
38 55 501 77 23 511 80 24 10 3 1 -1%
64 1 917 125 1042 890 124 1014 -27 -1 -28 2%
64 19 1118 138 119 965 141 103 -153 3 -16 13%
64 37 1202 151 94 1136 150 81 -66 -1 -13 5%
64 55 1118 141 61 1072 140 58 -46 -1 -3 4%
90 1 1342 177 1519 1275 174 1450 -67 -3 -69 4%
90 19 2392 199 192 2116 189 176 -276 -10 -16 11%
90 37 3313 238 175 2972 225 145 -341 -13 -30 10%
90 55 1948 210 104 1843 213 100 -105 3 -4 5%
Notes:
1) This test ran a memory hog program that started a specified number N of
threads, and had each thread allocate and touch 1/N'th of
the total memory to be used in the test run in a single loop,
writing a constant word to memory, one store every 4096 bytes.
Watching this test during some earlier trial runs, I would see
each of these threads sit down on one CPU and stay there, for
the remainder of the pass, a different CPU for each thread.
2) The 'real' column is not comparable to the 'sys' or 'user' columns.
The 'real' column is seconds wall clock time elapsed, from beginning
to end of that test pass. The 'sys' and 'user' columns are total
CPU seconds spent on that test pass. For a 19 thread test run,
for example, the sum of 'sys' and 'user' could be up to 19 times the
number of 'real' elapsed wall clock seconds.
3) Tests were run on a fresh, single-user boot, to minimize the amount
of memory already in use at the start of the test, and to minimize
the amount of background activity that might interfere.
4) Tests were done on a 56 CPU, 28 Node system with 96 GBytes of RAM.
5) Notice that the 'real' time gets large for the single thread runs, even
though the measured 'sys' and 'user' times are modest. I'm not sure what
that means - probably something to do with it being slow for one thread to
be accessing memory along ways away. Perhaps the fake numa system, running
ostensibly the same workload, would not show this substantial degradation
of 'real' time for one thread on many nodes -- lets hope not.
6) The high thread count passes (one thread per CPU - on 55 of 56 CPUs)
ran quite efficiently, as one might expect. Each pair of threads needed
to allocate and touch the memory on the node the two threads shared, a
pleasantly parallizable workload.
7) The intermediate thread count passes, when asking for alot of memory forcing
them to go to a few neighboring nodes, improved the most with this zonelist
caching patch.
Conclusions:
* This zonelist cache patch probably makes little difference one way or the
other for most workloads on real numa hardware, if those workloads avoid
heavy off node allocations.
* For memory intensive workloads requiring substantial off-node allocations
on real numa hardware, this patch improves both kernel and elapsed timings
up to ten per-cent.
* For fake numa systems, I'm optimistic, but will have to leave that up to
Rohit Seth to actually test (once I get him a 2.6.18 backport.)
Signed-off-by: Paul Jackson <pj@sgi.com>
Cc: Rohit Seth <rohitseth@google.com>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Cc: David Rientjes <rientjes@cs.washington.edu>
Cc: Paul Menage <menage@google.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-06 21:31:48 -07:00
|
|
|
* Scan zonelist, looking for a zone with enough free.
|
2014-10-20 05:50:30 -06:00
|
|
|
* See also __cpuset_node_allowed() comment in kernel/cpuset.c.
|
2005-11-13 17:06:43 -07:00
|
|
|
*/
|
2016-05-19 18:14:10 -06:00
|
|
|
for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
|
2015-02-11 16:25:41 -07:00
|
|
|
ac->nodemask) {
|
2016-05-19 18:13:47 -06:00
|
|
|
struct page *page;
|
2013-09-11 15:20:46 -06:00
|
|
|
unsigned long mark;
|
|
|
|
|
2014-06-04 17:10:08 -06:00
|
|
|
if (cpusets_enabled() &&
|
|
|
|
(alloc_flags & ALLOC_CPUSET) &&
|
2016-05-19 18:14:30 -06:00
|
|
|
!__cpuset_zone_allowed(zone, gfp_mask))
|
mm: page allocator: initialise ZLC for first zone eligible for zone_reclaim
There have been a small number of complaints about significant stalls
while copying large amounts of data on NUMA machines reported on a
distribution bugzilla. In these cases, zone_reclaim was enabled by
default due to large NUMA distances. In general, the complaints have not
been about the workload itself unless it was a file server (in which case
the recommendation was disable zone_reclaim).
The stalls are mostly due to significant amounts of time spent scanning
the preferred zone for pages to free. After a failure, it might fallback
to another node (as zonelists are often node-ordered rather than
zone-ordered) but stall quickly again when the next allocation attempt
occurs. In bad cases, each page allocated results in a full scan of the
preferred zone.
Patch 1 checks the preferred zone for recent allocation failure
which is particularly important if zone_reclaim has failed
recently. This avoids rescanning the zone in the near future and
instead falling back to another node. This may hurt node locality
in some cases but a failure to zone_reclaim is more expensive than
a remote access.
Patch 2 clears the zlc information after direct reclaim.
Otherwise, zone_reclaim can mark zones full, direct reclaim can
reclaim enough pages but the zone is still not considered for
allocation.
This was tested on a 24-thread 2-node x86_64 machine. The tests were
focused on large amounts of IO. All tests were bound to the CPUs on
node-0 to avoid disturbances due to processes being scheduled on different
nodes. The kernels tested are
3.0-rc6-vanilla Vanilla 3.0-rc6
zlcfirst Patch 1 applied
zlcreconsider Patches 1+2 applied
FS-Mark
./fs_mark -d /tmp/fsmark-10813 -D 100 -N 5000 -n 208 -L 35 -t 24 -S0 -s 524288
fsmark-3.0-rc6 3.0-rc6 3.0-rc6
vanilla zlcfirs zlcreconsider
Files/s min 54.90 ( 0.00%) 49.80 (-10.24%) 49.10 (-11.81%)
Files/s mean 100.11 ( 0.00%) 135.17 (25.94%) 146.93 (31.87%)
Files/s stddev 57.51 ( 0.00%) 138.97 (58.62%) 158.69 (63.76%)
Files/s max 361.10 ( 0.00%) 834.40 (56.72%) 802.40 (55.00%)
Overhead min 76704.00 ( 0.00%) 76501.00 ( 0.27%) 77784.00 (-1.39%)
Overhead mean 1485356.51 ( 0.00%) 1035797.83 (43.40%) 1594680.26 (-6.86%)
Overhead stddev 1848122.53 ( 0.00%) 881489.88 (109.66%) 1772354.90 ( 4.27%)
Overhead max 7989060.00 ( 0.00%) 3369118.00 (137.13%) 10135324.00 (-21.18%)
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 501.49 493.91 499.93
Total Elapsed Time (seconds) 2451.57 2257.48 2215.92
MMTests Statistics: vmstat
Page Ins 46268 63840 66008
Page Outs 90821596 90671128 88043732
Swap Ins 0 0 0
Swap Outs 0 0 0
Direct pages scanned 13091697 8966863 8971790
Kswapd pages scanned 0 1830011 1831116
Kswapd pages reclaimed 0 1829068 1829930
Direct pages reclaimed 13037777 8956828 8648314
Kswapd efficiency 100% 99% 99%
Kswapd velocity 0.000 810.643 826.346
Direct efficiency 99% 99% 96%
Direct velocity 5340.128 3972.068 4048.788
Percentage direct scans 100% 83% 83%
Page writes by reclaim 0 3 0
Slabs scanned 796672 720640 720256
Direct inode steals 7422667 7160012 7088638
Kswapd inode steals 0 1736840 2021238
Test completes far faster with a large increase in the number of files
created per second. Standard deviation is high as a small number of
iterations were much higher than the mean. The number of pages scanned by
zone_reclaim is reduced and kswapd is used for more work.
LARGE DD
3.0-rc6 3.0-rc6 3.0-rc6
vanilla zlcfirst zlcreconsider
download tar 59 ( 0.00%) 59 ( 0.00%) 55 ( 7.27%)
dd source files 527 ( 0.00%) 296 (78.04%) 320 (64.69%)
delete source 36 ( 0.00%) 19 (89.47%) 20 (80.00%)
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 125.03 118.98 122.01
Total Elapsed Time (seconds) 624.56 375.02 398.06
MMTests Statistics: vmstat
Page Ins 3594216 439368 407032
Page Outs 23380832 23380488 23377444
Swap Ins 0 0 0
Swap Outs 0 436 287
Direct pages scanned 17482342 69315973 82864918
Kswapd pages scanned 0 519123 575425
Kswapd pages reclaimed 0 466501 522487
Direct pages reclaimed 5858054 2732949 2712547
Kswapd efficiency 100% 89% 90%
Kswapd velocity 0.000 1384.254 1445.574
Direct efficiency 33% 3% 3%
Direct velocity 27991.453 184832.737 208171.929
Percentage direct scans 100% 99% 99%
Page writes by reclaim 0 5082 13917
Slabs scanned 17280 29952 35328
Direct inode steals 115257 1431122 332201
Kswapd inode steals 0 0 979532
This test downloads a large tarfile and copies it with dd a number of
times - similar to the most recent bug report I've dealt with. Time to
completion is reduced. The number of pages scanned directly is still
disturbingly high with a low efficiency but this is likely due to the
number of dirty pages encountered. The figures could probably be improved
with more work around how kswapd is used and how dirty pages are handled
but that is separate work and this result is significant on its own.
Streaming Mapped Writer
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 124.47 111.67 112.64
Total Elapsed Time (seconds) 2138.14 1816.30 1867.56
MMTests Statistics: vmstat
Page Ins 90760 89124 89516
Page Outs 121028340 120199524 120736696
Swap Ins 0 86 55
Swap Outs 0 0 0
Direct pages scanned 114989363 96461439 96330619
Kswapd pages scanned 56430948 56965763 57075875
Kswapd pages reclaimed 27743219 27752044 27766606
Direct pages reclaimed 49777 46884 36655
Kswapd efficiency 49% 48% 48%
Kswapd velocity 26392.541 31363.631 30561.736
Direct efficiency 0% 0% 0%
Direct velocity 53780.091 53108.759 51581.004
Percentage direct scans 67% 62% 62%
Page writes by reclaim 385 122 1513
Slabs scanned 43008 39040 42112
Direct inode steals 0 10 8
Kswapd inode steals 733 534 477
This test just creates a large file mapping and writes to it linearly.
Time to completion is again reduced.
The gains are mostly down to two things. In many cases, there is less
scanning as zone_reclaim simply gives up faster due to recent failures.
The second reason is that memory is used more efficiently. Instead of
scanning the preferred zone every time, the allocator falls back to
another zone and uses it instead improving overall memory utilisation.
This patch: initialise ZLC for first zone eligible for zone_reclaim.
The zonelist cache (ZLC) is used among other things to record if
zone_reclaim() failed for a particular zone recently. The intention is to
avoid a high cost scanning extremely long zonelists or scanning within the
zone uselessly.
Currently the zonelist cache is setup only after the first zone has been
considered and zone_reclaim() has been called. The objective was to avoid
a costly setup but zone_reclaim is itself quite expensive. If it is
failing regularly such as the first eligible zone having mostly mapped
pages, the cost in scanning and allocation stalls is far higher than the
ZLC initialisation step.
This patch initialises ZLC before the first eligible zone calls
zone_reclaim(). Once initialised, it is checked whether the zone failed
zone_reclaim recently. If it has, the zone is skipped. As the first zone
is now being checked, additional care has to be taken about zones marked
full. A zone can be marked "full" because it should not have enough
unmapped pages for zone_reclaim but this is excessive as direct reclaim or
kswapd may succeed where zone_reclaim fails. Only mark zones "full" after
zone_reclaim fails if it failed to reclaim enough pages after scanning.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-07-25 18:12:29 -06:00
|
|
|
continue;
|
mm: try to distribute dirty pages fairly across zones
The maximum number of dirty pages that exist in the system at any time is
determined by a number of pages considered dirtyable and a user-configured
percentage of those, or an absolute number in bytes.
This number of dirtyable pages is the sum of memory provided by all the
zones in the system minus their lowmem reserves and high watermarks, so
that the system can retain a healthy number of free pages without having
to reclaim dirty pages.
But there is a flaw in that we have a zoned page allocator which does not
care about the global state but rather the state of individual memory
zones. And right now there is nothing that prevents one zone from filling
up with dirty pages while other zones are spared, which frequently leads
to situations where kswapd, in order to restore the watermark of free
pages, does indeed have to write pages from that zone's LRU list. This
can interfere so badly with IO from the flusher threads that major
filesystems (btrfs, xfs, ext4) mostly ignore write requests from reclaim
already, taking away the VM's only possibility to keep such a zone
balanced, aside from hoping the flushers will soon clean pages from that
zone.
Enter per-zone dirty limits. They are to a zone's dirtyable memory what
the global limit is to the global amount of dirtyable memory, and try to
make sure that no single zone receives more than its fair share of the
globally allowed dirty pages in the first place. As the number of pages
considered dirtyable excludes the zones' lowmem reserves and high
watermarks, the maximum number of dirty pages in a zone is such that the
zone can always be balanced without requiring page cleaning.
As this is a placement decision in the page allocator and pages are
dirtied only after the allocation, this patch allows allocators to pass
__GFP_WRITE when they know in advance that the page will be written to and
become dirty soon. The page allocator will then attempt to allocate from
the first zone of the zonelist - which on NUMA is determined by the task's
NUMA memory policy - that has not exceeded its dirty limit.
At first glance, it would appear that the diversion to lower zones can
increase pressure on them, but this is not the case. With a full high
zone, allocations will be diverted to lower zones eventually, so it is
more of a shift in timing of the lower zone allocations. Workloads that
previously could fit their dirty pages completely in the higher zone may
be forced to allocate from lower zones, but the amount of pages that
"spill over" are limited themselves by the lower zones' dirty constraints,
and thus unlikely to become a problem.
For now, the problem of unfair dirty page distribution remains for NUMA
configurations where the zones allowed for allocation are in sum not big
enough to trigger the global dirty limits, wake up the flusher threads and
remedy the situation. Because of this, an allocation that could not
succeed on any of the considered zones is allowed to ignore the dirty
limits before going into direct reclaim or even failing the allocation,
until a future patch changes the global dirty throttling and flusher
thread activation so that they take individual zone states into account.
Test results
15M DMA + 3246M DMA32 + 504 Normal = 3765M memory
40% dirty ratio
16G USB thumb drive
10 runs of dd if=/dev/zero of=disk/zeroes bs=32k count=$((10 << 15))
seconds nr_vmscan_write
(stddev) min| median| max
xfs
vanilla: 549.747( 3.492) 0.000| 0.000| 0.000
patched: 550.996( 3.802) 0.000| 0.000| 0.000
fuse-ntfs
vanilla: 1183.094(53.178) 54349.000| 59341.000| 65163.000
patched: 558.049(17.914) 0.000| 0.000| 43.000
btrfs
vanilla: 573.679(14.015) 156657.000| 460178.000| 606926.000
patched: 563.365(11.368) 0.000| 0.000| 1362.000
ext4
vanilla: 561.197(15.782) 0.000|2725438.000|4143837.000
patched: 568.806(17.496) 0.000| 0.000| 0.000
Signed-off-by: Johannes Weiner <jweiner@redhat.com>
Reviewed-by: Minchan Kim <minchan.kim@gmail.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Tested-by: Wu Fengguang <fengguang.wu@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Shaohua Li <shaohua.li@intel.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Chris Mason <chris.mason@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-01-10 16:07:49 -07:00
|
|
|
/*
|
|
|
|
* When allocating a page cache page for writing, we
|
2016-07-28 16:46:11 -06:00
|
|
|
* want to get it from a node that is within its dirty
|
|
|
|
* limit, such that no single node holds more than its
|
mm: try to distribute dirty pages fairly across zones
The maximum number of dirty pages that exist in the system at any time is
determined by a number of pages considered dirtyable and a user-configured
percentage of those, or an absolute number in bytes.
This number of dirtyable pages is the sum of memory provided by all the
zones in the system minus their lowmem reserves and high watermarks, so
that the system can retain a healthy number of free pages without having
to reclaim dirty pages.
But there is a flaw in that we have a zoned page allocator which does not
care about the global state but rather the state of individual memory
zones. And right now there is nothing that prevents one zone from filling
up with dirty pages while other zones are spared, which frequently leads
to situations where kswapd, in order to restore the watermark of free
pages, does indeed have to write pages from that zone's LRU list. This
can interfere so badly with IO from the flusher threads that major
filesystems (btrfs, xfs, ext4) mostly ignore write requests from reclaim
already, taking away the VM's only possibility to keep such a zone
balanced, aside from hoping the flushers will soon clean pages from that
zone.
Enter per-zone dirty limits. They are to a zone's dirtyable memory what
the global limit is to the global amount of dirtyable memory, and try to
make sure that no single zone receives more than its fair share of the
globally allowed dirty pages in the first place. As the number of pages
considered dirtyable excludes the zones' lowmem reserves and high
watermarks, the maximum number of dirty pages in a zone is such that the
zone can always be balanced without requiring page cleaning.
As this is a placement decision in the page allocator and pages are
dirtied only after the allocation, this patch allows allocators to pass
__GFP_WRITE when they know in advance that the page will be written to and
become dirty soon. The page allocator will then attempt to allocate from
the first zone of the zonelist - which on NUMA is determined by the task's
NUMA memory policy - that has not exceeded its dirty limit.
At first glance, it would appear that the diversion to lower zones can
increase pressure on them, but this is not the case. With a full high
zone, allocations will be diverted to lower zones eventually, so it is
more of a shift in timing of the lower zone allocations. Workloads that
previously could fit their dirty pages completely in the higher zone may
be forced to allocate from lower zones, but the amount of pages that
"spill over" are limited themselves by the lower zones' dirty constraints,
and thus unlikely to become a problem.
For now, the problem of unfair dirty page distribution remains for NUMA
configurations where the zones allowed for allocation are in sum not big
enough to trigger the global dirty limits, wake up the flusher threads and
remedy the situation. Because of this, an allocation that could not
succeed on any of the considered zones is allowed to ignore the dirty
limits before going into direct reclaim or even failing the allocation,
until a future patch changes the global dirty throttling and flusher
thread activation so that they take individual zone states into account.
Test results
15M DMA + 3246M DMA32 + 504 Normal = 3765M memory
40% dirty ratio
16G USB thumb drive
10 runs of dd if=/dev/zero of=disk/zeroes bs=32k count=$((10 << 15))
seconds nr_vmscan_write
(stddev) min| median| max
xfs
vanilla: 549.747( 3.492) 0.000| 0.000| 0.000
patched: 550.996( 3.802) 0.000| 0.000| 0.000
fuse-ntfs
vanilla: 1183.094(53.178) 54349.000| 59341.000| 65163.000
patched: 558.049(17.914) 0.000| 0.000| 43.000
btrfs
vanilla: 573.679(14.015) 156657.000| 460178.000| 606926.000
patched: 563.365(11.368) 0.000| 0.000| 1362.000
ext4
vanilla: 561.197(15.782) 0.000|2725438.000|4143837.000
patched: 568.806(17.496) 0.000| 0.000| 0.000
Signed-off-by: Johannes Weiner <jweiner@redhat.com>
Reviewed-by: Minchan Kim <minchan.kim@gmail.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Tested-by: Wu Fengguang <fengguang.wu@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Shaohua Li <shaohua.li@intel.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Chris Mason <chris.mason@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-01-10 16:07:49 -07:00
|
|
|
* proportional share of globally allowed dirty pages.
|
2016-07-28 16:46:11 -06:00
|
|
|
* The dirty limits take into account the node's
|
mm: try to distribute dirty pages fairly across zones
The maximum number of dirty pages that exist in the system at any time is
determined by a number of pages considered dirtyable and a user-configured
percentage of those, or an absolute number in bytes.
This number of dirtyable pages is the sum of memory provided by all the
zones in the system minus their lowmem reserves and high watermarks, so
that the system can retain a healthy number of free pages without having
to reclaim dirty pages.
But there is a flaw in that we have a zoned page allocator which does not
care about the global state but rather the state of individual memory
zones. And right now there is nothing that prevents one zone from filling
up with dirty pages while other zones are spared, which frequently leads
to situations where kswapd, in order to restore the watermark of free
pages, does indeed have to write pages from that zone's LRU list. This
can interfere so badly with IO from the flusher threads that major
filesystems (btrfs, xfs, ext4) mostly ignore write requests from reclaim
already, taking away the VM's only possibility to keep such a zone
balanced, aside from hoping the flushers will soon clean pages from that
zone.
Enter per-zone dirty limits. They are to a zone's dirtyable memory what
the global limit is to the global amount of dirtyable memory, and try to
make sure that no single zone receives more than its fair share of the
globally allowed dirty pages in the first place. As the number of pages
considered dirtyable excludes the zones' lowmem reserves and high
watermarks, the maximum number of dirty pages in a zone is such that the
zone can always be balanced without requiring page cleaning.
As this is a placement decision in the page allocator and pages are
dirtied only after the allocation, this patch allows allocators to pass
__GFP_WRITE when they know in advance that the page will be written to and
become dirty soon. The page allocator will then attempt to allocate from
the first zone of the zonelist - which on NUMA is determined by the task's
NUMA memory policy - that has not exceeded its dirty limit.
At first glance, it would appear that the diversion to lower zones can
increase pressure on them, but this is not the case. With a full high
zone, allocations will be diverted to lower zones eventually, so it is
more of a shift in timing of the lower zone allocations. Workloads that
previously could fit their dirty pages completely in the higher zone may
be forced to allocate from lower zones, but the amount of pages that
"spill over" are limited themselves by the lower zones' dirty constraints,
and thus unlikely to become a problem.
For now, the problem of unfair dirty page distribution remains for NUMA
configurations where the zones allowed for allocation are in sum not big
enough to trigger the global dirty limits, wake up the flusher threads and
remedy the situation. Because of this, an allocation that could not
succeed on any of the considered zones is allowed to ignore the dirty
limits before going into direct reclaim or even failing the allocation,
until a future patch changes the global dirty throttling and flusher
thread activation so that they take individual zone states into account.
Test results
15M DMA + 3246M DMA32 + 504 Normal = 3765M memory
40% dirty ratio
16G USB thumb drive
10 runs of dd if=/dev/zero of=disk/zeroes bs=32k count=$((10 << 15))
seconds nr_vmscan_write
(stddev) min| median| max
xfs
vanilla: 549.747( 3.492) 0.000| 0.000| 0.000
patched: 550.996( 3.802) 0.000| 0.000| 0.000
fuse-ntfs
vanilla: 1183.094(53.178) 54349.000| 59341.000| 65163.000
patched: 558.049(17.914) 0.000| 0.000| 43.000
btrfs
vanilla: 573.679(14.015) 156657.000| 460178.000| 606926.000
patched: 563.365(11.368) 0.000| 0.000| 1362.000
ext4
vanilla: 561.197(15.782) 0.000|2725438.000|4143837.000
patched: 568.806(17.496) 0.000| 0.000| 0.000
Signed-off-by: Johannes Weiner <jweiner@redhat.com>
Reviewed-by: Minchan Kim <minchan.kim@gmail.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Tested-by: Wu Fengguang <fengguang.wu@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Shaohua Li <shaohua.li@intel.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Chris Mason <chris.mason@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-01-10 16:07:49 -07:00
|
|
|
* lowmem reserves and high watermark so that kswapd
|
|
|
|
* should be able to balance it without having to
|
|
|
|
* write pages from its LRU list.
|
|
|
|
*
|
|
|
|
* XXX: For now, allow allocations to potentially
|
2016-07-28 16:46:11 -06:00
|
|
|
* exceed the per-node dirty limit in the slowpath
|
2015-11-06 17:28:12 -07:00
|
|
|
* (spread_dirty_pages unset) before going into reclaim,
|
mm: try to distribute dirty pages fairly across zones
The maximum number of dirty pages that exist in the system at any time is
determined by a number of pages considered dirtyable and a user-configured
percentage of those, or an absolute number in bytes.
This number of dirtyable pages is the sum of memory provided by all the
zones in the system minus their lowmem reserves and high watermarks, so
that the system can retain a healthy number of free pages without having
to reclaim dirty pages.
But there is a flaw in that we have a zoned page allocator which does not
care about the global state but rather the state of individual memory
zones. And right now there is nothing that prevents one zone from filling
up with dirty pages while other zones are spared, which frequently leads
to situations where kswapd, in order to restore the watermark of free
pages, does indeed have to write pages from that zone's LRU list. This
can interfere so badly with IO from the flusher threads that major
filesystems (btrfs, xfs, ext4) mostly ignore write requests from reclaim
already, taking away the VM's only possibility to keep such a zone
balanced, aside from hoping the flushers will soon clean pages from that
zone.
Enter per-zone dirty limits. They are to a zone's dirtyable memory what
the global limit is to the global amount of dirtyable memory, and try to
make sure that no single zone receives more than its fair share of the
globally allowed dirty pages in the first place. As the number of pages
considered dirtyable excludes the zones' lowmem reserves and high
watermarks, the maximum number of dirty pages in a zone is such that the
zone can always be balanced without requiring page cleaning.
As this is a placement decision in the page allocator and pages are
dirtied only after the allocation, this patch allows allocators to pass
__GFP_WRITE when they know in advance that the page will be written to and
become dirty soon. The page allocator will then attempt to allocate from
the first zone of the zonelist - which on NUMA is determined by the task's
NUMA memory policy - that has not exceeded its dirty limit.
At first glance, it would appear that the diversion to lower zones can
increase pressure on them, but this is not the case. With a full high
zone, allocations will be diverted to lower zones eventually, so it is
more of a shift in timing of the lower zone allocations. Workloads that
previously could fit their dirty pages completely in the higher zone may
be forced to allocate from lower zones, but the amount of pages that
"spill over" are limited themselves by the lower zones' dirty constraints,
and thus unlikely to become a problem.
For now, the problem of unfair dirty page distribution remains for NUMA
configurations where the zones allowed for allocation are in sum not big
enough to trigger the global dirty limits, wake up the flusher threads and
remedy the situation. Because of this, an allocation that could not
succeed on any of the considered zones is allowed to ignore the dirty
limits before going into direct reclaim or even failing the allocation,
until a future patch changes the global dirty throttling and flusher
thread activation so that they take individual zone states into account.
Test results
15M DMA + 3246M DMA32 + 504 Normal = 3765M memory
40% dirty ratio
16G USB thumb drive
10 runs of dd if=/dev/zero of=disk/zeroes bs=32k count=$((10 << 15))
seconds nr_vmscan_write
(stddev) min| median| max
xfs
vanilla: 549.747( 3.492) 0.000| 0.000| 0.000
patched: 550.996( 3.802) 0.000| 0.000| 0.000
fuse-ntfs
vanilla: 1183.094(53.178) 54349.000| 59341.000| 65163.000
patched: 558.049(17.914) 0.000| 0.000| 43.000
btrfs
vanilla: 573.679(14.015) 156657.000| 460178.000| 606926.000
patched: 563.365(11.368) 0.000| 0.000| 1362.000
ext4
vanilla: 561.197(15.782) 0.000|2725438.000|4143837.000
patched: 568.806(17.496) 0.000| 0.000| 0.000
Signed-off-by: Johannes Weiner <jweiner@redhat.com>
Reviewed-by: Minchan Kim <minchan.kim@gmail.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Tested-by: Wu Fengguang <fengguang.wu@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Shaohua Li <shaohua.li@intel.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Chris Mason <chris.mason@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-01-10 16:07:49 -07:00
|
|
|
* which is important when on a NUMA setup the allowed
|
2016-07-28 16:46:11 -06:00
|
|
|
* nodes are together not big enough to reach the
|
mm: try to distribute dirty pages fairly across zones
The maximum number of dirty pages that exist in the system at any time is
determined by a number of pages considered dirtyable and a user-configured
percentage of those, or an absolute number in bytes.
This number of dirtyable pages is the sum of memory provided by all the
zones in the system minus their lowmem reserves and high watermarks, so
that the system can retain a healthy number of free pages without having
to reclaim dirty pages.
But there is a flaw in that we have a zoned page allocator which does not
care about the global state but rather the state of individual memory
zones. And right now there is nothing that prevents one zone from filling
up with dirty pages while other zones are spared, which frequently leads
to situations where kswapd, in order to restore the watermark of free
pages, does indeed have to write pages from that zone's LRU list. This
can interfere so badly with IO from the flusher threads that major
filesystems (btrfs, xfs, ext4) mostly ignore write requests from reclaim
already, taking away the VM's only possibility to keep such a zone
balanced, aside from hoping the flushers will soon clean pages from that
zone.
Enter per-zone dirty limits. They are to a zone's dirtyable memory what
the global limit is to the global amount of dirtyable memory, and try to
make sure that no single zone receives more than its fair share of the
globally allowed dirty pages in the first place. As the number of pages
considered dirtyable excludes the zones' lowmem reserves and high
watermarks, the maximum number of dirty pages in a zone is such that the
zone can always be balanced without requiring page cleaning.
As this is a placement decision in the page allocator and pages are
dirtied only after the allocation, this patch allows allocators to pass
__GFP_WRITE when they know in advance that the page will be written to and
become dirty soon. The page allocator will then attempt to allocate from
the first zone of the zonelist - which on NUMA is determined by the task's
NUMA memory policy - that has not exceeded its dirty limit.
At first glance, it would appear that the diversion to lower zones can
increase pressure on them, but this is not the case. With a full high
zone, allocations will be diverted to lower zones eventually, so it is
more of a shift in timing of the lower zone allocations. Workloads that
previously could fit their dirty pages completely in the higher zone may
be forced to allocate from lower zones, but the amount of pages that
"spill over" are limited themselves by the lower zones' dirty constraints,
and thus unlikely to become a problem.
For now, the problem of unfair dirty page distribution remains for NUMA
configurations where the zones allowed for allocation are in sum not big
enough to trigger the global dirty limits, wake up the flusher threads and
remedy the situation. Because of this, an allocation that could not
succeed on any of the considered zones is allowed to ignore the dirty
limits before going into direct reclaim or even failing the allocation,
until a future patch changes the global dirty throttling and flusher
thread activation so that they take individual zone states into account.
Test results
15M DMA + 3246M DMA32 + 504 Normal = 3765M memory
40% dirty ratio
16G USB thumb drive
10 runs of dd if=/dev/zero of=disk/zeroes bs=32k count=$((10 << 15))
seconds nr_vmscan_write
(stddev) min| median| max
xfs
vanilla: 549.747( 3.492) 0.000| 0.000| 0.000
patched: 550.996( 3.802) 0.000| 0.000| 0.000
fuse-ntfs
vanilla: 1183.094(53.178) 54349.000| 59341.000| 65163.000
patched: 558.049(17.914) 0.000| 0.000| 43.000
btrfs
vanilla: 573.679(14.015) 156657.000| 460178.000| 606926.000
patched: 563.365(11.368) 0.000| 0.000| 1362.000
ext4
vanilla: 561.197(15.782) 0.000|2725438.000|4143837.000
patched: 568.806(17.496) 0.000| 0.000| 0.000
Signed-off-by: Johannes Weiner <jweiner@redhat.com>
Reviewed-by: Minchan Kim <minchan.kim@gmail.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Tested-by: Wu Fengguang <fengguang.wu@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Shaohua Li <shaohua.li@intel.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Chris Mason <chris.mason@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-01-10 16:07:49 -07:00
|
|
|
* global limit. The proper fix for these situations
|
2016-07-28 16:46:11 -06:00
|
|
|
* will require awareness of nodes in the
|
mm: try to distribute dirty pages fairly across zones
The maximum number of dirty pages that exist in the system at any time is
determined by a number of pages considered dirtyable and a user-configured
percentage of those, or an absolute number in bytes.
This number of dirtyable pages is the sum of memory provided by all the
zones in the system minus their lowmem reserves and high watermarks, so
that the system can retain a healthy number of free pages without having
to reclaim dirty pages.
But there is a flaw in that we have a zoned page allocator which does not
care about the global state but rather the state of individual memory
zones. And right now there is nothing that prevents one zone from filling
up with dirty pages while other zones are spared, which frequently leads
to situations where kswapd, in order to restore the watermark of free
pages, does indeed have to write pages from that zone's LRU list. This
can interfere so badly with IO from the flusher threads that major
filesystems (btrfs, xfs, ext4) mostly ignore write requests from reclaim
already, taking away the VM's only possibility to keep such a zone
balanced, aside from hoping the flushers will soon clean pages from that
zone.
Enter per-zone dirty limits. They are to a zone's dirtyable memory what
the global limit is to the global amount of dirtyable memory, and try to
make sure that no single zone receives more than its fair share of the
globally allowed dirty pages in the first place. As the number of pages
considered dirtyable excludes the zones' lowmem reserves and high
watermarks, the maximum number of dirty pages in a zone is such that the
zone can always be balanced without requiring page cleaning.
As this is a placement decision in the page allocator and pages are
dirtied only after the allocation, this patch allows allocators to pass
__GFP_WRITE when they know in advance that the page will be written to and
become dirty soon. The page allocator will then attempt to allocate from
the first zone of the zonelist - which on NUMA is determined by the task's
NUMA memory policy - that has not exceeded its dirty limit.
At first glance, it would appear that the diversion to lower zones can
increase pressure on them, but this is not the case. With a full high
zone, allocations will be diverted to lower zones eventually, so it is
more of a shift in timing of the lower zone allocations. Workloads that
previously could fit their dirty pages completely in the higher zone may
be forced to allocate from lower zones, but the amount of pages that
"spill over" are limited themselves by the lower zones' dirty constraints,
and thus unlikely to become a problem.
For now, the problem of unfair dirty page distribution remains for NUMA
configurations where the zones allowed for allocation are in sum not big
enough to trigger the global dirty limits, wake up the flusher threads and
remedy the situation. Because of this, an allocation that could not
succeed on any of the considered zones is allowed to ignore the dirty
limits before going into direct reclaim or even failing the allocation,
until a future patch changes the global dirty throttling and flusher
thread activation so that they take individual zone states into account.
Test results
15M DMA + 3246M DMA32 + 504 Normal = 3765M memory
40% dirty ratio
16G USB thumb drive
10 runs of dd if=/dev/zero of=disk/zeroes bs=32k count=$((10 << 15))
seconds nr_vmscan_write
(stddev) min| median| max
xfs
vanilla: 549.747( 3.492) 0.000| 0.000| 0.000
patched: 550.996( 3.802) 0.000| 0.000| 0.000
fuse-ntfs
vanilla: 1183.094(53.178) 54349.000| 59341.000| 65163.000
patched: 558.049(17.914) 0.000| 0.000| 43.000
btrfs
vanilla: 573.679(14.015) 156657.000| 460178.000| 606926.000
patched: 563.365(11.368) 0.000| 0.000| 1362.000
ext4
vanilla: 561.197(15.782) 0.000|2725438.000|4143837.000
patched: 568.806(17.496) 0.000| 0.000| 0.000
Signed-off-by: Johannes Weiner <jweiner@redhat.com>
Reviewed-by: Minchan Kim <minchan.kim@gmail.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Tested-by: Wu Fengguang <fengguang.wu@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Shaohua Li <shaohua.li@intel.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Chris Mason <chris.mason@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-01-10 16:07:49 -07:00
|
|
|
* dirty-throttling and the flusher threads.
|
|
|
|
*/
|
2016-07-28 16:46:53 -06:00
|
|
|
if (ac->spread_dirty_pages) {
|
|
|
|
if (last_pgdat_dirty_limit == zone->zone_pgdat)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (!node_dirty_ok(zone->zone_pgdat)) {
|
|
|
|
last_pgdat_dirty_limit = zone->zone_pgdat;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
2005-11-13 17:06:43 -07:00
|
|
|
|
2013-09-11 15:20:46 -06:00
|
|
|
mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
|
2016-05-19 18:14:07 -06:00
|
|
|
if (!zone_watermark_fast(zone, order, mark,
|
2016-05-19 18:14:13 -06:00
|
|
|
ac_classzone_idx(ac), alloc_flags)) {
|
2009-06-16 16:33:22 -06:00
|
|
|
int ret;
|
|
|
|
|
2014-06-04 17:10:14 -06:00
|
|
|
/* Checked here to keep the fast path fast */
|
|
|
|
BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
|
|
|
|
if (alloc_flags & ALLOC_NO_WATERMARKS)
|
|
|
|
goto try_this_zone;
|
|
|
|
|
2016-07-28 16:46:32 -06:00
|
|
|
if (node_reclaim_mode == 0 ||
|
2016-05-19 18:14:10 -06:00
|
|
|
!zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
|
mm: page allocator: initialise ZLC for first zone eligible for zone_reclaim
There have been a small number of complaints about significant stalls
while copying large amounts of data on NUMA machines reported on a
distribution bugzilla. In these cases, zone_reclaim was enabled by
default due to large NUMA distances. In general, the complaints have not
been about the workload itself unless it was a file server (in which case
the recommendation was disable zone_reclaim).
The stalls are mostly due to significant amounts of time spent scanning
the preferred zone for pages to free. After a failure, it might fallback
to another node (as zonelists are often node-ordered rather than
zone-ordered) but stall quickly again when the next allocation attempt
occurs. In bad cases, each page allocated results in a full scan of the
preferred zone.
Patch 1 checks the preferred zone for recent allocation failure
which is particularly important if zone_reclaim has failed
recently. This avoids rescanning the zone in the near future and
instead falling back to another node. This may hurt node locality
in some cases but a failure to zone_reclaim is more expensive than
a remote access.
Patch 2 clears the zlc information after direct reclaim.
Otherwise, zone_reclaim can mark zones full, direct reclaim can
reclaim enough pages but the zone is still not considered for
allocation.
This was tested on a 24-thread 2-node x86_64 machine. The tests were
focused on large amounts of IO. All tests were bound to the CPUs on
node-0 to avoid disturbances due to processes being scheduled on different
nodes. The kernels tested are
3.0-rc6-vanilla Vanilla 3.0-rc6
zlcfirst Patch 1 applied
zlcreconsider Patches 1+2 applied
FS-Mark
./fs_mark -d /tmp/fsmark-10813 -D 100 -N 5000 -n 208 -L 35 -t 24 -S0 -s 524288
fsmark-3.0-rc6 3.0-rc6 3.0-rc6
vanilla zlcfirs zlcreconsider
Files/s min 54.90 ( 0.00%) 49.80 (-10.24%) 49.10 (-11.81%)
Files/s mean 100.11 ( 0.00%) 135.17 (25.94%) 146.93 (31.87%)
Files/s stddev 57.51 ( 0.00%) 138.97 (58.62%) 158.69 (63.76%)
Files/s max 361.10 ( 0.00%) 834.40 (56.72%) 802.40 (55.00%)
Overhead min 76704.00 ( 0.00%) 76501.00 ( 0.27%) 77784.00 (-1.39%)
Overhead mean 1485356.51 ( 0.00%) 1035797.83 (43.40%) 1594680.26 (-6.86%)
Overhead stddev 1848122.53 ( 0.00%) 881489.88 (109.66%) 1772354.90 ( 4.27%)
Overhead max 7989060.00 ( 0.00%) 3369118.00 (137.13%) 10135324.00 (-21.18%)
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 501.49 493.91 499.93
Total Elapsed Time (seconds) 2451.57 2257.48 2215.92
MMTests Statistics: vmstat
Page Ins 46268 63840 66008
Page Outs 90821596 90671128 88043732
Swap Ins 0 0 0
Swap Outs 0 0 0
Direct pages scanned 13091697 8966863 8971790
Kswapd pages scanned 0 1830011 1831116
Kswapd pages reclaimed 0 1829068 1829930
Direct pages reclaimed 13037777 8956828 8648314
Kswapd efficiency 100% 99% 99%
Kswapd velocity 0.000 810.643 826.346
Direct efficiency 99% 99% 96%
Direct velocity 5340.128 3972.068 4048.788
Percentage direct scans 100% 83% 83%
Page writes by reclaim 0 3 0
Slabs scanned 796672 720640 720256
Direct inode steals 7422667 7160012 7088638
Kswapd inode steals 0 1736840 2021238
Test completes far faster with a large increase in the number of files
created per second. Standard deviation is high as a small number of
iterations were much higher than the mean. The number of pages scanned by
zone_reclaim is reduced and kswapd is used for more work.
LARGE DD
3.0-rc6 3.0-rc6 3.0-rc6
vanilla zlcfirst zlcreconsider
download tar 59 ( 0.00%) 59 ( 0.00%) 55 ( 7.27%)
dd source files 527 ( 0.00%) 296 (78.04%) 320 (64.69%)
delete source 36 ( 0.00%) 19 (89.47%) 20 (80.00%)
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 125.03 118.98 122.01
Total Elapsed Time (seconds) 624.56 375.02 398.06
MMTests Statistics: vmstat
Page Ins 3594216 439368 407032
Page Outs 23380832 23380488 23377444
Swap Ins 0 0 0
Swap Outs 0 436 287
Direct pages scanned 17482342 69315973 82864918
Kswapd pages scanned 0 519123 575425
Kswapd pages reclaimed 0 466501 522487
Direct pages reclaimed 5858054 2732949 2712547
Kswapd efficiency 100% 89% 90%
Kswapd velocity 0.000 1384.254 1445.574
Direct efficiency 33% 3% 3%
Direct velocity 27991.453 184832.737 208171.929
Percentage direct scans 100% 99% 99%
Page writes by reclaim 0 5082 13917
Slabs scanned 17280 29952 35328
Direct inode steals 115257 1431122 332201
Kswapd inode steals 0 0 979532
This test downloads a large tarfile and copies it with dd a number of
times - similar to the most recent bug report I've dealt with. Time to
completion is reduced. The number of pages scanned directly is still
disturbingly high with a low efficiency but this is likely due to the
number of dirty pages encountered. The figures could probably be improved
with more work around how kswapd is used and how dirty pages are handled
but that is separate work and this result is significant on its own.
Streaming Mapped Writer
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 124.47 111.67 112.64
Total Elapsed Time (seconds) 2138.14 1816.30 1867.56
MMTests Statistics: vmstat
Page Ins 90760 89124 89516
Page Outs 121028340 120199524 120736696
Swap Ins 0 86 55
Swap Outs 0 0 0
Direct pages scanned 114989363 96461439 96330619
Kswapd pages scanned 56430948 56965763 57075875
Kswapd pages reclaimed 27743219 27752044 27766606
Direct pages reclaimed 49777 46884 36655
Kswapd efficiency 49% 48% 48%
Kswapd velocity 26392.541 31363.631 30561.736
Direct efficiency 0% 0% 0%
Direct velocity 53780.091 53108.759 51581.004
Percentage direct scans 67% 62% 62%
Page writes by reclaim 385 122 1513
Slabs scanned 43008 39040 42112
Direct inode steals 0 10 8
Kswapd inode steals 733 534 477
This test just creates a large file mapping and writes to it linearly.
Time to completion is again reduced.
The gains are mostly down to two things. In many cases, there is less
scanning as zone_reclaim simply gives up faster due to recent failures.
The second reason is that memory is used more efficiently. Instead of
scanning the preferred zone every time, the allocator falls back to
another zone and uses it instead improving overall memory utilisation.
This patch: initialise ZLC for first zone eligible for zone_reclaim.
The zonelist cache (ZLC) is used among other things to record if
zone_reclaim() failed for a particular zone recently. The intention is to
avoid a high cost scanning extremely long zonelists or scanning within the
zone uselessly.
Currently the zonelist cache is setup only after the first zone has been
considered and zone_reclaim() has been called. The objective was to avoid
a costly setup but zone_reclaim is itself quite expensive. If it is
failing regularly such as the first eligible zone having mostly mapped
pages, the cost in scanning and allocation stalls is far higher than the
ZLC initialisation step.
This patch initialises ZLC before the first eligible zone calls
zone_reclaim(). Once initialised, it is checked whether the zone failed
zone_reclaim recently. If it has, the zone is skipped. As the first zone
is now being checked, additional care has to be taken about zones marked
full. A zone can be marked "full" because it should not have enough
unmapped pages for zone_reclaim but this is excessive as direct reclaim or
kswapd may succeed where zone_reclaim fails. Only mark zones "full" after
zone_reclaim fails if it failed to reclaim enough pages after scanning.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-07-25 18:12:29 -06:00
|
|
|
continue;
|
|
|
|
|
2016-07-28 16:46:32 -06:00
|
|
|
ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
|
2009-06-16 16:33:22 -06:00
|
|
|
switch (ret) {
|
2016-07-28 16:46:32 -06:00
|
|
|
case NODE_RECLAIM_NOSCAN:
|
2009-06-16 16:33:22 -06:00
|
|
|
/* did not scan */
|
mm: page allocator: initialise ZLC for first zone eligible for zone_reclaim
There have been a small number of complaints about significant stalls
while copying large amounts of data on NUMA machines reported on a
distribution bugzilla. In these cases, zone_reclaim was enabled by
default due to large NUMA distances. In general, the complaints have not
been about the workload itself unless it was a file server (in which case
the recommendation was disable zone_reclaim).
The stalls are mostly due to significant amounts of time spent scanning
the preferred zone for pages to free. After a failure, it might fallback
to another node (as zonelists are often node-ordered rather than
zone-ordered) but stall quickly again when the next allocation attempt
occurs. In bad cases, each page allocated results in a full scan of the
preferred zone.
Patch 1 checks the preferred zone for recent allocation failure
which is particularly important if zone_reclaim has failed
recently. This avoids rescanning the zone in the near future and
instead falling back to another node. This may hurt node locality
in some cases but a failure to zone_reclaim is more expensive than
a remote access.
Patch 2 clears the zlc information after direct reclaim.
Otherwise, zone_reclaim can mark zones full, direct reclaim can
reclaim enough pages but the zone is still not considered for
allocation.
This was tested on a 24-thread 2-node x86_64 machine. The tests were
focused on large amounts of IO. All tests were bound to the CPUs on
node-0 to avoid disturbances due to processes being scheduled on different
nodes. The kernels tested are
3.0-rc6-vanilla Vanilla 3.0-rc6
zlcfirst Patch 1 applied
zlcreconsider Patches 1+2 applied
FS-Mark
./fs_mark -d /tmp/fsmark-10813 -D 100 -N 5000 -n 208 -L 35 -t 24 -S0 -s 524288
fsmark-3.0-rc6 3.0-rc6 3.0-rc6
vanilla zlcfirs zlcreconsider
Files/s min 54.90 ( 0.00%) 49.80 (-10.24%) 49.10 (-11.81%)
Files/s mean 100.11 ( 0.00%) 135.17 (25.94%) 146.93 (31.87%)
Files/s stddev 57.51 ( 0.00%) 138.97 (58.62%) 158.69 (63.76%)
Files/s max 361.10 ( 0.00%) 834.40 (56.72%) 802.40 (55.00%)
Overhead min 76704.00 ( 0.00%) 76501.00 ( 0.27%) 77784.00 (-1.39%)
Overhead mean 1485356.51 ( 0.00%) 1035797.83 (43.40%) 1594680.26 (-6.86%)
Overhead stddev 1848122.53 ( 0.00%) 881489.88 (109.66%) 1772354.90 ( 4.27%)
Overhead max 7989060.00 ( 0.00%) 3369118.00 (137.13%) 10135324.00 (-21.18%)
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 501.49 493.91 499.93
Total Elapsed Time (seconds) 2451.57 2257.48 2215.92
MMTests Statistics: vmstat
Page Ins 46268 63840 66008
Page Outs 90821596 90671128 88043732
Swap Ins 0 0 0
Swap Outs 0 0 0
Direct pages scanned 13091697 8966863 8971790
Kswapd pages scanned 0 1830011 1831116
Kswapd pages reclaimed 0 1829068 1829930
Direct pages reclaimed 13037777 8956828 8648314
Kswapd efficiency 100% 99% 99%
Kswapd velocity 0.000 810.643 826.346
Direct efficiency 99% 99% 96%
Direct velocity 5340.128 3972.068 4048.788
Percentage direct scans 100% 83% 83%
Page writes by reclaim 0 3 0
Slabs scanned 796672 720640 720256
Direct inode steals 7422667 7160012 7088638
Kswapd inode steals 0 1736840 2021238
Test completes far faster with a large increase in the number of files
created per second. Standard deviation is high as a small number of
iterations were much higher than the mean. The number of pages scanned by
zone_reclaim is reduced and kswapd is used for more work.
LARGE DD
3.0-rc6 3.0-rc6 3.0-rc6
vanilla zlcfirst zlcreconsider
download tar 59 ( 0.00%) 59 ( 0.00%) 55 ( 7.27%)
dd source files 527 ( 0.00%) 296 (78.04%) 320 (64.69%)
delete source 36 ( 0.00%) 19 (89.47%) 20 (80.00%)
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 125.03 118.98 122.01
Total Elapsed Time (seconds) 624.56 375.02 398.06
MMTests Statistics: vmstat
Page Ins 3594216 439368 407032
Page Outs 23380832 23380488 23377444
Swap Ins 0 0 0
Swap Outs 0 436 287
Direct pages scanned 17482342 69315973 82864918
Kswapd pages scanned 0 519123 575425
Kswapd pages reclaimed 0 466501 522487
Direct pages reclaimed 5858054 2732949 2712547
Kswapd efficiency 100% 89% 90%
Kswapd velocity 0.000 1384.254 1445.574
Direct efficiency 33% 3% 3%
Direct velocity 27991.453 184832.737 208171.929
Percentage direct scans 100% 99% 99%
Page writes by reclaim 0 5082 13917
Slabs scanned 17280 29952 35328
Direct inode steals 115257 1431122 332201
Kswapd inode steals 0 0 979532
This test downloads a large tarfile and copies it with dd a number of
times - similar to the most recent bug report I've dealt with. Time to
completion is reduced. The number of pages scanned directly is still
disturbingly high with a low efficiency but this is likely due to the
number of dirty pages encountered. The figures could probably be improved
with more work around how kswapd is used and how dirty pages are handled
but that is separate work and this result is significant on its own.
Streaming Mapped Writer
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 124.47 111.67 112.64
Total Elapsed Time (seconds) 2138.14 1816.30 1867.56
MMTests Statistics: vmstat
Page Ins 90760 89124 89516
Page Outs 121028340 120199524 120736696
Swap Ins 0 86 55
Swap Outs 0 0 0
Direct pages scanned 114989363 96461439 96330619
Kswapd pages scanned 56430948 56965763 57075875
Kswapd pages reclaimed 27743219 27752044 27766606
Direct pages reclaimed 49777 46884 36655
Kswapd efficiency 49% 48% 48%
Kswapd velocity 26392.541 31363.631 30561.736
Direct efficiency 0% 0% 0%
Direct velocity 53780.091 53108.759 51581.004
Percentage direct scans 67% 62% 62%
Page writes by reclaim 385 122 1513
Slabs scanned 43008 39040 42112
Direct inode steals 0 10 8
Kswapd inode steals 733 534 477
This test just creates a large file mapping and writes to it linearly.
Time to completion is again reduced.
The gains are mostly down to two things. In many cases, there is less
scanning as zone_reclaim simply gives up faster due to recent failures.
The second reason is that memory is used more efficiently. Instead of
scanning the preferred zone every time, the allocator falls back to
another zone and uses it instead improving overall memory utilisation.
This patch: initialise ZLC for first zone eligible for zone_reclaim.
The zonelist cache (ZLC) is used among other things to record if
zone_reclaim() failed for a particular zone recently. The intention is to
avoid a high cost scanning extremely long zonelists or scanning within the
zone uselessly.
Currently the zonelist cache is setup only after the first zone has been
considered and zone_reclaim() has been called. The objective was to avoid
a costly setup but zone_reclaim is itself quite expensive. If it is
failing regularly such as the first eligible zone having mostly mapped
pages, the cost in scanning and allocation stalls is far higher than the
ZLC initialisation step.
This patch initialises ZLC before the first eligible zone calls
zone_reclaim(). Once initialised, it is checked whether the zone failed
zone_reclaim recently. If it has, the zone is skipped. As the first zone
is now being checked, additional care has to be taken about zones marked
full. A zone can be marked "full" because it should not have enough
unmapped pages for zone_reclaim but this is excessive as direct reclaim or
kswapd may succeed where zone_reclaim fails. Only mark zones "full" after
zone_reclaim fails if it failed to reclaim enough pages after scanning.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-07-25 18:12:29 -06:00
|
|
|
continue;
|
2016-07-28 16:46:32 -06:00
|
|
|
case NODE_RECLAIM_FULL:
|
2009-06-16 16:33:22 -06:00
|
|
|
/* scanned but unreclaimable */
|
mm: page allocator: initialise ZLC for first zone eligible for zone_reclaim
There have been a small number of complaints about significant stalls
while copying large amounts of data on NUMA machines reported on a
distribution bugzilla. In these cases, zone_reclaim was enabled by
default due to large NUMA distances. In general, the complaints have not
been about the workload itself unless it was a file server (in which case
the recommendation was disable zone_reclaim).
The stalls are mostly due to significant amounts of time spent scanning
the preferred zone for pages to free. After a failure, it might fallback
to another node (as zonelists are often node-ordered rather than
zone-ordered) but stall quickly again when the next allocation attempt
occurs. In bad cases, each page allocated results in a full scan of the
preferred zone.
Patch 1 checks the preferred zone for recent allocation failure
which is particularly important if zone_reclaim has failed
recently. This avoids rescanning the zone in the near future and
instead falling back to another node. This may hurt node locality
in some cases but a failure to zone_reclaim is more expensive than
a remote access.
Patch 2 clears the zlc information after direct reclaim.
Otherwise, zone_reclaim can mark zones full, direct reclaim can
reclaim enough pages but the zone is still not considered for
allocation.
This was tested on a 24-thread 2-node x86_64 machine. The tests were
focused on large amounts of IO. All tests were bound to the CPUs on
node-0 to avoid disturbances due to processes being scheduled on different
nodes. The kernels tested are
3.0-rc6-vanilla Vanilla 3.0-rc6
zlcfirst Patch 1 applied
zlcreconsider Patches 1+2 applied
FS-Mark
./fs_mark -d /tmp/fsmark-10813 -D 100 -N 5000 -n 208 -L 35 -t 24 -S0 -s 524288
fsmark-3.0-rc6 3.0-rc6 3.0-rc6
vanilla zlcfirs zlcreconsider
Files/s min 54.90 ( 0.00%) 49.80 (-10.24%) 49.10 (-11.81%)
Files/s mean 100.11 ( 0.00%) 135.17 (25.94%) 146.93 (31.87%)
Files/s stddev 57.51 ( 0.00%) 138.97 (58.62%) 158.69 (63.76%)
Files/s max 361.10 ( 0.00%) 834.40 (56.72%) 802.40 (55.00%)
Overhead min 76704.00 ( 0.00%) 76501.00 ( 0.27%) 77784.00 (-1.39%)
Overhead mean 1485356.51 ( 0.00%) 1035797.83 (43.40%) 1594680.26 (-6.86%)
Overhead stddev 1848122.53 ( 0.00%) 881489.88 (109.66%) 1772354.90 ( 4.27%)
Overhead max 7989060.00 ( 0.00%) 3369118.00 (137.13%) 10135324.00 (-21.18%)
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 501.49 493.91 499.93
Total Elapsed Time (seconds) 2451.57 2257.48 2215.92
MMTests Statistics: vmstat
Page Ins 46268 63840 66008
Page Outs 90821596 90671128 88043732
Swap Ins 0 0 0
Swap Outs 0 0 0
Direct pages scanned 13091697 8966863 8971790
Kswapd pages scanned 0 1830011 1831116
Kswapd pages reclaimed 0 1829068 1829930
Direct pages reclaimed 13037777 8956828 8648314
Kswapd efficiency 100% 99% 99%
Kswapd velocity 0.000 810.643 826.346
Direct efficiency 99% 99% 96%
Direct velocity 5340.128 3972.068 4048.788
Percentage direct scans 100% 83% 83%
Page writes by reclaim 0 3 0
Slabs scanned 796672 720640 720256
Direct inode steals 7422667 7160012 7088638
Kswapd inode steals 0 1736840 2021238
Test completes far faster with a large increase in the number of files
created per second. Standard deviation is high as a small number of
iterations were much higher than the mean. The number of pages scanned by
zone_reclaim is reduced and kswapd is used for more work.
LARGE DD
3.0-rc6 3.0-rc6 3.0-rc6
vanilla zlcfirst zlcreconsider
download tar 59 ( 0.00%) 59 ( 0.00%) 55 ( 7.27%)
dd source files 527 ( 0.00%) 296 (78.04%) 320 (64.69%)
delete source 36 ( 0.00%) 19 (89.47%) 20 (80.00%)
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 125.03 118.98 122.01
Total Elapsed Time (seconds) 624.56 375.02 398.06
MMTests Statistics: vmstat
Page Ins 3594216 439368 407032
Page Outs 23380832 23380488 23377444
Swap Ins 0 0 0
Swap Outs 0 436 287
Direct pages scanned 17482342 69315973 82864918
Kswapd pages scanned 0 519123 575425
Kswapd pages reclaimed 0 466501 522487
Direct pages reclaimed 5858054 2732949 2712547
Kswapd efficiency 100% 89% 90%
Kswapd velocity 0.000 1384.254 1445.574
Direct efficiency 33% 3% 3%
Direct velocity 27991.453 184832.737 208171.929
Percentage direct scans 100% 99% 99%
Page writes by reclaim 0 5082 13917
Slabs scanned 17280 29952 35328
Direct inode steals 115257 1431122 332201
Kswapd inode steals 0 0 979532
This test downloads a large tarfile and copies it with dd a number of
times - similar to the most recent bug report I've dealt with. Time to
completion is reduced. The number of pages scanned directly is still
disturbingly high with a low efficiency but this is likely due to the
number of dirty pages encountered. The figures could probably be improved
with more work around how kswapd is used and how dirty pages are handled
but that is separate work and this result is significant on its own.
Streaming Mapped Writer
MMTests Statistics: duration
User/Sys Time Running Test (seconds) 124.47 111.67 112.64
Total Elapsed Time (seconds) 2138.14 1816.30 1867.56
MMTests Statistics: vmstat
Page Ins 90760 89124 89516
Page Outs 121028340 120199524 120736696
Swap Ins 0 86 55
Swap Outs 0 0 0
Direct pages scanned 114989363 96461439 96330619
Kswapd pages scanned 56430948 56965763 57075875
Kswapd pages reclaimed 27743219 27752044 27766606
Direct pages reclaimed 49777 46884 36655
Kswapd efficiency 49% 48% 48%
Kswapd velocity 26392.541 31363.631 30561.736
Direct efficiency 0% 0% 0%
Direct velocity 53780.091 53108.759 51581.004
Percentage direct scans 67% 62% 62%
Page writes by reclaim 385 122 1513
Slabs scanned 43008 39040 42112
Direct inode steals 0 10 8
Kswapd inode steals 733 534 477
This test just creates a large file mapping and writes to it linearly.
Time to completion is again reduced.
The gains are mostly down to two things. In many cases, there is less
scanning as zone_reclaim simply gives up faster due to recent failures.
The second reason is that memory is used more efficiently. Instead of
scanning the preferred zone every time, the allocator falls back to
another zone and uses it instead improving overall memory utilisation.
This patch: initialise ZLC for first zone eligible for zone_reclaim.
The zonelist cache (ZLC) is used among other things to record if
zone_reclaim() failed for a particular zone recently. The intention is to
avoid a high cost scanning extremely long zonelists or scanning within the
zone uselessly.
Currently the zonelist cache is setup only after the first zone has been
considered and zone_reclaim() has been called. The objective was to avoid
a costly setup but zone_reclaim is itself quite expensive. If it is
failing regularly such as the first eligible zone having mostly mapped
pages, the cost in scanning and allocation stalls is far higher than the
ZLC initialisation step.
This patch initialises ZLC before the first eligible zone calls
zone_reclaim(). Once initialised, it is checked whether the zone failed
zone_reclaim recently. If it has, the zone is skipped. As the first zone
is now being checked, additional care has to be taken about zones marked
full. A zone can be marked "full" because it should not have enough
unmapped pages for zone_reclaim but this is excessive as direct reclaim or
kswapd may succeed where zone_reclaim fails. Only mark zones "full" after
zone_reclaim fails if it failed to reclaim enough pages after scanning.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-07-25 18:12:29 -06:00
|
|
|
continue;
|
2009-06-16 16:33:22 -06:00
|
|
|
default:
|
|
|
|
/* did we reclaim enough */
|
2013-04-29 16:07:57 -06:00
|
|
|
if (zone_watermark_ok(zone, order, mark,
|
2016-05-19 18:14:13 -06:00
|
|
|
ac_classzone_idx(ac), alloc_flags))
|
2013-04-29 16:07:57 -06:00
|
|
|
goto try_this_zone;
|
|
|
|
|
|
|
|
continue;
|
2006-12-06 21:31:38 -07:00
|
|
|
}
|
2005-11-13 17:06:43 -07:00
|
|
|
}
|
|
|
|
|
2009-06-16 16:33:22 -06:00
|
|
|
try_this_zone:
|
2016-05-19 18:14:10 -06:00
|
|
|
page = buffered_rmqueue(ac->preferred_zoneref->zone, zone, order,
|
2015-11-06 17:28:37 -07:00
|
|
|
gfp_mask, alloc_flags, ac->migratetype);
|
mm: set page->pfmemalloc in prep_new_page()
The possibility of replacing the numerous parameters of alloc_pages*
functions with a single structure has been discussed when Minchan proposed
to expand the x86 kernel stack [1]. This series implements the change,
along with few more cleanups/microoptimizations.
The series is based on next-20150108 and I used gcc 4.8.3 20140627 on
openSUSE 13.2 for compiling. Config includess NUMA and COMPACTION.
The core change is the introduction of a new struct alloc_context, which looks
like this:
struct alloc_context {
struct zonelist *zonelist;
nodemask_t *nodemask;
struct zone *preferred_zone;
int classzone_idx;
int migratetype;
enum zone_type high_zoneidx;
};
All the contents is mostly constant, except that __alloc_pages_slowpath()
changes preferred_zone, classzone_idx and potentially zonelist. But
that's not a problem in case control returns to retry_cpuset: in
__alloc_pages_nodemask(), those will be reset to initial values again
(although it's a bit subtle). On the other hand, gfp_flags and alloc_info
mutate so much that it doesn't make sense to put them into alloc_context.
Still, the result is one parameter instead of up to 7. This is all in
Patch 2.
Patch 3 is a step to expand alloc_context usage out of page_alloc.c
itself. The function try_to_compact_pages() can also much benefit from
the parameter reduction, but it means the struct definition has to be
moved to a shared header.
Patch 1 should IMHO be included even if the rest is deemed not useful
enough. It improves maintainability and also has some code/stack
reduction. Patch 4 is OTOH a tiny optimization.
Overall bloat-o-meter results:
add/remove: 0/0 grow/shrink: 0/4 up/down: 0/-460 (-460)
function old new delta
nr_free_zone_pages 129 115 -14
__alloc_pages_direct_compact 329 256 -73
get_page_from_freelist 2670 2576 -94
__alloc_pages_nodemask 2564 2285 -279
try_to_compact_pages 582 579 -3
Overall stack sizes per ./scripts/checkstack.pl:
old new delta
get_page_from_freelist: 184 184 0
__alloc_pages_nodemask 248 200 -48
__alloc_pages_direct_c 40 - -40
try_to_compact_pages 72 72 0
-88
[1] http://marc.info/?l=linux-mm&m=140142462528257&w=2
This patch (of 4):
prep_new_page() sets almost everything in the struct page of the page
being allocated, except page->pfmemalloc. This is not obvious and has at
least once led to a bug where page->pfmemalloc was forgotten to be set
correctly, see commit 8fb74b9fb2b1 ("mm: compaction: partially revert
capture of suitable high-order page").
This patch moves the pfmemalloc setting to prep_new_page(), which means it
needs to gain alloc_flags parameter. The call to prep_new_page is moved
from buffered_rmqueue() to get_page_from_freelist(), which also leads to
simpler code. An obsolete comment for buffered_rmqueue() is replaced.
In addition to better maintainability there is a small reduction of code
and stack usage for get_page_from_freelist(), which inlines the other
functions involved.
add/remove: 0/0 grow/shrink: 0/1 up/down: 0/-145 (-145)
function old new delta
get_page_from_freelist 2670 2525 -145
Stack usage is reduced from 184 to 168 bytes.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.cz>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-11 16:25:38 -07:00
|
|
|
if (page) {
|
2016-05-19 18:14:35 -06:00
|
|
|
prep_new_page(page, order, gfp_mask, alloc_flags);
|
2015-11-06 17:28:37 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If this is a high-order atomic allocation then check
|
|
|
|
* if the pageblock should be reserved for the future
|
|
|
|
*/
|
|
|
|
if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
|
|
|
|
reserve_highatomic_pageblock(page, zone, order);
|
|
|
|
|
mm: set page->pfmemalloc in prep_new_page()
The possibility of replacing the numerous parameters of alloc_pages*
functions with a single structure has been discussed when Minchan proposed
to expand the x86 kernel stack [1]. This series implements the change,
along with few more cleanups/microoptimizations.
The series is based on next-20150108 and I used gcc 4.8.3 20140627 on
openSUSE 13.2 for compiling. Config includess NUMA and COMPACTION.
The core change is the introduction of a new struct alloc_context, which looks
like this:
struct alloc_context {
struct zonelist *zonelist;
nodemask_t *nodemask;
struct zone *preferred_zone;
int classzone_idx;
int migratetype;
enum zone_type high_zoneidx;
};
All the contents is mostly constant, except that __alloc_pages_slowpath()
changes preferred_zone, classzone_idx and potentially zonelist. But
that's not a problem in case control returns to retry_cpuset: in
__alloc_pages_nodemask(), those will be reset to initial values again
(although it's a bit subtle). On the other hand, gfp_flags and alloc_info
mutate so much that it doesn't make sense to put them into alloc_context.
Still, the result is one parameter instead of up to 7. This is all in
Patch 2.
Patch 3 is a step to expand alloc_context usage out of page_alloc.c
itself. The function try_to_compact_pages() can also much benefit from
the parameter reduction, but it means the struct definition has to be
moved to a shared header.
Patch 1 should IMHO be included even if the rest is deemed not useful
enough. It improves maintainability and also has some code/stack
reduction. Patch 4 is OTOH a tiny optimization.
Overall bloat-o-meter results:
add/remove: 0/0 grow/shrink: 0/4 up/down: 0/-460 (-460)
function old new delta
nr_free_zone_pages 129 115 -14
__alloc_pages_direct_compact 329 256 -73
get_page_from_freelist 2670 2576 -94
__alloc_pages_nodemask 2564 2285 -279
try_to_compact_pages 582 579 -3
Overall stack sizes per ./scripts/checkstack.pl:
old new delta
get_page_from_freelist: 184 184 0
__alloc_pages_nodemask 248 200 -48
__alloc_pages_direct_c 40 - -40
try_to_compact_pages 72 72 0
-88
[1] http://marc.info/?l=linux-mm&m=140142462528257&w=2
This patch (of 4):
prep_new_page() sets almost everything in the struct page of the page
being allocated, except page->pfmemalloc. This is not obvious and has at
least once led to a bug where page->pfmemalloc was forgotten to be set
correctly, see commit 8fb74b9fb2b1 ("mm: compaction: partially revert
capture of suitable high-order page").
This patch moves the pfmemalloc setting to prep_new_page(), which means it
needs to gain alloc_flags parameter. The call to prep_new_page is moved
from buffered_rmqueue() to get_page_from_freelist(), which also leads to
simpler code. An obsolete comment for buffered_rmqueue() is replaced.
In addition to better maintainability there is a small reduction of code
and stack usage for get_page_from_freelist(), which inlines the other
functions involved.
add/remove: 0/0 grow/shrink: 0/1 up/down: 0/-145 (-145)
function old new delta
get_page_from_freelist 2670 2525 -145
Stack usage is reduced from 184 to 168 bytes.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.cz>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-11 16:25:38 -07:00
|
|
|
return page;
|
|
|
|
}
|
2008-04-28 03:12:16 -06:00
|
|
|
}
|
[PATCH] memory page_alloc zonelist caching speedup
Optimize the critical zonelist scanning for free pages in the kernel memory
allocator by caching the zones that were found to be full recently, and
skipping them.
Remembers the zones in a zonelist that were short of free memory in the
last second. And it stashes a zone-to-node table in the zonelist struct,
to optimize that conversion (minimize its cache footprint.)
Recent changes:
This differs in a significant way from a similar patch that I
posted a week ago. Now, instead of having a nodemask_t of
recently full nodes, I have a bitmask of recently full zones.
This solves a problem that last weeks patch had, which on
systems with multiple zones per node (such as DMA zone) would
take seeing any of these zones full as meaning that all zones
on that node were full.
Also I changed names - from "zonelist faster" to "zonelist cache",
as that seemed to better convey what we're doing here - caching
some of the key zonelist state (for faster access.)
See below for some performance benchmark results. After all that
discussion with David on why I didn't need them, I went and got
some ;). I wanted to verify that I had not hurt the normal case
of memory allocation noticeably. At least for my one little
microbenchmark, I found (1) the normal case wasn't affected, and
(2) workloads that forced scanning across multiple nodes for
memory improved up to 10% fewer System CPU cycles and lower
elapsed clock time ('sys' and 'real'). Good. See details, below.
I didn't have the logic in get_page_from_freelist() for various
full nodes and zone reclaim failures correct. That should be
fixed up now - notice the new goto labels zonelist_scan,
this_zone_full, and try_next_zone, in get_page_from_freelist().
There are two reasons I persued this alternative, over some earlier
proposals that would have focused on optimizing the fake numa
emulation case by caching the last useful zone:
1) Contrary to what I said before, we (SGI, on large ia64 sn2 systems)
have seen real customer loads where the cost to scan the zonelist
was a problem, due to many nodes being full of memory before
we got to a node we could use. Or at least, I think we have.
This was related to me by another engineer, based on experiences
from some time past. So this is not guaranteed. Most likely, though.
The following approach should help such real numa systems just as
much as it helps fake numa systems, or any combination thereof.
2) The effort to distinguish fake from real numa, using node_distance,
so that we could cache a fake numa node and optimize choosing
it over equivalent distance fake nodes, while continuing to
properly scan all real nodes in distance order, was going to
require a nasty blob of zonelist and node distance munging.
The following approach has no new dependency on node distances or
zone sorting.
See comment in the patch below for a description of what it actually does.
Technical details of note (or controversy):
- See the use of "zlc_active" and "did_zlc_setup" below, to delay
adding any work for this new mechanism until we've looked at the
first zone in zonelist. I figured the odds of the first zone
having the memory we needed were high enough that we should just
look there, first, then get fancy only if we need to keep looking.
- Some odd hackery was needed to add items to struct zonelist, while
not tripping up the custom zonelists built by the mm/mempolicy.c
code for MPOL_BIND. My usual wordy comments below explain this.
Search for "MPOL_BIND".
- Some per-node data in the struct zonelist is now modified frequently,
with no locking. Multiple CPU cores on a node could hit and mangle
this data. The theory is that this is just performance hint data,
and the memory allocator will work just fine despite any such mangling.
The fields at risk are the struct 'zonelist_cache' fields 'fullzones'
(a bitmask) and 'last_full_zap' (unsigned long jiffies). It should
all be self correcting after at most a one second delay.
- This still does a linear scan of the same lengths as before. All
I've optimized is making the scan faster, not algorithmically
shorter. It is now able to scan a compact array of 'unsigned
short' in the case of many full nodes, so one cache line should
cover quite a few nodes, rather than each node hitting another
one or two new and distinct cache lines.
- If both Andi and Nick don't find this too complicated, I will be
(pleasantly) flabbergasted.
- I removed the comment claiming we only use one cachline's worth of
zonelist. We seem, at least in the fake numa case, to have put the
lie to that claim.
- I pay no attention to the various watermarks and such in this performance
hint. A node could be marked full for one watermark, and then skipped
over when searching for a page using a different watermark. I think
that's actually quite ok, as it will tend to slightly increase the
spreading of memory over other nodes, away from a memory stressed node.
===============
Performance - some benchmark results and analysis:
This benchmark runs a memory hog program that uses multiple
threads to touch alot of memory as quickly as it can.
Multiple runs were made, touching 12, 38, 64 or 90 GBytes out of
the total 96 GBytes on the system, and using 1, 19, 37, or 55
threads (on a 56 CPU system.) System, user and real (elapsed)
timings were recorded for each run, shown in units of seconds,
in the table below.
Two kernels were tested - 2.6.18-mm3 and the same kernel with
this zonelist caching patch added. The table also shows the
percentage improvement the zonelist caching sys time is over
(lower than) the stock *-mm kernel.
number 2.6.18-mm3 zonelist-cache delta (< 0 good) percent
GBs N ------------ -------------- ---------------- systime
mem threads sys user real sys user real sys user real better
12 1 153 24 177 151 24 176 -2 0 -1 1%
12 19 99 22 8 99 22 8 0 0 0 0%
12 37 111 25 6 112 25 6 1 0 0 -0%
12 55 115 25 5 110 23 5 -5 -2 0 4%
38 1 502 74 576 497 73 570 -5 -1 -6 0%
38 19 426 78 48 373 76 39 -53 -2 -9 12%
38 37 544 83 36 547 82 36 3 -1 0 -0%
38 55 501 77 23 511 80 24 10 3 1 -1%
64 1 917 125 1042 890 124 1014 -27 -1 -28 2%
64 19 1118 138 119 965 141 103 -153 3 -16 13%
64 37 1202 151 94 1136 150 81 -66 -1 -13 5%
64 55 1118 141 61 1072 140 58 -46 -1 -3 4%
90 1 1342 177 1519 1275 174 1450 -67 -3 -69 4%
90 19 2392 199 192 2116 189 176 -276 -10 -16 11%
90 37 3313 238 175 2972 225 145 -341 -13 -30 10%
90 55 1948 210 104 1843 213 100 -105 3 -4 5%
Notes:
1) This test ran a memory hog program that started a specified number N of
threads, and had each thread allocate and touch 1/N'th of
the total memory to be used in the test run in a single loop,
writing a constant word to memory, one store every 4096 bytes.
Watching this test during some earlier trial runs, I would see
each of these threads sit down on one CPU and stay there, for
the remainder of the pass, a different CPU for each thread.
2) The 'real' column is not comparable to the 'sys' or 'user' columns.
The 'real' column is seconds wall clock time elapsed, from beginning
to end of that test pass. The 'sys' and 'user' columns are total
CPU seconds spent on that test pass. For a 19 thread test run,
for example, the sum of 'sys' and 'user' could be up to 19 times the
number of 'real' elapsed wall clock seconds.
3) Tests were run on a fresh, single-user boot, to minimize the amount
of memory already in use at the start of the test, and to minimize
the amount of background activity that might interfere.
4) Tests were done on a 56 CPU, 28 Node system with 96 GBytes of RAM.
5) Notice that the 'real' time gets large for the single thread runs, even
though the measured 'sys' and 'user' times are modest. I'm not sure what
that means - probably something to do with it being slow for one thread to
be accessing memory along ways away. Perhaps the fake numa system, running
ostensibly the same workload, would not show this substantial degradation
of 'real' time for one thread on many nodes -- lets hope not.
6) The high thread count passes (one thread per CPU - on 55 of 56 CPUs)
ran quite efficiently, as one might expect. Each pair of threads needed
to allocate and touch the memory on the node the two threads shared, a
pleasantly parallizable workload.
7) The intermediate thread count passes, when asking for alot of memory forcing
them to go to a few neighboring nodes, improved the most with this zonelist
caching patch.
Conclusions:
* This zonelist cache patch probably makes little difference one way or the
other for most workloads on real numa hardware, if those workloads avoid
heavy off node allocations.
* For memory intensive workloads requiring substantial off-node allocations
on real numa hardware, this patch improves both kernel and elapsed timings
up to ten per-cent.
* For fake numa systems, I'm optimistic, but will have to leave that up to
Rohit Seth to actually test (once I get him a 2.6.18 backport.)
Signed-off-by: Paul Jackson <pj@sgi.com>
Cc: Rohit Seth <rohitseth@google.com>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Cc: David Rientjes <rientjes@cs.washington.edu>
Cc: Paul Menage <menage@google.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-06 21:31:48 -07:00
|
|
|
|
2014-08-06 17:07:22 -06:00
|
|
|
return NULL;
|
2005-06-21 18:14:41 -06:00
|
|
|
}
|
|
|
|
|
2011-03-22 17:30:47 -06:00
|
|
|
/*
|
|
|
|
* Large machines with many possible nodes should not always dump per-node
|
|
|
|
* meminfo in irq context.
|
|
|
|
*/
|
|
|
|
static inline bool should_suppress_show_mem(void)
|
|
|
|
{
|
|
|
|
bool ret = false;
|
|
|
|
|
|
|
|
#if NODES_SHIFT > 8
|
|
|
|
ret = in_interrupt();
|
|
|
|
#endif
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2011-05-24 18:12:16 -06:00
|
|
|
static DEFINE_RATELIMIT_STATE(nopage_rs,
|
|
|
|
DEFAULT_RATELIMIT_INTERVAL,
|
|
|
|
DEFAULT_RATELIMIT_BURST);
|
|
|
|
|
2015-11-06 17:29:57 -07:00
|
|
|
void warn_alloc_failed(gfp_t gfp_mask, unsigned int order, const char *fmt, ...)
|
2011-05-24 18:12:16 -06:00
|
|
|
{
|
|
|
|
unsigned int filter = SHOW_MEM_FILTER_NODES;
|
|
|
|
|
2012-01-10 16:07:28 -07:00
|
|
|
if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
|
|
|
|
debug_guardpage_minorder() > 0)
|
2011-05-24 18:12:16 -06:00
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This documents exceptions given to allocations in certain
|
|
|
|
* contexts that are allowed to allocate outside current's set
|
|
|
|
* of allowed nodes.
|
|
|
|
*/
|
|
|
|
if (!(gfp_mask & __GFP_NOMEMALLOC))
|
|
|
|
if (test_thread_flag(TIF_MEMDIE) ||
|
|
|
|
(current->flags & (PF_MEMALLOC | PF_EXITING)))
|
|
|
|
filter &= ~SHOW_MEM_FILTER_NODES;
|
2015-11-06 17:28:21 -07:00
|
|
|
if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
|
2011-05-24 18:12:16 -06:00
|
|
|
filter &= ~SHOW_MEM_FILTER_NODES;
|
|
|
|
|
|
|
|
if (fmt) {
|
2011-10-31 18:08:35 -06:00
|
|
|
struct va_format vaf;
|
|
|
|
va_list args;
|
|
|
|
|
2011-05-24 18:12:16 -06:00
|
|
|
va_start(args, fmt);
|
2011-10-31 18:08:35 -06:00
|
|
|
|
|
|
|
vaf.fmt = fmt;
|
|
|
|
vaf.va = &args;
|
|
|
|
|
|
|
|
pr_warn("%pV", &vaf);
|
|
|
|
|
2011-05-24 18:12:16 -06:00
|
|
|
va_end(args);
|
|
|
|
}
|
|
|
|
|
2016-03-15 15:56:02 -06:00
|
|
|
pr_warn("%s: page allocation failure: order:%u, mode:%#x(%pGg)\n",
|
|
|
|
current->comm, order, gfp_mask, &gfp_mask);
|
2011-05-24 18:12:16 -06:00
|
|
|
dump_stack();
|
|
|
|
if (!should_suppress_show_mem())
|
|
|
|
show_mem(filter);
|
|
|
|
}
|
|
|
|
|
2009-06-16 16:31:57 -06:00
|
|
|
static inline struct page *
|
|
|
|
__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
|
2015-02-11 16:25:41 -07:00
|
|
|
const struct alloc_context *ac, unsigned long *did_some_progress)
|
2009-06-16 16:31:57 -06:00
|
|
|
{
|
2015-09-08 16:00:36 -06:00
|
|
|
struct oom_control oc = {
|
|
|
|
.zonelist = ac->zonelist,
|
|
|
|
.nodemask = ac->nodemask,
|
2016-07-26 16:22:33 -06:00
|
|
|
.memcg = NULL,
|
2015-09-08 16:00:36 -06:00
|
|
|
.gfp_mask = gfp_mask,
|
|
|
|
.order = order,
|
|
|
|
};
|
2009-06-16 16:31:57 -06:00
|
|
|
struct page *page;
|
|
|
|
|
2015-01-26 13:58:32 -07:00
|
|
|
*did_some_progress = 0;
|
|
|
|
|
|
|
|
/*
|
2015-06-24 17:57:19 -06:00
|
|
|
* Acquire the oom lock. If that fails, somebody else is
|
|
|
|
* making progress for us.
|
2015-01-26 13:58:32 -07:00
|
|
|
*/
|
2015-06-24 17:57:19 -06:00
|
|
|
if (!mutex_trylock(&oom_lock)) {
|
2015-01-26 13:58:32 -07:00
|
|
|
*did_some_progress = 1;
|
2009-06-16 16:31:57 -06:00
|
|
|
schedule_timeout_uninterruptible(1);
|
2005-04-16 16:20:36 -06:00
|
|
|
return NULL;
|
|
|
|
}
|
2005-11-17 13:35:02 -07:00
|
|
|
|
2009-06-16 16:31:57 -06:00
|
|
|
/*
|
|
|
|
* Go through the zonelist yet one more time, keep very high watermark
|
|
|
|
* here, this is only to catch a parallel oom killing, we must fail if
|
|
|
|
* we're still under heavy pressure.
|
|
|
|
*/
|
2015-02-11 16:25:41 -07:00
|
|
|
page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order,
|
|
|
|
ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
|
2005-11-13 17:06:43 -07:00
|
|
|
if (page)
|
2009-06-16 16:31:57 -06:00
|
|
|
goto out;
|
|
|
|
|
2009-12-15 17:45:33 -07:00
|
|
|
if (!(gfp_mask & __GFP_NOFAIL)) {
|
2015-01-26 13:58:32 -07:00
|
|
|
/* Coredumps can quickly deplete all memory reserves */
|
|
|
|
if (current->flags & PF_DUMPCORE)
|
|
|
|
goto out;
|
2009-12-15 17:45:33 -07:00
|
|
|
/* The OOM killer will not help higher order allocs */
|
|
|
|
if (order > PAGE_ALLOC_COSTLY_ORDER)
|
|
|
|
goto out;
|
2010-08-09 18:18:54 -06:00
|
|
|
/* The OOM killer does not needlessly kill tasks for lowmem */
|
2015-02-11 16:25:41 -07:00
|
|
|
if (ac->high_zoneidx < ZONE_NORMAL)
|
2010-08-09 18:18:54 -06:00
|
|
|
goto out;
|
2015-06-24 17:57:21 -06:00
|
|
|
if (pm_suspended_storage())
|
|
|
|
goto out;
|
2016-05-19 18:13:09 -06:00
|
|
|
/*
|
|
|
|
* XXX: GFP_NOFS allocations should rather fail than rely on
|
|
|
|
* other request to make a forward progress.
|
|
|
|
* We are in an unfortunate situation where out_of_memory cannot
|
|
|
|
* do much for this context but let's try it to at least get
|
|
|
|
* access to memory reserved if the current task is killed (see
|
|
|
|
* out_of_memory). Once filesystems are ready to handle allocation
|
|
|
|
* failures more gracefully we should just bail out here.
|
|
|
|
*/
|
|
|
|
|
mm: remove GFP_THISNODE
NOTE: this is not about __GFP_THISNODE, this is only about GFP_THISNODE.
GFP_THISNODE is a secret combination of gfp bits that have different
behavior than expected. It is a combination of __GFP_THISNODE,
__GFP_NORETRY, and __GFP_NOWARN and is special-cased in the page
allocator slowpath to fail without trying reclaim even though it may be
used in combination with __GFP_WAIT.
An example of the problem this creates: commit e97ca8e5b864 ("mm: fix
GFP_THISNODE callers and clarify") fixed up many users of GFP_THISNODE
that really just wanted __GFP_THISNODE. The problem doesn't end there,
however, because even it was a no-op for alloc_misplaced_dst_page(),
which also sets __GFP_NORETRY and __GFP_NOWARN, and
migrate_misplaced_transhuge_page(), where __GFP_NORETRY and __GFP_NOWAIT
is set in GFP_TRANSHUGE. Converting GFP_THISNODE to __GFP_THISNODE is a
no-op in these cases since the page allocator special-cases
__GFP_THISNODE && __GFP_NORETRY && __GFP_NOWARN.
It's time to just remove GFP_THISNODE entirely. We leave __GFP_THISNODE
to restrict an allocation to a local node, but remove GFP_THISNODE and
its obscurity. Instead, we require that a caller clear __GFP_WAIT if it
wants to avoid reclaim.
This allows the aforementioned functions to actually reclaim as they
should. It also enables any future callers that want to do
__GFP_THISNODE but also __GFP_NORETRY && __GFP_NOWARN to reclaim. The
rule is simple: if you don't want to reclaim, then don't set __GFP_WAIT.
Aside: ovs_flow_stats_update() really wants to avoid reclaim as well, so
it is unchanged.
Signed-off-by: David Rientjes <rientjes@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Acked-by: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Pravin Shelar <pshelar@nicira.com>
Cc: Jarno Rajahalme <jrajahalme@nicira.com>
Cc: Li Zefan <lizefan@huawei.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 16:46:55 -06:00
|
|
|
/* The OOM killer may not free memory on a specific node */
|
2009-12-15 17:45:33 -07:00
|
|
|
if (gfp_mask & __GFP_THISNODE)
|
|
|
|
goto out;
|
|
|
|
}
|
2009-06-16 16:31:57 -06:00
|
|
|
/* Exhausted what can be done so it's blamo time */
|
2016-01-14 16:20:36 -07:00
|
|
|
if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
|
2015-02-11 16:26:24 -07:00
|
|
|
*did_some_progress = 1;
|
2016-01-14 16:20:36 -07:00
|
|
|
|
|
|
|
if (gfp_mask & __GFP_NOFAIL) {
|
|
|
|
page = get_page_from_freelist(gfp_mask, order,
|
|
|
|
ALLOC_NO_WATERMARKS|ALLOC_CPUSET, ac);
|
|
|
|
/*
|
|
|
|
* fallback to ignore cpuset restriction if our nodes
|
|
|
|
* are depleted
|
|
|
|
*/
|
|
|
|
if (!page)
|
|
|
|
page = get_page_from_freelist(gfp_mask, order,
|
|
|
|
ALLOC_NO_WATERMARKS, ac);
|
|
|
|
}
|
|
|
|
}
|
2009-06-16 16:31:57 -06:00
|
|
|
out:
|
2015-06-24 17:57:19 -06:00
|
|
|
mutex_unlock(&oom_lock);
|
2009-06-16 16:31:57 -06:00
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
2016-05-20 17:57:06 -06:00
|
|
|
/*
|
|
|
|
* Maximum number of compaction retries wit a progress before OOM
|
|
|
|
* killer is consider as the only way to move forward.
|
|
|
|
*/
|
|
|
|
#define MAX_COMPACT_RETRIES 16
|
|
|
|
|
2010-05-24 15:32:30 -06:00
|
|
|
#ifdef CONFIG_COMPACTION
|
|
|
|
/* Try memory compaction for high-order allocations before reclaim */
|
|
|
|
static struct page *
|
|
|
|
__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
|
2016-05-19 18:13:38 -06:00
|
|
|
unsigned int alloc_flags, const struct alloc_context *ac,
|
2016-07-28 16:49:28 -06:00
|
|
|
enum compact_priority prio, enum compact_result *compact_result)
|
2010-05-24 15:32:30 -06:00
|
|
|
{
|
2014-10-09 16:27:04 -06:00
|
|
|
struct page *page;
|
mm, compaction: defer each zone individually instead of preferred zone
When direct sync compaction is often unsuccessful, it may become deferred
for some time to avoid further useless attempts, both sync and async.
Successful high-order allocations un-defer compaction, while further
unsuccessful compaction attempts prolong the compaction deferred period.
Currently the checking and setting deferred status is performed only on
the preferred zone of the allocation that invoked direct compaction. But
compaction itself is attempted on all eligible zones in the zonelist, so
the behavior is suboptimal and may lead both to scenarios where 1)
compaction is attempted uselessly, or 2) where it's not attempted despite
good chances of succeeding, as shown on the examples below:
1) A direct compaction with Normal preferred zone failed and set
deferred compaction for the Normal zone. Another unrelated direct
compaction with DMA32 as preferred zone will attempt to compact DMA32
zone even though the first compaction attempt also included DMA32 zone.
In another scenario, compaction with Normal preferred zone failed to
compact Normal zone, but succeeded in the DMA32 zone, so it will not
defer compaction. In the next attempt, it will try Normal zone which
will fail again, instead of skipping Normal zone and trying DMA32
directly.
2) Kswapd will balance DMA32 zone and reset defer status based on
watermarks looking good. A direct compaction with preferred Normal
zone will skip compaction of all zones including DMA32 because Normal
was still deferred. The allocation might have succeeded in DMA32, but
won't.
This patch makes compaction deferring work on individual zone basis
instead of preferred zone. For each zone, it checks compaction_deferred()
to decide if the zone should be skipped. If watermarks fail after
compacting the zone, defer_compaction() is called. The zone where
watermarks passed can still be deferred when the allocation attempt is
unsuccessful. When allocation is successful, compaction_defer_reset() is
called for the zone containing the allocated page. This approach should
approximate calling defer_compaction() only on zones where compaction was
attempted and did not yield allocated page. There might be corner cases
but that is inevitable as long as the decision to stop compacting dues not
guarantee that a page will be allocated.
Due to a new COMPACT_DEFERRED return value, some functions relying
implicitly on COMPACT_SKIPPED = 0 had to be updated, with comments made
more accurate. The did_some_progress output parameter of
__alloc_pages_direct_compact() is removed completely, as the caller
actually does not use it after compaction sets it - it is only considered
when direct reclaim sets it.
During testing on a two-node machine with a single very small Normal zone
on node 1, this patch has improved success rates in stress-highalloc
mmtests benchmark. The success here were previously made worse by commit
3a025760fc15 ("mm: page_alloc: spill to remote nodes before waking
kswapd") as kswapd was no longer resetting often enough the deferred
compaction for the Normal zone, and DMA32 zones on both nodes were thus
not considered for compaction. On different machine, success rates were
improved with __GFP_NO_KSWAPD allocations.
[akpm@linux-foundation.org: fix CONFIG_COMPACTION=n build]
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Minchan Kim <minchan@kernel.org>
Reviewed-by: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-09 16:27:02 -06:00
|
|
|
|
|
|
|
if (!order)
|
2012-01-12 18:19:41 -07:00
|
|
|
return NULL;
|
|
|
|
|
2011-01-13 16:47:32 -07:00
|
|
|
current->flags |= PF_MEMALLOC;
|
2016-05-20 17:56:53 -06:00
|
|
|
*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
|
mm, compaction: simplify contended compaction handling
Async compaction detects contention either due to failing trylock on
zone->lock or lru_lock, or by need_resched(). Since 1f9efdef4f3f ("mm,
compaction: khugepaged should not give up due to need_resched()") the
code got quite complicated to distinguish these two up to the
__alloc_pages_slowpath() level, so different decisions could be taken
for khugepaged allocations.
After the recent changes, khugepaged allocations don't check for
contended compaction anymore, so we again don't need to distinguish lock
and sched contention, and simplify the current convoluted code a lot.
However, I believe it's also possible to simplify even more and
completely remove the check for contended compaction after the initial
async compaction for costly orders, which was originally aimed at THP
page fault allocations. There are several reasons why this can be done
now:
- with the new defaults, THP page faults no longer do reclaim/compaction at
all, unless the system admin has overridden the default, or application has
indicated via madvise that it can benefit from THP's. In both cases, it
means that the potential extra latency is expected and worth the benefits.
- even if reclaim/compaction proceeds after this patch where it previously
wouldn't, the second compaction attempt is still async and will detect the
contention and back off, if the contention persists
- there are still heuristics like deferred compaction and pageblock skip bits
in place that prevent excessive THP page fault latencies
Link: http://lkml.kernel.org/r/20160721073614.24395-9-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:30 -06:00
|
|
|
prio);
|
2011-01-13 16:47:32 -07:00
|
|
|
current->flags &= ~PF_MEMALLOC;
|
2010-05-24 15:32:30 -06:00
|
|
|
|
2016-05-20 17:56:53 -06:00
|
|
|
if (*compact_result <= COMPACT_INACTIVE)
|
2014-10-09 16:27:04 -06:00
|
|
|
return NULL;
|
mm, compaction: defer each zone individually instead of preferred zone
When direct sync compaction is often unsuccessful, it may become deferred
for some time to avoid further useless attempts, both sync and async.
Successful high-order allocations un-defer compaction, while further
unsuccessful compaction attempts prolong the compaction deferred period.
Currently the checking and setting deferred status is performed only on
the preferred zone of the allocation that invoked direct compaction. But
compaction itself is attempted on all eligible zones in the zonelist, so
the behavior is suboptimal and may lead both to scenarios where 1)
compaction is attempted uselessly, or 2) where it's not attempted despite
good chances of succeeding, as shown on the examples below:
1) A direct compaction with Normal preferred zone failed and set
deferred compaction for the Normal zone. Another unrelated direct
compaction with DMA32 as preferred zone will attempt to compact DMA32
zone even though the first compaction attempt also included DMA32 zone.
In another scenario, compaction with Normal preferred zone failed to
compact Normal zone, but succeeded in the DMA32 zone, so it will not
defer compaction. In the next attempt, it will try Normal zone which
will fail again, instead of skipping Normal zone and trying DMA32
directly.
2) Kswapd will balance DMA32 zone and reset defer status based on
watermarks looking good. A direct compaction with preferred Normal
zone will skip compaction of all zones including DMA32 because Normal
was still deferred. The allocation might have succeeded in DMA32, but
won't.
This patch makes compaction deferring work on individual zone basis
instead of preferred zone. For each zone, it checks compaction_deferred()
to decide if the zone should be skipped. If watermarks fail after
compacting the zone, defer_compaction() is called. The zone where
watermarks passed can still be deferred when the allocation attempt is
unsuccessful. When allocation is successful, compaction_defer_reset() is
called for the zone containing the allocated page. This approach should
approximate calling defer_compaction() only on zones where compaction was
attempted and did not yield allocated page. There might be corner cases
but that is inevitable as long as the decision to stop compacting dues not
guarantee that a page will be allocated.
Due to a new COMPACT_DEFERRED return value, some functions relying
implicitly on COMPACT_SKIPPED = 0 had to be updated, with comments made
more accurate. The did_some_progress output parameter of
__alloc_pages_direct_compact() is removed completely, as the caller
actually does not use it after compaction sets it - it is only considered
when direct reclaim sets it.
During testing on a two-node machine with a single very small Normal zone
on node 1, this patch has improved success rates in stress-highalloc
mmtests benchmark. The success here were previously made worse by commit
3a025760fc15 ("mm: page_alloc: spill to remote nodes before waking
kswapd") as kswapd was no longer resetting often enough the deferred
compaction for the Normal zone, and DMA32 zones on both nodes were thus
not considered for compaction. On different machine, success rates were
improved with __GFP_NO_KSWAPD allocations.
[akpm@linux-foundation.org: fix CONFIG_COMPACTION=n build]
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Minchan Kim <minchan@kernel.org>
Reviewed-by: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-09 16:27:02 -06:00
|
|
|
|
2014-10-09 16:27:04 -06:00
|
|
|
/*
|
|
|
|
* At least in one zone compaction wasn't deferred or skipped, so let's
|
|
|
|
* count a compaction stall
|
|
|
|
*/
|
|
|
|
count_vm_event(COMPACTSTALL);
|
2013-01-11 15:32:16 -07:00
|
|
|
|
2016-07-28 16:49:13 -06:00
|
|
|
page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
|
mm, compaction: defer each zone individually instead of preferred zone
When direct sync compaction is often unsuccessful, it may become deferred
for some time to avoid further useless attempts, both sync and async.
Successful high-order allocations un-defer compaction, while further
unsuccessful compaction attempts prolong the compaction deferred period.
Currently the checking and setting deferred status is performed only on
the preferred zone of the allocation that invoked direct compaction. But
compaction itself is attempted on all eligible zones in the zonelist, so
the behavior is suboptimal and may lead both to scenarios where 1)
compaction is attempted uselessly, or 2) where it's not attempted despite
good chances of succeeding, as shown on the examples below:
1) A direct compaction with Normal preferred zone failed and set
deferred compaction for the Normal zone. Another unrelated direct
compaction with DMA32 as preferred zone will attempt to compact DMA32
zone even though the first compaction attempt also included DMA32 zone.
In another scenario, compaction with Normal preferred zone failed to
compact Normal zone, but succeeded in the DMA32 zone, so it will not
defer compaction. In the next attempt, it will try Normal zone which
will fail again, instead of skipping Normal zone and trying DMA32
directly.
2) Kswapd will balance DMA32 zone and reset defer status based on
watermarks looking good. A direct compaction with preferred Normal
zone will skip compaction of all zones including DMA32 because Normal
was still deferred. The allocation might have succeeded in DMA32, but
won't.
This patch makes compaction deferring work on individual zone basis
instead of preferred zone. For each zone, it checks compaction_deferred()
to decide if the zone should be skipped. If watermarks fail after
compacting the zone, defer_compaction() is called. The zone where
watermarks passed can still be deferred when the allocation attempt is
unsuccessful. When allocation is successful, compaction_defer_reset() is
called for the zone containing the allocated page. This approach should
approximate calling defer_compaction() only on zones where compaction was
attempted and did not yield allocated page. There might be corner cases
but that is inevitable as long as the decision to stop compacting dues not
guarantee that a page will be allocated.
Due to a new COMPACT_DEFERRED return value, some functions relying
implicitly on COMPACT_SKIPPED = 0 had to be updated, with comments made
more accurate. The did_some_progress output parameter of
__alloc_pages_direct_compact() is removed completely, as the caller
actually does not use it after compaction sets it - it is only considered
when direct reclaim sets it.
During testing on a two-node machine with a single very small Normal zone
on node 1, this patch has improved success rates in stress-highalloc
mmtests benchmark. The success here were previously made worse by commit
3a025760fc15 ("mm: page_alloc: spill to remote nodes before waking
kswapd") as kswapd was no longer resetting often enough the deferred
compaction for the Normal zone, and DMA32 zones on both nodes were thus
not considered for compaction. On different machine, success rates were
improved with __GFP_NO_KSWAPD allocations.
[akpm@linux-foundation.org: fix CONFIG_COMPACTION=n build]
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Minchan Kim <minchan@kernel.org>
Reviewed-by: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-09 16:27:02 -06:00
|
|
|
|
2014-10-09 16:27:04 -06:00
|
|
|
if (page) {
|
|
|
|
struct zone *zone = page_zone(page);
|
mm, compaction: defer each zone individually instead of preferred zone
When direct sync compaction is often unsuccessful, it may become deferred
for some time to avoid further useless attempts, both sync and async.
Successful high-order allocations un-defer compaction, while further
unsuccessful compaction attempts prolong the compaction deferred period.
Currently the checking and setting deferred status is performed only on
the preferred zone of the allocation that invoked direct compaction. But
compaction itself is attempted on all eligible zones in the zonelist, so
the behavior is suboptimal and may lead both to scenarios where 1)
compaction is attempted uselessly, or 2) where it's not attempted despite
good chances of succeeding, as shown on the examples below:
1) A direct compaction with Normal preferred zone failed and set
deferred compaction for the Normal zone. Another unrelated direct
compaction with DMA32 as preferred zone will attempt to compact DMA32
zone even though the first compaction attempt also included DMA32 zone.
In another scenario, compaction with Normal preferred zone failed to
compact Normal zone, but succeeded in the DMA32 zone, so it will not
defer compaction. In the next attempt, it will try Normal zone which
will fail again, instead of skipping Normal zone and trying DMA32
directly.
2) Kswapd will balance DMA32 zone and reset defer status based on
watermarks looking good. A direct compaction with preferred Normal
zone will skip compaction of all zones including DMA32 because Normal
was still deferred. The allocation might have succeeded in DMA32, but
won't.
This patch makes compaction deferring work on individual zone basis
instead of preferred zone. For each zone, it checks compaction_deferred()
to decide if the zone should be skipped. If watermarks fail after
compacting the zone, defer_compaction() is called. The zone where
watermarks passed can still be deferred when the allocation attempt is
unsuccessful. When allocation is successful, compaction_defer_reset() is
called for the zone containing the allocated page. This approach should
approximate calling defer_compaction() only on zones where compaction was
attempted and did not yield allocated page. There might be corner cases
but that is inevitable as long as the decision to stop compacting dues not
guarantee that a page will be allocated.
Due to a new COMPACT_DEFERRED return value, some functions relying
implicitly on COMPACT_SKIPPED = 0 had to be updated, with comments made
more accurate. The did_some_progress output parameter of
__alloc_pages_direct_compact() is removed completely, as the caller
actually does not use it after compaction sets it - it is only considered
when direct reclaim sets it.
During testing on a two-node machine with a single very small Normal zone
on node 1, this patch has improved success rates in stress-highalloc
mmtests benchmark. The success here were previously made worse by commit
3a025760fc15 ("mm: page_alloc: spill to remote nodes before waking
kswapd") as kswapd was no longer resetting often enough the deferred
compaction for the Normal zone, and DMA32 zones on both nodes were thus
not considered for compaction. On different machine, success rates were
improved with __GFP_NO_KSWAPD allocations.
[akpm@linux-foundation.org: fix CONFIG_COMPACTION=n build]
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Minchan Kim <minchan@kernel.org>
Reviewed-by: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-09 16:27:02 -06:00
|
|
|
|
2014-10-09 16:27:04 -06:00
|
|
|
zone->compact_blockskip_flush = false;
|
|
|
|
compaction_defer_reset(zone, order, true);
|
|
|
|
count_vm_event(COMPACTSUCCESS);
|
|
|
|
return page;
|
|
|
|
}
|
2010-05-24 15:32:30 -06:00
|
|
|
|
2014-10-09 16:27:04 -06:00
|
|
|
/*
|
|
|
|
* It's bad if compaction run occurs and fails. The most likely reason
|
|
|
|
* is that pages exist, but not enough to satisfy watermarks.
|
|
|
|
*/
|
|
|
|
count_vm_event(COMPACTFAIL);
|
2012-01-12 18:19:41 -07:00
|
|
|
|
2014-10-09 16:27:04 -06:00
|
|
|
cond_resched();
|
2010-05-24 15:32:30 -06:00
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
2016-05-20 17:57:06 -06:00
|
|
|
|
|
|
|
static inline bool
|
2016-05-20 17:57:12 -06:00
|
|
|
should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
|
2016-07-28 16:49:28 -06:00
|
|
|
enum compact_result compact_result,
|
|
|
|
enum compact_priority *compact_priority,
|
2016-05-20 17:57:06 -06:00
|
|
|
int compaction_retries)
|
|
|
|
{
|
mm: consider compaction feedback also for costly allocation
PAGE_ALLOC_COSTLY_ORDER retry logic is mostly handled inside
should_reclaim_retry currently where we decide to not retry after at
least order worth of pages were reclaimed or the watermark check for at
least one zone would succeed after reclaiming all pages if the reclaim
hasn't made any progress. Compaction feedback is mostly ignored and we
just try to make sure that the compaction did at least something before
giving up.
The first condition was added by a41f24ea9fd6 ("page allocator: smarter
retry of costly-order allocations) and it assumed that lumpy reclaim
could have created a page of the sufficient order. Lumpy reclaim, has
been removed quite some time ago so the assumption doesn't hold anymore.
Remove the check for the number of reclaimed pages and rely on the
compaction feedback solely. should_reclaim_retry now only makes sure
that we keep retrying reclaim for high order pages only if they are
hidden by watermaks so order-0 reclaim makes really sense.
should_compact_retry now keeps retrying even for the costly allocations.
The number of retries is reduced wrt. !costly requests because they are
less important and harder to grant and so their pressure shouldn't cause
contention for other requests or cause an over reclaim. We also do not
reset no_progress_loops for costly request to make sure we do not keep
reclaiming too agressively.
This has been tested by running a process which fragments memory:
- compact memory
- mmap large portion of the memory (1920M on 2GRAM machine with 2G
of swapspace)
- MADV_DONTNEED single page in PAGE_SIZE*((1UL<<MAX_ORDER)-1)
steps until certain amount of memory is freed (250M in my test)
and reduce the step to (step / 2) + 1 after reaching the end of
the mapping
- then run a script which populates the page cache 2G (MemTotal)
from /dev/zero to a new file
And then tries to allocate
nr_hugepages=$(awk '/MemAvailable/{printf "%d\n", $2/(2*1024)}' /proc/meminfo)
huge pages.
root@test1:~# echo 1 > /proc/sys/vm/overcommit_memory;echo 1 > /proc/sys/vm/compact_memory; ./fragment-mem-and-run /root/alloc_hugepages.sh 1920M 250M
Node 0, zone DMA 31 28 31 10 2 0 2 1 2 3 1
Node 0, zone DMA32 437 319 171 50 28 25 20 16 16 14 437
* This is the /proc/buddyinfo after the compaction
Done fragmenting. size=2013265920 freed=262144000
Node 0, zone DMA 165 48 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 35109 14575 185 51 41 12 6 0 0 0 0
* /proc/buddyinfo after memory got fragmented
Executing "/root/alloc_hugepages.sh"
Eating some pagecache
508623+0 records in
508623+0 records out
2083319808 bytes (2.1 GB) copied, 11.7292 s, 178 MB/s
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 111 344 153 20 24 10 3 0 0 0 0
* /proc/buddyinfo after page cache got eaten
Trying to allocate 129
129
* 129 hugepages requested and all of them granted.
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 127 97 30 99 11 6 2 1 4 0 0
* /proc/buddyinfo after hugetlb allocation.
10 runs will behave as follows:
Trying to allocate 130
130
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 132
132
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
So basically 100% success for all 10 attempts.
Without the patch numbers looked much worse:
Trying to allocate 128
12
--
Trying to allocate 129
14
--
Trying to allocate 129
7
--
Trying to allocate 129
16
--
Trying to allocate 129
30
--
Trying to allocate 129
38
--
Trying to allocate 129
19
--
Trying to allocate 129
37
--
Trying to allocate 129
28
--
Trying to allocate 129
37
Just for completness the base kernel without oom detection rework looks
as follows:
Trying to allocate 127
30
--
Trying to allocate 129
12
--
Trying to allocate 129
52
--
Trying to allocate 128
32
--
Trying to allocate 129
12
--
Trying to allocate 129
10
--
Trying to allocate 129
32
--
Trying to allocate 128
14
--
Trying to allocate 128
16
--
Trying to allocate 129
8
As we can see the success rate is much more volatile and smaller without
this patch. So the patch not only makes the retry logic for costly
requests more sensible the success rate is even higher.
Signed-off-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 17:57:09 -06:00
|
|
|
int max_retries = MAX_COMPACT_RETRIES;
|
|
|
|
|
2016-05-20 17:57:06 -06:00
|
|
|
if (!order)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* compaction considers all the zone as desperately out of memory
|
|
|
|
* so it doesn't really make much sense to retry except when the
|
2016-07-28 16:49:28 -06:00
|
|
|
* failure could be caused by insufficient priority
|
2016-05-20 17:57:06 -06:00
|
|
|
*/
|
|
|
|
if (compaction_failed(compact_result)) {
|
2016-07-28 16:49:28 -06:00
|
|
|
if (*compact_priority > MIN_COMPACT_PRIORITY) {
|
|
|
|
(*compact_priority)--;
|
2016-05-20 17:57:06 -06:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
mm: consider compaction feedback also for costly allocation
PAGE_ALLOC_COSTLY_ORDER retry logic is mostly handled inside
should_reclaim_retry currently where we decide to not retry after at
least order worth of pages were reclaimed or the watermark check for at
least one zone would succeed after reclaiming all pages if the reclaim
hasn't made any progress. Compaction feedback is mostly ignored and we
just try to make sure that the compaction did at least something before
giving up.
The first condition was added by a41f24ea9fd6 ("page allocator: smarter
retry of costly-order allocations) and it assumed that lumpy reclaim
could have created a page of the sufficient order. Lumpy reclaim, has
been removed quite some time ago so the assumption doesn't hold anymore.
Remove the check for the number of reclaimed pages and rely on the
compaction feedback solely. should_reclaim_retry now only makes sure
that we keep retrying reclaim for high order pages only if they are
hidden by watermaks so order-0 reclaim makes really sense.
should_compact_retry now keeps retrying even for the costly allocations.
The number of retries is reduced wrt. !costly requests because they are
less important and harder to grant and so their pressure shouldn't cause
contention for other requests or cause an over reclaim. We also do not
reset no_progress_loops for costly request to make sure we do not keep
reclaiming too agressively.
This has been tested by running a process which fragments memory:
- compact memory
- mmap large portion of the memory (1920M on 2GRAM machine with 2G
of swapspace)
- MADV_DONTNEED single page in PAGE_SIZE*((1UL<<MAX_ORDER)-1)
steps until certain amount of memory is freed (250M in my test)
and reduce the step to (step / 2) + 1 after reaching the end of
the mapping
- then run a script which populates the page cache 2G (MemTotal)
from /dev/zero to a new file
And then tries to allocate
nr_hugepages=$(awk '/MemAvailable/{printf "%d\n", $2/(2*1024)}' /proc/meminfo)
huge pages.
root@test1:~# echo 1 > /proc/sys/vm/overcommit_memory;echo 1 > /proc/sys/vm/compact_memory; ./fragment-mem-and-run /root/alloc_hugepages.sh 1920M 250M
Node 0, zone DMA 31 28 31 10 2 0 2 1 2 3 1
Node 0, zone DMA32 437 319 171 50 28 25 20 16 16 14 437
* This is the /proc/buddyinfo after the compaction
Done fragmenting. size=2013265920 freed=262144000
Node 0, zone DMA 165 48 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 35109 14575 185 51 41 12 6 0 0 0 0
* /proc/buddyinfo after memory got fragmented
Executing "/root/alloc_hugepages.sh"
Eating some pagecache
508623+0 records in
508623+0 records out
2083319808 bytes (2.1 GB) copied, 11.7292 s, 178 MB/s
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 111 344 153 20 24 10 3 0 0 0 0
* /proc/buddyinfo after page cache got eaten
Trying to allocate 129
129
* 129 hugepages requested and all of them granted.
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 127 97 30 99 11 6 2 1 4 0 0
* /proc/buddyinfo after hugetlb allocation.
10 runs will behave as follows:
Trying to allocate 130
130
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 132
132
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
So basically 100% success for all 10 attempts.
Without the patch numbers looked much worse:
Trying to allocate 128
12
--
Trying to allocate 129
14
--
Trying to allocate 129
7
--
Trying to allocate 129
16
--
Trying to allocate 129
30
--
Trying to allocate 129
38
--
Trying to allocate 129
19
--
Trying to allocate 129
37
--
Trying to allocate 129
28
--
Trying to allocate 129
37
Just for completness the base kernel without oom detection rework looks
as follows:
Trying to allocate 127
30
--
Trying to allocate 129
12
--
Trying to allocate 129
52
--
Trying to allocate 128
32
--
Trying to allocate 129
12
--
Trying to allocate 129
10
--
Trying to allocate 129
32
--
Trying to allocate 128
14
--
Trying to allocate 128
16
--
Trying to allocate 129
8
As we can see the success rate is much more volatile and smaller without
this patch. So the patch not only makes the retry logic for costly
requests more sensible the success rate is even higher.
Signed-off-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 17:57:09 -06:00
|
|
|
* make sure the compaction wasn't deferred or didn't bail out early
|
|
|
|
* due to locks contention before we declare that we should give up.
|
2016-05-20 17:57:12 -06:00
|
|
|
* But do not retry if the given zonelist is not suitable for
|
|
|
|
* compaction.
|
2016-05-20 17:57:06 -06:00
|
|
|
*/
|
mm: consider compaction feedback also for costly allocation
PAGE_ALLOC_COSTLY_ORDER retry logic is mostly handled inside
should_reclaim_retry currently where we decide to not retry after at
least order worth of pages were reclaimed or the watermark check for at
least one zone would succeed after reclaiming all pages if the reclaim
hasn't made any progress. Compaction feedback is mostly ignored and we
just try to make sure that the compaction did at least something before
giving up.
The first condition was added by a41f24ea9fd6 ("page allocator: smarter
retry of costly-order allocations) and it assumed that lumpy reclaim
could have created a page of the sufficient order. Lumpy reclaim, has
been removed quite some time ago so the assumption doesn't hold anymore.
Remove the check for the number of reclaimed pages and rely on the
compaction feedback solely. should_reclaim_retry now only makes sure
that we keep retrying reclaim for high order pages only if they are
hidden by watermaks so order-0 reclaim makes really sense.
should_compact_retry now keeps retrying even for the costly allocations.
The number of retries is reduced wrt. !costly requests because they are
less important and harder to grant and so their pressure shouldn't cause
contention for other requests or cause an over reclaim. We also do not
reset no_progress_loops for costly request to make sure we do not keep
reclaiming too agressively.
This has been tested by running a process which fragments memory:
- compact memory
- mmap large portion of the memory (1920M on 2GRAM machine with 2G
of swapspace)
- MADV_DONTNEED single page in PAGE_SIZE*((1UL<<MAX_ORDER)-1)
steps until certain amount of memory is freed (250M in my test)
and reduce the step to (step / 2) + 1 after reaching the end of
the mapping
- then run a script which populates the page cache 2G (MemTotal)
from /dev/zero to a new file
And then tries to allocate
nr_hugepages=$(awk '/MemAvailable/{printf "%d\n", $2/(2*1024)}' /proc/meminfo)
huge pages.
root@test1:~# echo 1 > /proc/sys/vm/overcommit_memory;echo 1 > /proc/sys/vm/compact_memory; ./fragment-mem-and-run /root/alloc_hugepages.sh 1920M 250M
Node 0, zone DMA 31 28 31 10 2 0 2 1 2 3 1
Node 0, zone DMA32 437 319 171 50 28 25 20 16 16 14 437
* This is the /proc/buddyinfo after the compaction
Done fragmenting. size=2013265920 freed=262144000
Node 0, zone DMA 165 48 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 35109 14575 185 51 41 12 6 0 0 0 0
* /proc/buddyinfo after memory got fragmented
Executing "/root/alloc_hugepages.sh"
Eating some pagecache
508623+0 records in
508623+0 records out
2083319808 bytes (2.1 GB) copied, 11.7292 s, 178 MB/s
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 111 344 153 20 24 10 3 0 0 0 0
* /proc/buddyinfo after page cache got eaten
Trying to allocate 129
129
* 129 hugepages requested and all of them granted.
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 127 97 30 99 11 6 2 1 4 0 0
* /proc/buddyinfo after hugetlb allocation.
10 runs will behave as follows:
Trying to allocate 130
130
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 132
132
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
So basically 100% success for all 10 attempts.
Without the patch numbers looked much worse:
Trying to allocate 128
12
--
Trying to allocate 129
14
--
Trying to allocate 129
7
--
Trying to allocate 129
16
--
Trying to allocate 129
30
--
Trying to allocate 129
38
--
Trying to allocate 129
19
--
Trying to allocate 129
37
--
Trying to allocate 129
28
--
Trying to allocate 129
37
Just for completness the base kernel without oom detection rework looks
as follows:
Trying to allocate 127
30
--
Trying to allocate 129
12
--
Trying to allocate 129
52
--
Trying to allocate 128
32
--
Trying to allocate 129
12
--
Trying to allocate 129
10
--
Trying to allocate 129
32
--
Trying to allocate 128
14
--
Trying to allocate 128
16
--
Trying to allocate 129
8
As we can see the success rate is much more volatile and smaller without
this patch. So the patch not only makes the retry logic for costly
requests more sensible the success rate is even higher.
Signed-off-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 17:57:09 -06:00
|
|
|
if (compaction_withdrawn(compact_result))
|
2016-05-20 17:57:12 -06:00
|
|
|
return compaction_zonelist_suitable(ac, order, alloc_flags);
|
mm: consider compaction feedback also for costly allocation
PAGE_ALLOC_COSTLY_ORDER retry logic is mostly handled inside
should_reclaim_retry currently where we decide to not retry after at
least order worth of pages were reclaimed or the watermark check for at
least one zone would succeed after reclaiming all pages if the reclaim
hasn't made any progress. Compaction feedback is mostly ignored and we
just try to make sure that the compaction did at least something before
giving up.
The first condition was added by a41f24ea9fd6 ("page allocator: smarter
retry of costly-order allocations) and it assumed that lumpy reclaim
could have created a page of the sufficient order. Lumpy reclaim, has
been removed quite some time ago so the assumption doesn't hold anymore.
Remove the check for the number of reclaimed pages and rely on the
compaction feedback solely. should_reclaim_retry now only makes sure
that we keep retrying reclaim for high order pages only if they are
hidden by watermaks so order-0 reclaim makes really sense.
should_compact_retry now keeps retrying even for the costly allocations.
The number of retries is reduced wrt. !costly requests because they are
less important and harder to grant and so their pressure shouldn't cause
contention for other requests or cause an over reclaim. We also do not
reset no_progress_loops for costly request to make sure we do not keep
reclaiming too agressively.
This has been tested by running a process which fragments memory:
- compact memory
- mmap large portion of the memory (1920M on 2GRAM machine with 2G
of swapspace)
- MADV_DONTNEED single page in PAGE_SIZE*((1UL<<MAX_ORDER)-1)
steps until certain amount of memory is freed (250M in my test)
and reduce the step to (step / 2) + 1 after reaching the end of
the mapping
- then run a script which populates the page cache 2G (MemTotal)
from /dev/zero to a new file
And then tries to allocate
nr_hugepages=$(awk '/MemAvailable/{printf "%d\n", $2/(2*1024)}' /proc/meminfo)
huge pages.
root@test1:~# echo 1 > /proc/sys/vm/overcommit_memory;echo 1 > /proc/sys/vm/compact_memory; ./fragment-mem-and-run /root/alloc_hugepages.sh 1920M 250M
Node 0, zone DMA 31 28 31 10 2 0 2 1 2 3 1
Node 0, zone DMA32 437 319 171 50 28 25 20 16 16 14 437
* This is the /proc/buddyinfo after the compaction
Done fragmenting. size=2013265920 freed=262144000
Node 0, zone DMA 165 48 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 35109 14575 185 51 41 12 6 0 0 0 0
* /proc/buddyinfo after memory got fragmented
Executing "/root/alloc_hugepages.sh"
Eating some pagecache
508623+0 records in
508623+0 records out
2083319808 bytes (2.1 GB) copied, 11.7292 s, 178 MB/s
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 111 344 153 20 24 10 3 0 0 0 0
* /proc/buddyinfo after page cache got eaten
Trying to allocate 129
129
* 129 hugepages requested and all of them granted.
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 127 97 30 99 11 6 2 1 4 0 0
* /proc/buddyinfo after hugetlb allocation.
10 runs will behave as follows:
Trying to allocate 130
130
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 132
132
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
So basically 100% success for all 10 attempts.
Without the patch numbers looked much worse:
Trying to allocate 128
12
--
Trying to allocate 129
14
--
Trying to allocate 129
7
--
Trying to allocate 129
16
--
Trying to allocate 129
30
--
Trying to allocate 129
38
--
Trying to allocate 129
19
--
Trying to allocate 129
37
--
Trying to allocate 129
28
--
Trying to allocate 129
37
Just for completness the base kernel without oom detection rework looks
as follows:
Trying to allocate 127
30
--
Trying to allocate 129
12
--
Trying to allocate 129
52
--
Trying to allocate 128
32
--
Trying to allocate 129
12
--
Trying to allocate 129
10
--
Trying to allocate 129
32
--
Trying to allocate 128
14
--
Trying to allocate 128
16
--
Trying to allocate 129
8
As we can see the success rate is much more volatile and smaller without
this patch. So the patch not only makes the retry logic for costly
requests more sensible the success rate is even higher.
Signed-off-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 17:57:09 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* !costly requests are much more important than __GFP_REPEAT
|
|
|
|
* costly ones because they are de facto nofail and invoke OOM
|
|
|
|
* killer to move on while costly can fail and users are ready
|
|
|
|
* to cope with that. 1/4 retries is rather arbitrary but we
|
|
|
|
* would need much more detailed feedback from compaction to
|
|
|
|
* make a better decision.
|
|
|
|
*/
|
|
|
|
if (order > PAGE_ALLOC_COSTLY_ORDER)
|
|
|
|
max_retries /= 4;
|
|
|
|
if (compaction_retries <= max_retries)
|
|
|
|
return true;
|
2016-05-20 17:57:06 -06:00
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
2010-05-24 15:32:30 -06:00
|
|
|
#else
|
|
|
|
static inline struct page *
|
|
|
|
__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
|
2016-05-19 18:13:38 -06:00
|
|
|
unsigned int alloc_flags, const struct alloc_context *ac,
|
2016-07-28 16:49:28 -06:00
|
|
|
enum compact_priority prio, enum compact_result *compact_result)
|
2010-05-24 15:32:30 -06:00
|
|
|
{
|
2016-05-20 17:57:06 -06:00
|
|
|
*compact_result = COMPACT_SKIPPED;
|
2010-05-24 15:32:30 -06:00
|
|
|
return NULL;
|
|
|
|
}
|
2016-05-20 17:57:06 -06:00
|
|
|
|
|
|
|
static inline bool
|
2016-05-20 17:57:12 -06:00
|
|
|
should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
|
|
|
|
enum compact_result compact_result,
|
2016-07-28 16:49:28 -06:00
|
|
|
enum compact_priority *compact_priority,
|
2016-05-20 17:57:06 -06:00
|
|
|
int compaction_retries)
|
|
|
|
{
|
2016-05-20 17:57:15 -06:00
|
|
|
struct zone *zone;
|
|
|
|
struct zoneref *z;
|
|
|
|
|
|
|
|
if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* There are setups with compaction disabled which would prefer to loop
|
|
|
|
* inside the allocator rather than hit the oom killer prematurely.
|
|
|
|
* Let's give them a good hope and keep retrying while the order-0
|
|
|
|
* watermarks are OK.
|
|
|
|
*/
|
|
|
|
for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
|
|
|
|
ac->nodemask) {
|
|
|
|
if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
|
|
|
|
ac_classzone_idx(ac), alloc_flags))
|
|
|
|
return true;
|
|
|
|
}
|
2016-05-20 17:57:06 -06:00
|
|
|
return false;
|
|
|
|
}
|
2010-05-24 15:32:30 -06:00
|
|
|
#endif /* CONFIG_COMPACTION */
|
|
|
|
|
2012-01-25 04:09:52 -07:00
|
|
|
/* Perform direct synchronous page reclaim */
|
|
|
|
static int
|
2015-02-11 16:25:41 -07:00
|
|
|
__perform_reclaim(gfp_t gfp_mask, unsigned int order,
|
|
|
|
const struct alloc_context *ac)
|
2009-06-16 16:31:57 -06:00
|
|
|
{
|
|
|
|
struct reclaim_state reclaim_state;
|
2012-01-25 04:09:52 -07:00
|
|
|
int progress;
|
2009-06-16 16:31:57 -06:00
|
|
|
|
|
|
|
cond_resched();
|
|
|
|
|
|
|
|
/* We now go into synchronous reclaim */
|
|
|
|
cpuset_memory_pressure_bump();
|
2011-01-13 16:47:32 -07:00
|
|
|
current->flags |= PF_MEMALLOC;
|
2009-06-16 16:31:57 -06:00
|
|
|
lockdep_set_current_reclaim_state(gfp_mask);
|
|
|
|
reclaim_state.reclaimed_slab = 0;
|
2011-01-13 16:47:32 -07:00
|
|
|
current->reclaim_state = &reclaim_state;
|
2009-06-16 16:31:57 -06:00
|
|
|
|
2015-02-11 16:25:41 -07:00
|
|
|
progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
|
|
|
|
ac->nodemask);
|
2009-06-16 16:31:57 -06:00
|
|
|
|
2011-01-13 16:47:32 -07:00
|
|
|
current->reclaim_state = NULL;
|
2009-06-16 16:31:57 -06:00
|
|
|
lockdep_clear_current_reclaim_state();
|
2011-01-13 16:47:32 -07:00
|
|
|
current->flags &= ~PF_MEMALLOC;
|
2009-06-16 16:31:57 -06:00
|
|
|
|
|
|
|
cond_resched();
|
|
|
|
|
2012-01-25 04:09:52 -07:00
|
|
|
return progress;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* The really slow allocator path where we enter direct reclaim */
|
|
|
|
static inline struct page *
|
|
|
|
__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
|
2016-05-19 18:13:38 -06:00
|
|
|
unsigned int alloc_flags, const struct alloc_context *ac,
|
2015-02-11 16:25:41 -07:00
|
|
|
unsigned long *did_some_progress)
|
2012-01-25 04:09:52 -07:00
|
|
|
{
|
|
|
|
struct page *page = NULL;
|
|
|
|
bool drained = false;
|
|
|
|
|
2015-02-11 16:25:41 -07:00
|
|
|
*did_some_progress = __perform_reclaim(gfp_mask, order, ac);
|
2010-09-09 17:38:18 -06:00
|
|
|
if (unlikely(!(*did_some_progress)))
|
|
|
|
return NULL;
|
2009-06-16 16:31:57 -06:00
|
|
|
|
2010-09-09 17:38:18 -06:00
|
|
|
retry:
|
2016-07-28 16:49:13 -06:00
|
|
|
page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
|
2010-09-09 17:38:18 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If an allocation failed after direct reclaim, it could be because
|
2015-11-06 17:28:37 -07:00
|
|
|
* pages are pinned on the per-cpu lists or in high alloc reserves.
|
|
|
|
* Shrink them them and try again
|
2010-09-09 17:38:18 -06:00
|
|
|
*/
|
|
|
|
if (!page && !drained) {
|
2015-11-06 17:28:37 -07:00
|
|
|
unreserve_highatomic_pageblock(ac);
|
2014-12-10 16:43:01 -07:00
|
|
|
drain_all_pages(NULL);
|
2010-09-09 17:38:18 -06:00
|
|
|
drained = true;
|
|
|
|
goto retry;
|
|
|
|
}
|
|
|
|
|
2009-06-16 16:31:57 -06:00
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
2015-02-11 16:25:41 -07:00
|
|
|
static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac)
|
mm: page_alloc: spill to remote nodes before waking kswapd
On NUMA systems, a node may start thrashing cache or even swap anonymous
pages while there are still free pages on remote nodes.
This is a result of commits 81c0a2bb515f ("mm: page_alloc: fair zone
allocator policy") and fff4068cba48 ("mm: page_alloc: revert NUMA aspect
of fair allocation policy").
Before those changes, the allocator would first try all allowed zones,
including those on remote nodes, before waking any kswapds. But now,
the allocator fastpath doubles as the fairness pass, which in turn can
only consider the local node to prevent remote spilling based on
exhausted fairness batches alone. Remote nodes are only considered in
the slowpath, after the kswapds are woken up. But if remote nodes still
have free memory, kswapd should not be woken to rebalance the local node
or it may thrash cash or swap prematurely.
Fix this by adding one more unfair pass over the zonelist that is
allowed to spill to remote nodes after the local fairness pass fails but
before entering the slowpath and waking the kswapds.
This also gets rid of the GFP_THISNODE exemption from the fairness
protocol because the unfair pass is no longer tied to kswapd, which
GFP_THISNODE is not allowed to wake up.
However, because remote spills can be more frequent now - we prefer them
over local kswapd reclaim - the allocation batches on remote nodes could
underflow more heavily. When resetting the batches, use
atomic_long_read() directly instead of zone_page_state() to calculate the
delta as the latter filters negative counter values.
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: <stable@kernel.org> [3.12+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 16:37:48 -06:00
|
|
|
{
|
|
|
|
struct zoneref *z;
|
|
|
|
struct zone *zone;
|
2016-07-28 16:46:26 -06:00
|
|
|
pg_data_t *last_pgdat = NULL;
|
mm: page_alloc: spill to remote nodes before waking kswapd
On NUMA systems, a node may start thrashing cache or even swap anonymous
pages while there are still free pages on remote nodes.
This is a result of commits 81c0a2bb515f ("mm: page_alloc: fair zone
allocator policy") and fff4068cba48 ("mm: page_alloc: revert NUMA aspect
of fair allocation policy").
Before those changes, the allocator would first try all allowed zones,
including those on remote nodes, before waking any kswapds. But now,
the allocator fastpath doubles as the fairness pass, which in turn can
only consider the local node to prevent remote spilling based on
exhausted fairness batches alone. Remote nodes are only considered in
the slowpath, after the kswapds are woken up. But if remote nodes still
have free memory, kswapd should not be woken to rebalance the local node
or it may thrash cash or swap prematurely.
Fix this by adding one more unfair pass over the zonelist that is
allowed to spill to remote nodes after the local fairness pass fails but
before entering the slowpath and waking the kswapds.
This also gets rid of the GFP_THISNODE exemption from the fairness
protocol because the unfair pass is no longer tied to kswapd, which
GFP_THISNODE is not allowed to wake up.
However, because remote spills can be more frequent now - we prefer them
over local kswapd reclaim - the allocation batches on remote nodes could
underflow more heavily. When resetting the batches, use
atomic_long_read() directly instead of zone_page_state() to calculate the
delta as the latter filters negative counter values.
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: <stable@kernel.org> [3.12+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 16:37:48 -06:00
|
|
|
|
2015-02-11 16:25:41 -07:00
|
|
|
for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
|
2016-07-28 16:46:26 -06:00
|
|
|
ac->high_zoneidx, ac->nodemask) {
|
|
|
|
if (last_pgdat != zone->zone_pgdat)
|
2016-07-28 16:46:29 -06:00
|
|
|
wakeup_kswapd(zone, order, ac->high_zoneidx);
|
2016-07-28 16:46:26 -06:00
|
|
|
last_pgdat = zone->zone_pgdat;
|
|
|
|
}
|
mm: page_alloc: spill to remote nodes before waking kswapd
On NUMA systems, a node may start thrashing cache or even swap anonymous
pages while there are still free pages on remote nodes.
This is a result of commits 81c0a2bb515f ("mm: page_alloc: fair zone
allocator policy") and fff4068cba48 ("mm: page_alloc: revert NUMA aspect
of fair allocation policy").
Before those changes, the allocator would first try all allowed zones,
including those on remote nodes, before waking any kswapds. But now,
the allocator fastpath doubles as the fairness pass, which in turn can
only consider the local node to prevent remote spilling based on
exhausted fairness batches alone. Remote nodes are only considered in
the slowpath, after the kswapds are woken up. But if remote nodes still
have free memory, kswapd should not be woken to rebalance the local node
or it may thrash cash or swap prematurely.
Fix this by adding one more unfair pass over the zonelist that is
allowed to spill to remote nodes after the local fairness pass fails but
before entering the slowpath and waking the kswapds.
This also gets rid of the GFP_THISNODE exemption from the fairness
protocol because the unfair pass is no longer tied to kswapd, which
GFP_THISNODE is not allowed to wake up.
However, because remote spills can be more frequent now - we prefer them
over local kswapd reclaim - the allocation batches on remote nodes could
underflow more heavily. When resetting the batches, use
atomic_long_read() directly instead of zone_page_state() to calculate the
delta as the latter filters negative counter values.
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: <stable@kernel.org> [3.12+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 16:37:48 -06:00
|
|
|
}
|
|
|
|
|
2016-05-19 18:13:38 -06:00
|
|
|
static inline unsigned int
|
2009-06-16 16:32:02 -06:00
|
|
|
gfp_to_alloc_flags(gfp_t gfp_mask)
|
|
|
|
{
|
2016-05-19 18:13:38 -06:00
|
|
|
unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-06-16 16:32:02 -06:00
|
|
|
/* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
|
2010-10-26 15:21:59 -06:00
|
|
|
BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
|
2006-12-08 03:39:45 -07:00
|
|
|
|
2009-06-16 16:32:02 -06:00
|
|
|
/*
|
|
|
|
* The caller may dip into page reserves a bit more if the caller
|
|
|
|
* cannot run direct reclaim, or if the caller has realtime scheduling
|
|
|
|
* policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
|
2015-11-06 17:28:21 -07:00
|
|
|
* set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
|
2009-06-16 16:32:02 -06:00
|
|
|
*/
|
2010-10-26 15:21:59 -06:00
|
|
|
alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2015-11-06 17:28:21 -07:00
|
|
|
if (gfp_mask & __GFP_ATOMIC) {
|
2011-01-13 16:46:49 -07:00
|
|
|
/*
|
2014-07-30 17:08:24 -06:00
|
|
|
* Not worth trying to allocate harder for __GFP_NOMEMALLOC even
|
|
|
|
* if it can't schedule.
|
2011-01-13 16:46:49 -07:00
|
|
|
*/
|
2014-07-30 17:08:24 -06:00
|
|
|
if (!(gfp_mask & __GFP_NOMEMALLOC))
|
2011-01-13 16:46:49 -07:00
|
|
|
alloc_flags |= ALLOC_HARDER;
|
2007-10-16 02:25:37 -06:00
|
|
|
/*
|
2014-07-30 17:08:24 -06:00
|
|
|
* Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
|
2014-10-20 05:50:30 -06:00
|
|
|
* comment for __cpuset_node_allowed().
|
2007-10-16 02:25:37 -06:00
|
|
|
*/
|
2009-06-16 16:32:02 -06:00
|
|
|
alloc_flags &= ~ALLOC_CPUSET;
|
2011-01-13 16:47:32 -07:00
|
|
|
} else if (unlikely(rt_task(current)) && !in_interrupt())
|
2009-06-16 16:32:02 -06:00
|
|
|
alloc_flags |= ALLOC_HARDER;
|
|
|
|
|
2012-10-08 17:32:05 -06:00
|
|
|
#ifdef CONFIG_CMA
|
2014-10-09 16:27:25 -06:00
|
|
|
if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
|
2012-10-08 17:32:05 -06:00
|
|
|
alloc_flags |= ALLOC_CMA;
|
|
|
|
#endif
|
2009-06-16 16:32:02 -06:00
|
|
|
return alloc_flags;
|
|
|
|
}
|
|
|
|
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
|
|
|
|
{
|
2016-07-28 16:49:13 -06:00
|
|
|
if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (gfp_mask & __GFP_MEMALLOC)
|
|
|
|
return true;
|
|
|
|
if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
|
|
|
|
return true;
|
|
|
|
if (!in_interrupt() &&
|
|
|
|
((current->flags & PF_MEMALLOC) ||
|
|
|
|
unlikely(test_thread_flag(TIF_MEMDIE))))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
}
|
|
|
|
|
2016-05-20 17:57:00 -06:00
|
|
|
/*
|
|
|
|
* Maximum number of reclaim retries without any progress before OOM killer
|
|
|
|
* is consider as the only way to move forward.
|
|
|
|
*/
|
|
|
|
#define MAX_RECLAIM_RETRIES 16
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Checks whether it makes sense to retry the reclaim to make a forward progress
|
|
|
|
* for the given allocation request.
|
|
|
|
* The reclaim feedback represented by did_some_progress (any progress during
|
mm: consider compaction feedback also for costly allocation
PAGE_ALLOC_COSTLY_ORDER retry logic is mostly handled inside
should_reclaim_retry currently where we decide to not retry after at
least order worth of pages were reclaimed or the watermark check for at
least one zone would succeed after reclaiming all pages if the reclaim
hasn't made any progress. Compaction feedback is mostly ignored and we
just try to make sure that the compaction did at least something before
giving up.
The first condition was added by a41f24ea9fd6 ("page allocator: smarter
retry of costly-order allocations) and it assumed that lumpy reclaim
could have created a page of the sufficient order. Lumpy reclaim, has
been removed quite some time ago so the assumption doesn't hold anymore.
Remove the check for the number of reclaimed pages and rely on the
compaction feedback solely. should_reclaim_retry now only makes sure
that we keep retrying reclaim for high order pages only if they are
hidden by watermaks so order-0 reclaim makes really sense.
should_compact_retry now keeps retrying even for the costly allocations.
The number of retries is reduced wrt. !costly requests because they are
less important and harder to grant and so their pressure shouldn't cause
contention for other requests or cause an over reclaim. We also do not
reset no_progress_loops for costly request to make sure we do not keep
reclaiming too agressively.
This has been tested by running a process which fragments memory:
- compact memory
- mmap large portion of the memory (1920M on 2GRAM machine with 2G
of swapspace)
- MADV_DONTNEED single page in PAGE_SIZE*((1UL<<MAX_ORDER)-1)
steps until certain amount of memory is freed (250M in my test)
and reduce the step to (step / 2) + 1 after reaching the end of
the mapping
- then run a script which populates the page cache 2G (MemTotal)
from /dev/zero to a new file
And then tries to allocate
nr_hugepages=$(awk '/MemAvailable/{printf "%d\n", $2/(2*1024)}' /proc/meminfo)
huge pages.
root@test1:~# echo 1 > /proc/sys/vm/overcommit_memory;echo 1 > /proc/sys/vm/compact_memory; ./fragment-mem-and-run /root/alloc_hugepages.sh 1920M 250M
Node 0, zone DMA 31 28 31 10 2 0 2 1 2 3 1
Node 0, zone DMA32 437 319 171 50 28 25 20 16 16 14 437
* This is the /proc/buddyinfo after the compaction
Done fragmenting. size=2013265920 freed=262144000
Node 0, zone DMA 165 48 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 35109 14575 185 51 41 12 6 0 0 0 0
* /proc/buddyinfo after memory got fragmented
Executing "/root/alloc_hugepages.sh"
Eating some pagecache
508623+0 records in
508623+0 records out
2083319808 bytes (2.1 GB) copied, 11.7292 s, 178 MB/s
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 111 344 153 20 24 10 3 0 0 0 0
* /proc/buddyinfo after page cache got eaten
Trying to allocate 129
129
* 129 hugepages requested and all of them granted.
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 127 97 30 99 11 6 2 1 4 0 0
* /proc/buddyinfo after hugetlb allocation.
10 runs will behave as follows:
Trying to allocate 130
130
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 132
132
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
So basically 100% success for all 10 attempts.
Without the patch numbers looked much worse:
Trying to allocate 128
12
--
Trying to allocate 129
14
--
Trying to allocate 129
7
--
Trying to allocate 129
16
--
Trying to allocate 129
30
--
Trying to allocate 129
38
--
Trying to allocate 129
19
--
Trying to allocate 129
37
--
Trying to allocate 129
28
--
Trying to allocate 129
37
Just for completness the base kernel without oom detection rework looks
as follows:
Trying to allocate 127
30
--
Trying to allocate 129
12
--
Trying to allocate 129
52
--
Trying to allocate 128
32
--
Trying to allocate 129
12
--
Trying to allocate 129
10
--
Trying to allocate 129
32
--
Trying to allocate 128
14
--
Trying to allocate 128
16
--
Trying to allocate 129
8
As we can see the success rate is much more volatile and smaller without
this patch. So the patch not only makes the retry logic for costly
requests more sensible the success rate is even higher.
Signed-off-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 17:57:09 -06:00
|
|
|
* the last reclaim round) and no_progress_loops (number of reclaim rounds without
|
|
|
|
* any progress in a row) is considered as well as the reclaimable pages on the
|
|
|
|
* applicable zone list (with a backoff mechanism which is a function of
|
|
|
|
* no_progress_loops).
|
2016-05-20 17:57:00 -06:00
|
|
|
*
|
|
|
|
* Returns true if a retry is viable or false to enter the oom path.
|
|
|
|
*/
|
|
|
|
static inline bool
|
|
|
|
should_reclaim_retry(gfp_t gfp_mask, unsigned order,
|
|
|
|
struct alloc_context *ac, int alloc_flags,
|
mm: consider compaction feedback also for costly allocation
PAGE_ALLOC_COSTLY_ORDER retry logic is mostly handled inside
should_reclaim_retry currently where we decide to not retry after at
least order worth of pages were reclaimed or the watermark check for at
least one zone would succeed after reclaiming all pages if the reclaim
hasn't made any progress. Compaction feedback is mostly ignored and we
just try to make sure that the compaction did at least something before
giving up.
The first condition was added by a41f24ea9fd6 ("page allocator: smarter
retry of costly-order allocations) and it assumed that lumpy reclaim
could have created a page of the sufficient order. Lumpy reclaim, has
been removed quite some time ago so the assumption doesn't hold anymore.
Remove the check for the number of reclaimed pages and rely on the
compaction feedback solely. should_reclaim_retry now only makes sure
that we keep retrying reclaim for high order pages only if they are
hidden by watermaks so order-0 reclaim makes really sense.
should_compact_retry now keeps retrying even for the costly allocations.
The number of retries is reduced wrt. !costly requests because they are
less important and harder to grant and so their pressure shouldn't cause
contention for other requests or cause an over reclaim. We also do not
reset no_progress_loops for costly request to make sure we do not keep
reclaiming too agressively.
This has been tested by running a process which fragments memory:
- compact memory
- mmap large portion of the memory (1920M on 2GRAM machine with 2G
of swapspace)
- MADV_DONTNEED single page in PAGE_SIZE*((1UL<<MAX_ORDER)-1)
steps until certain amount of memory is freed (250M in my test)
and reduce the step to (step / 2) + 1 after reaching the end of
the mapping
- then run a script which populates the page cache 2G (MemTotal)
from /dev/zero to a new file
And then tries to allocate
nr_hugepages=$(awk '/MemAvailable/{printf "%d\n", $2/(2*1024)}' /proc/meminfo)
huge pages.
root@test1:~# echo 1 > /proc/sys/vm/overcommit_memory;echo 1 > /proc/sys/vm/compact_memory; ./fragment-mem-and-run /root/alloc_hugepages.sh 1920M 250M
Node 0, zone DMA 31 28 31 10 2 0 2 1 2 3 1
Node 0, zone DMA32 437 319 171 50 28 25 20 16 16 14 437
* This is the /proc/buddyinfo after the compaction
Done fragmenting. size=2013265920 freed=262144000
Node 0, zone DMA 165 48 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 35109 14575 185 51 41 12 6 0 0 0 0
* /proc/buddyinfo after memory got fragmented
Executing "/root/alloc_hugepages.sh"
Eating some pagecache
508623+0 records in
508623+0 records out
2083319808 bytes (2.1 GB) copied, 11.7292 s, 178 MB/s
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 111 344 153 20 24 10 3 0 0 0 0
* /proc/buddyinfo after page cache got eaten
Trying to allocate 129
129
* 129 hugepages requested and all of them granted.
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 127 97 30 99 11 6 2 1 4 0 0
* /proc/buddyinfo after hugetlb allocation.
10 runs will behave as follows:
Trying to allocate 130
130
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 132
132
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
So basically 100% success for all 10 attempts.
Without the patch numbers looked much worse:
Trying to allocate 128
12
--
Trying to allocate 129
14
--
Trying to allocate 129
7
--
Trying to allocate 129
16
--
Trying to allocate 129
30
--
Trying to allocate 129
38
--
Trying to allocate 129
19
--
Trying to allocate 129
37
--
Trying to allocate 129
28
--
Trying to allocate 129
37
Just for completness the base kernel without oom detection rework looks
as follows:
Trying to allocate 127
30
--
Trying to allocate 129
12
--
Trying to allocate 129
52
--
Trying to allocate 128
32
--
Trying to allocate 129
12
--
Trying to allocate 129
10
--
Trying to allocate 129
32
--
Trying to allocate 128
14
--
Trying to allocate 128
16
--
Trying to allocate 129
8
As we can see the success rate is much more volatile and smaller without
this patch. So the patch not only makes the retry logic for costly
requests more sensible the success rate is even higher.
Signed-off-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 17:57:09 -06:00
|
|
|
bool did_some_progress, int no_progress_loops)
|
2016-05-20 17:57:00 -06:00
|
|
|
{
|
|
|
|
struct zone *zone;
|
|
|
|
struct zoneref *z;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Make sure we converge to OOM if we cannot make any progress
|
|
|
|
* several times in the row.
|
|
|
|
*/
|
|
|
|
if (no_progress_loops > MAX_RECLAIM_RETRIES)
|
|
|
|
return false;
|
|
|
|
|
2016-07-28 16:47:05 -06:00
|
|
|
/*
|
|
|
|
* Keep reclaiming pages while there is a chance this will lead
|
|
|
|
* somewhere. If none of the target zones can satisfy our allocation
|
|
|
|
* request even if all reclaimable pages are considered then we are
|
|
|
|
* screwed and have to go OOM.
|
2016-05-20 17:57:00 -06:00
|
|
|
*/
|
|
|
|
for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
|
|
|
|
ac->nodemask) {
|
|
|
|
unsigned long available;
|
2016-05-20 17:57:03 -06:00
|
|
|
unsigned long reclaimable;
|
2016-05-20 17:57:00 -06:00
|
|
|
|
2016-07-28 16:47:31 -06:00
|
|
|
available = reclaimable = zone_reclaimable_pages(zone);
|
2016-05-20 17:57:00 -06:00
|
|
|
available -= DIV_ROUND_UP(no_progress_loops * available,
|
|
|
|
MAX_RECLAIM_RETRIES);
|
2016-07-28 16:47:31 -06:00
|
|
|
available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
|
2016-05-20 17:57:00 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Would the allocation succeed if we reclaimed the whole
|
2016-07-28 16:47:31 -06:00
|
|
|
* available?
|
2016-05-20 17:57:00 -06:00
|
|
|
*/
|
2016-07-28 16:47:31 -06:00
|
|
|
if (__zone_watermark_ok(zone, order, min_wmark_pages(zone),
|
|
|
|
ac_classzone_idx(ac), alloc_flags, available)) {
|
2016-05-20 17:57:03 -06:00
|
|
|
/*
|
|
|
|
* If we didn't make any progress and have a lot of
|
|
|
|
* dirty + writeback pages then we should wait for
|
|
|
|
* an IO to complete to slow down the reclaim and
|
|
|
|
* prevent from pre mature OOM
|
|
|
|
*/
|
|
|
|
if (!did_some_progress) {
|
2016-07-28 16:46:20 -06:00
|
|
|
unsigned long write_pending;
|
2016-05-20 17:57:03 -06:00
|
|
|
|
2016-07-28 16:47:31 -06:00
|
|
|
write_pending = zone_page_state_snapshot(zone,
|
|
|
|
NR_ZONE_WRITE_PENDING);
|
2016-05-20 17:57:03 -06:00
|
|
|
|
2016-07-28 16:46:20 -06:00
|
|
|
if (2 * write_pending > reclaimable) {
|
2016-05-20 17:57:03 -06:00
|
|
|
congestion_wait(BLK_RW_ASYNC, HZ/10);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
2016-07-28 16:47:31 -06:00
|
|
|
|
2016-05-20 17:57:03 -06:00
|
|
|
/*
|
|
|
|
* Memory allocation/reclaim might be called from a WQ
|
|
|
|
* context and the current implementation of the WQ
|
|
|
|
* concurrency control doesn't recognize that
|
|
|
|
* a particular WQ is congested if the worker thread is
|
|
|
|
* looping without ever sleeping. Therefore we have to
|
|
|
|
* do a short sleep here rather than calling
|
|
|
|
* cond_resched().
|
|
|
|
*/
|
|
|
|
if (current->flags & PF_WQ_WORKER)
|
|
|
|
schedule_timeout_uninterruptible(1);
|
|
|
|
else
|
|
|
|
cond_resched();
|
|
|
|
|
2016-05-20 17:57:00 -06:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2009-06-16 16:31:57 -06:00
|
|
|
static inline struct page *
|
|
|
|
__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
|
2015-02-11 16:25:41 -07:00
|
|
|
struct alloc_context *ac)
|
2009-06-16 16:31:57 -06:00
|
|
|
{
|
2015-11-06 17:28:21 -07:00
|
|
|
bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
|
2009-06-16 16:31:57 -06:00
|
|
|
struct page *page = NULL;
|
2016-05-19 18:13:38 -06:00
|
|
|
unsigned int alloc_flags;
|
2009-06-16 16:31:57 -06:00
|
|
|
unsigned long did_some_progress;
|
2016-07-28 16:49:28 -06:00
|
|
|
enum compact_priority compact_priority = DEF_COMPACT_PRIORITY;
|
2016-05-20 17:56:53 -06:00
|
|
|
enum compact_result compact_result;
|
2016-05-20 17:57:06 -06:00
|
|
|
int compaction_retries = 0;
|
2016-05-20 17:57:00 -06:00
|
|
|
int no_progress_loops = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-06-16 16:32:18 -06:00
|
|
|
/*
|
|
|
|
* In the slowpath, we sanity check order to avoid ever trying to
|
|
|
|
* reclaim >= MAX_ORDER areas which will never succeed. Callers may
|
|
|
|
* be using allocators in order of preference for an area that is
|
|
|
|
* too large.
|
|
|
|
*/
|
2009-07-29 16:04:08 -06:00
|
|
|
if (order >= MAX_ORDER) {
|
|
|
|
WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
|
2009-06-16 16:32:18 -06:00
|
|
|
return NULL;
|
2009-07-29 16:04:08 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2015-11-06 17:28:21 -07:00
|
|
|
/*
|
|
|
|
* We also sanity check to catch abuse of atomic reserves being used by
|
|
|
|
* callers that are not in atomic context.
|
|
|
|
*/
|
|
|
|
if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
|
|
|
|
(__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
|
|
|
|
gfp_mask &= ~__GFP_ATOMIC;
|
|
|
|
|
[PATCH] cpusets: formalize intermediate GFP_KERNEL containment
This patch makes use of the previously underutilized cpuset flag
'mem_exclusive' to provide what amounts to another layer of memory placement
resolution. With this patch, there are now the following four layers of
memory placement available:
1) The whole system (interrupt and GFP_ATOMIC allocations can use this),
2) The nearest enclosing mem_exclusive cpuset (GFP_KERNEL allocations can use),
3) The current tasks cpuset (GFP_USER allocations constrained to here), and
4) Specific node placement, using mbind and set_mempolicy.
These nest - each layer is a subset (same or within) of the previous.
Layer (2) above is new, with this patch. The call used to check whether a
zone (its node, actually) is in a cpuset (in its mems_allowed, actually) is
extended to take a gfp_mask argument, and its logic is extended, in the case
that __GFP_HARDWALL is not set in the flag bits, to look up the cpuset
hierarchy for the nearest enclosing mem_exclusive cpuset, to determine if
placement is allowed. The definition of GFP_USER, which used to be identical
to GFP_KERNEL, is changed to also set the __GFP_HARDWALL bit, in the previous
cpuset_gfp_hardwall_flag patch.
GFP_ATOMIC and GFP_KERNEL allocations will stay within the current tasks
cpuset, so long as any node therein is not too tight on memory, but will
escape to the larger layer, if need be.
The intended use is to allow something like a batch manager to handle several
jobs, each job in its own cpuset, but using common kernel memory for caches
and such. Swapper and oom_kill activity is also constrained to Layer (2). A
task in or below one mem_exclusive cpuset should not cause swapping on nodes
in another non-overlapping mem_exclusive cpuset, nor provoke oom_killing of a
task in another such cpuset. Heavy use of kernel memory for i/o caching and
such by one job should not impact the memory available to jobs in other
non-overlapping mem_exclusive cpusets.
This patch enables providing hardwall, inescapable cpusets for memory
allocations of each job, while sharing kernel memory allocations between
several jobs, in an enclosing mem_exclusive cpuset.
Like Dinakar's patch earlier to enable administering sched domains using the
cpu_exclusive flag, this patch also provides a useful meaning to a cpuset flag
that had previously done nothing much useful other than restrict what cpuset
configurations were allowed.
Signed-off-by: Paul Jackson <pj@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-06 16:18:12 -06:00
|
|
|
/*
|
2016-07-28 16:49:13 -06:00
|
|
|
* The fast path uses conservative alloc_flags to succeed only until
|
|
|
|
* kswapd needs to be woken up, and to avoid the cost of setting up
|
|
|
|
* alloc_flags precisely. So we do that now.
|
[PATCH] cpusets: formalize intermediate GFP_KERNEL containment
This patch makes use of the previously underutilized cpuset flag
'mem_exclusive' to provide what amounts to another layer of memory placement
resolution. With this patch, there are now the following four layers of
memory placement available:
1) The whole system (interrupt and GFP_ATOMIC allocations can use this),
2) The nearest enclosing mem_exclusive cpuset (GFP_KERNEL allocations can use),
3) The current tasks cpuset (GFP_USER allocations constrained to here), and
4) Specific node placement, using mbind and set_mempolicy.
These nest - each layer is a subset (same or within) of the previous.
Layer (2) above is new, with this patch. The call used to check whether a
zone (its node, actually) is in a cpuset (in its mems_allowed, actually) is
extended to take a gfp_mask argument, and its logic is extended, in the case
that __GFP_HARDWALL is not set in the flag bits, to look up the cpuset
hierarchy for the nearest enclosing mem_exclusive cpuset, to determine if
placement is allowed. The definition of GFP_USER, which used to be identical
to GFP_KERNEL, is changed to also set the __GFP_HARDWALL bit, in the previous
cpuset_gfp_hardwall_flag patch.
GFP_ATOMIC and GFP_KERNEL allocations will stay within the current tasks
cpuset, so long as any node therein is not too tight on memory, but will
escape to the larger layer, if need be.
The intended use is to allow something like a batch manager to handle several
jobs, each job in its own cpuset, but using common kernel memory for caches
and such. Swapper and oom_kill activity is also constrained to Layer (2). A
task in or below one mem_exclusive cpuset should not cause swapping on nodes
in another non-overlapping mem_exclusive cpuset, nor provoke oom_killing of a
task in another such cpuset. Heavy use of kernel memory for i/o caching and
such by one job should not impact the memory available to jobs in other
non-overlapping mem_exclusive cpusets.
This patch enables providing hardwall, inescapable cpusets for memory
allocations of each job, while sharing kernel memory allocations between
several jobs, in an enclosing mem_exclusive cpuset.
Like Dinakar's patch earlier to enable administering sched domains using the
cpu_exclusive flag, this patch also provides a useful meaning to a cpuset flag
that had previously done nothing much useful other than restrict what cpuset
configurations were allowed.
Signed-off-by: Paul Jackson <pj@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-06 16:18:12 -06:00
|
|
|
*/
|
2009-06-16 16:32:02 -06:00
|
|
|
alloc_flags = gfp_to_alloc_flags(gfp_mask);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2016-07-28 16:49:16 -06:00
|
|
|
if (gfp_mask & __GFP_KSWAPD_RECLAIM)
|
|
|
|
wake_all_kswapds(order, ac);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The adjusted alloc_flags might result in immediate success, so try
|
|
|
|
* that first
|
|
|
|
*/
|
|
|
|
page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
|
|
|
|
if (page)
|
|
|
|
goto got_pg;
|
|
|
|
|
mm, page_alloc: restructure direct compaction handling in slowpath
The retry loop in __alloc_pages_slowpath is supposed to keep trying
reclaim and compaction (and OOM), until either the allocation succeeds,
or returns with failure. Success here is more probable when reclaim
precedes compaction, as certain watermarks have to be met for compaction
to even try, and more free pages increase the probability of compaction
success. On the other hand, starting with light async compaction (if
the watermarks allow it), can be more efficient, especially for smaller
orders, if there's enough free memory which is just fragmented.
Thus, the current code starts with compaction before reclaim, and to
make sure that the last reclaim is always followed by a final
compaction, there's another direct compaction call at the end of the
loop. This makes the code hard to follow and adds some duplicated
handling of migration_mode decisions. It's also somewhat inefficient
that even if reclaim or compaction decides not to retry, the final
compaction is still attempted. Some gfp flags combination also shortcut
these retry decisions by "goto noretry;", making it even harder to
follow.
This patch attempts to restructure the code with only minimal functional
changes. The call to the first compaction and THP-specific checks are
now placed above the retry loop, and the "noretry" direct compaction is
removed.
The initial compaction is additionally restricted only to costly orders,
as we can expect smaller orders to be held back by watermarks, and only
larger orders to suffer primarily from fragmentation. This better
matches the checks in reclaim's shrink_zones().
There are two other smaller functional changes. One is that the upgrade
from async migration to light sync migration will always occur after the
initial compaction. This is how it has been until recent patch "mm,
oom: protect !costly allocations some more", which introduced upgrading
the mode based on COMPACT_COMPLETE result, but kept the final compaction
always upgraded, which made it even more special. It's better to return
to the simpler handling for now, as migration modes will be further
modified later in the series.
The second change is that once both reclaim and compaction declare it's
not worth to retry the reclaim/compact loop, there is no final
compaction attempt. As argued above, this is intentional. If that
final compaction were to succeed, it would be due to a wrong retry
decision, or simply a race with somebody else freeing memory for us.
The main outcome of this patch should be simpler code. Logically, the
initial compaction without reclaim is the exceptional case to the
reclaim/compaction scheme, but prior to the patch, it was the last loop
iteration that was exceptional. Now the code matches the logic better.
The change also enable the following patches.
Link: http://lkml.kernel.org/r/20160721073614.24395-5-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:19 -06:00
|
|
|
/*
|
|
|
|
* For costly allocations, try direct compaction first, as it's likely
|
|
|
|
* that we have enough base pages and don't need to reclaim. Don't try
|
|
|
|
* that for allocations that are allowed to ignore watermarks, as the
|
|
|
|
* ALLOC_NO_WATERMARKS attempt didn't yet happen.
|
|
|
|
*/
|
|
|
|
if (can_direct_reclaim && order > PAGE_ALLOC_COSTLY_ORDER &&
|
|
|
|
!gfp_pfmemalloc_allowed(gfp_mask)) {
|
|
|
|
page = __alloc_pages_direct_compact(gfp_mask, order,
|
|
|
|
alloc_flags, ac,
|
2016-07-28 16:49:28 -06:00
|
|
|
INIT_COMPACT_PRIORITY,
|
mm, page_alloc: restructure direct compaction handling in slowpath
The retry loop in __alloc_pages_slowpath is supposed to keep trying
reclaim and compaction (and OOM), until either the allocation succeeds,
or returns with failure. Success here is more probable when reclaim
precedes compaction, as certain watermarks have to be met for compaction
to even try, and more free pages increase the probability of compaction
success. On the other hand, starting with light async compaction (if
the watermarks allow it), can be more efficient, especially for smaller
orders, if there's enough free memory which is just fragmented.
Thus, the current code starts with compaction before reclaim, and to
make sure that the last reclaim is always followed by a final
compaction, there's another direct compaction call at the end of the
loop. This makes the code hard to follow and adds some duplicated
handling of migration_mode decisions. It's also somewhat inefficient
that even if reclaim or compaction decides not to retry, the final
compaction is still attempted. Some gfp flags combination also shortcut
these retry decisions by "goto noretry;", making it even harder to
follow.
This patch attempts to restructure the code with only minimal functional
changes. The call to the first compaction and THP-specific checks are
now placed above the retry loop, and the "noretry" direct compaction is
removed.
The initial compaction is additionally restricted only to costly orders,
as we can expect smaller orders to be held back by watermarks, and only
larger orders to suffer primarily from fragmentation. This better
matches the checks in reclaim's shrink_zones().
There are two other smaller functional changes. One is that the upgrade
from async migration to light sync migration will always occur after the
initial compaction. This is how it has been until recent patch "mm,
oom: protect !costly allocations some more", which introduced upgrading
the mode based on COMPACT_COMPLETE result, but kept the final compaction
always upgraded, which made it even more special. It's better to return
to the simpler handling for now, as migration modes will be further
modified later in the series.
The second change is that once both reclaim and compaction declare it's
not worth to retry the reclaim/compact loop, there is no final
compaction attempt. As argued above, this is intentional. If that
final compaction were to succeed, it would be due to a wrong retry
decision, or simply a race with somebody else freeing memory for us.
The main outcome of this patch should be simpler code. Logically, the
initial compaction without reclaim is the exceptional case to the
reclaim/compaction scheme, but prior to the patch, it was the last loop
iteration that was exceptional. Now the code matches the logic better.
The change also enable the following patches.
Link: http://lkml.kernel.org/r/20160721073614.24395-5-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:19 -06:00
|
|
|
&compact_result);
|
|
|
|
if (page)
|
|
|
|
goto got_pg;
|
|
|
|
|
mm, page_alloc: make THP-specific decisions more generic
Since THP allocations during page faults can be costly, extra decisions
are employed for them to avoid excessive reclaim and compaction, if the
initial compaction doesn't look promising. The detection has never been
perfect as there is no gfp flag specific to THP allocations. At this
moment it checks the whole combination of flags that makes up
GFP_TRANSHUGE, and hopes that no other users of such combination exist,
or would mind being treated the same way. Extra care is also taken to
separate allocations from khugepaged, where latency doesn't matter that
much.
It is however possible to distinguish these allocations in a simpler and
more reliable way. The key observation is that after the initial
compaction followed by the first iteration of "standard"
reclaim/compaction, both __GFP_NORETRY allocations and costly
allocations without __GFP_REPEAT are declared as failures:
/* Do not loop if specifically requested */
if (gfp_mask & __GFP_NORETRY)
goto nopage;
/*
* Do not retry costly high order allocations unless they are
* __GFP_REPEAT
*/
if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_REPEAT))
goto nopage;
This means we can further distinguish allocations that are costly order
*and* additionally include the __GFP_NORETRY flag. As it happens,
GFP_TRANSHUGE allocations do already fall into this category. This will
also allow other costly allocations with similar high-order benefit vs
latency considerations to use this semantic. Furthermore, we can
distinguish THP allocations that should try a bit harder (such as from
khugepageed) by removing __GFP_NORETRY, as will be done in the next
patch.
Link: http://lkml.kernel.org/r/20160721073614.24395-6-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:22 -06:00
|
|
|
/*
|
|
|
|
* Checks for costly allocations with __GFP_NORETRY, which
|
|
|
|
* includes THP page fault allocations
|
|
|
|
*/
|
|
|
|
if (gfp_mask & __GFP_NORETRY) {
|
mm, page_alloc: restructure direct compaction handling in slowpath
The retry loop in __alloc_pages_slowpath is supposed to keep trying
reclaim and compaction (and OOM), until either the allocation succeeds,
or returns with failure. Success here is more probable when reclaim
precedes compaction, as certain watermarks have to be met for compaction
to even try, and more free pages increase the probability of compaction
success. On the other hand, starting with light async compaction (if
the watermarks allow it), can be more efficient, especially for smaller
orders, if there's enough free memory which is just fragmented.
Thus, the current code starts with compaction before reclaim, and to
make sure that the last reclaim is always followed by a final
compaction, there's another direct compaction call at the end of the
loop. This makes the code hard to follow and adds some duplicated
handling of migration_mode decisions. It's also somewhat inefficient
that even if reclaim or compaction decides not to retry, the final
compaction is still attempted. Some gfp flags combination also shortcut
these retry decisions by "goto noretry;", making it even harder to
follow.
This patch attempts to restructure the code with only minimal functional
changes. The call to the first compaction and THP-specific checks are
now placed above the retry loop, and the "noretry" direct compaction is
removed.
The initial compaction is additionally restricted only to costly orders,
as we can expect smaller orders to be held back by watermarks, and only
larger orders to suffer primarily from fragmentation. This better
matches the checks in reclaim's shrink_zones().
There are two other smaller functional changes. One is that the upgrade
from async migration to light sync migration will always occur after the
initial compaction. This is how it has been until recent patch "mm,
oom: protect !costly allocations some more", which introduced upgrading
the mode based on COMPACT_COMPLETE result, but kept the final compaction
always upgraded, which made it even more special. It's better to return
to the simpler handling for now, as migration modes will be further
modified later in the series.
The second change is that once both reclaim and compaction declare it's
not worth to retry the reclaim/compact loop, there is no final
compaction attempt. As argued above, this is intentional. If that
final compaction were to succeed, it would be due to a wrong retry
decision, or simply a race with somebody else freeing memory for us.
The main outcome of this patch should be simpler code. Logically, the
initial compaction without reclaim is the exceptional case to the
reclaim/compaction scheme, but prior to the patch, it was the last loop
iteration that was exceptional. Now the code matches the logic better.
The change also enable the following patches.
Link: http://lkml.kernel.org/r/20160721073614.24395-5-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:19 -06:00
|
|
|
/*
|
|
|
|
* If compaction is deferred for high-order allocations,
|
|
|
|
* it is because sync compaction recently failed. If
|
|
|
|
* this is the case and the caller requested a THP
|
|
|
|
* allocation, we do not want to heavily disrupt the
|
|
|
|
* system, so we fail the allocation instead of entering
|
|
|
|
* direct reclaim.
|
|
|
|
*/
|
|
|
|
if (compact_result == COMPACT_DEFERRED)
|
|
|
|
goto nopage;
|
|
|
|
|
|
|
|
/*
|
mm, page_alloc: make THP-specific decisions more generic
Since THP allocations during page faults can be costly, extra decisions
are employed for them to avoid excessive reclaim and compaction, if the
initial compaction doesn't look promising. The detection has never been
perfect as there is no gfp flag specific to THP allocations. At this
moment it checks the whole combination of flags that makes up
GFP_TRANSHUGE, and hopes that no other users of such combination exist,
or would mind being treated the same way. Extra care is also taken to
separate allocations from khugepaged, where latency doesn't matter that
much.
It is however possible to distinguish these allocations in a simpler and
more reliable way. The key observation is that after the initial
compaction followed by the first iteration of "standard"
reclaim/compaction, both __GFP_NORETRY allocations and costly
allocations without __GFP_REPEAT are declared as failures:
/* Do not loop if specifically requested */
if (gfp_mask & __GFP_NORETRY)
goto nopage;
/*
* Do not retry costly high order allocations unless they are
* __GFP_REPEAT
*/
if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_REPEAT))
goto nopage;
This means we can further distinguish allocations that are costly order
*and* additionally include the __GFP_NORETRY flag. As it happens,
GFP_TRANSHUGE allocations do already fall into this category. This will
also allow other costly allocations with similar high-order benefit vs
latency considerations to use this semantic. Furthermore, we can
distinguish THP allocations that should try a bit harder (such as from
khugepageed) by removing __GFP_NORETRY, as will be done in the next
patch.
Link: http://lkml.kernel.org/r/20160721073614.24395-6-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:22 -06:00
|
|
|
* Looks like reclaim/compaction is worth trying, but
|
|
|
|
* sync compaction could be very expensive, so keep
|
mm, thp: remove __GFP_NORETRY from khugepaged and madvised allocations
After the previous patch, we can distinguish costly allocations that
should be really lightweight, such as THP page faults, with
__GFP_NORETRY. This means we don't need to recognize khugepaged
allocations via PF_KTHREAD anymore. We can also change THP page faults
in areas where madvise(MADV_HUGEPAGE) was used to try as hard as
khugepaged, as the process has indicated that it benefits from THP's and
is willing to pay some initial latency costs.
We can also make the flags handling less cryptic by distinguishing
GFP_TRANSHUGE_LIGHT (no reclaim at all, default mode in page fault) from
GFP_TRANSHUGE (only direct reclaim, khugepaged default). Adding
__GFP_NORETRY or __GFP_KSWAPD_RECLAIM is done where needed.
The patch effectively changes the current GFP_TRANSHUGE users as
follows:
* get_huge_zero_page() - the zero page lifetime should be relatively
long and it's shared by multiple users, so it's worth spending some
effort on it. We use GFP_TRANSHUGE, and __GFP_NORETRY is not added.
This also restores direct reclaim to this allocation, which was
unintentionally removed by commit e4a49efe4e7e ("mm: thp: set THP defrag
by default to madvise and add a stall-free defrag option")
* alloc_hugepage_khugepaged_gfpmask() - this is khugepaged, so latency
is not an issue. So if khugepaged "defrag" is enabled (the default), do
reclaim via GFP_TRANSHUGE without __GFP_NORETRY. We can remove the
PF_KTHREAD check from page alloc.
As a side-effect, khugepaged will now no longer check if the initial
compaction was deferred or contended. This is OK, as khugepaged sleep
times between collapsion attempts are long enough to prevent noticeable
disruption, so we should allow it to spend some effort.
* migrate_misplaced_transhuge_page() - already was masking out
__GFP_RECLAIM, so just convert to GFP_TRANSHUGE_LIGHT which is
equivalent.
* alloc_hugepage_direct_gfpmask() - vma's with VM_HUGEPAGE (via madvise)
are now allocating without __GFP_NORETRY. Other vma's keep using
__GFP_NORETRY if direct reclaim/compaction is at all allowed (by default
it's allowed only for madvised vma's). The rest is conversion to
GFP_TRANSHUGE(_LIGHT).
[mhocko@suse.com: suggested GFP_TRANSHUGE_LIGHT]
Link: http://lkml.kernel.org/r/20160721073614.24395-7-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:25 -06:00
|
|
|
* using async compaction.
|
mm, page_alloc: restructure direct compaction handling in slowpath
The retry loop in __alloc_pages_slowpath is supposed to keep trying
reclaim and compaction (and OOM), until either the allocation succeeds,
or returns with failure. Success here is more probable when reclaim
precedes compaction, as certain watermarks have to be met for compaction
to even try, and more free pages increase the probability of compaction
success. On the other hand, starting with light async compaction (if
the watermarks allow it), can be more efficient, especially for smaller
orders, if there's enough free memory which is just fragmented.
Thus, the current code starts with compaction before reclaim, and to
make sure that the last reclaim is always followed by a final
compaction, there's another direct compaction call at the end of the
loop. This makes the code hard to follow and adds some duplicated
handling of migration_mode decisions. It's also somewhat inefficient
that even if reclaim or compaction decides not to retry, the final
compaction is still attempted. Some gfp flags combination also shortcut
these retry decisions by "goto noretry;", making it even harder to
follow.
This patch attempts to restructure the code with only minimal functional
changes. The call to the first compaction and THP-specific checks are
now placed above the retry loop, and the "noretry" direct compaction is
removed.
The initial compaction is additionally restricted only to costly orders,
as we can expect smaller orders to be held back by watermarks, and only
larger orders to suffer primarily from fragmentation. This better
matches the checks in reclaim's shrink_zones().
There are two other smaller functional changes. One is that the upgrade
from async migration to light sync migration will always occur after the
initial compaction. This is how it has been until recent patch "mm,
oom: protect !costly allocations some more", which introduced upgrading
the mode based on COMPACT_COMPLETE result, but kept the final compaction
always upgraded, which made it even more special. It's better to return
to the simpler handling for now, as migration modes will be further
modified later in the series.
The second change is that once both reclaim and compaction declare it's
not worth to retry the reclaim/compact loop, there is no final
compaction attempt. As argued above, this is intentional. If that
final compaction were to succeed, it would be due to a wrong retry
decision, or simply a race with somebody else freeing memory for us.
The main outcome of this patch should be simpler code. Logically, the
initial compaction without reclaim is the exceptional case to the
reclaim/compaction scheme, but prior to the patch, it was the last loop
iteration that was exceptional. Now the code matches the logic better.
The change also enable the following patches.
Link: http://lkml.kernel.org/r/20160721073614.24395-5-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:19 -06:00
|
|
|
*/
|
2016-07-28 16:49:28 -06:00
|
|
|
compact_priority = INIT_COMPACT_PRIORITY;
|
mm, page_alloc: restructure direct compaction handling in slowpath
The retry loop in __alloc_pages_slowpath is supposed to keep trying
reclaim and compaction (and OOM), until either the allocation succeeds,
or returns with failure. Success here is more probable when reclaim
precedes compaction, as certain watermarks have to be met for compaction
to even try, and more free pages increase the probability of compaction
success. On the other hand, starting with light async compaction (if
the watermarks allow it), can be more efficient, especially for smaller
orders, if there's enough free memory which is just fragmented.
Thus, the current code starts with compaction before reclaim, and to
make sure that the last reclaim is always followed by a final
compaction, there's another direct compaction call at the end of the
loop. This makes the code hard to follow and adds some duplicated
handling of migration_mode decisions. It's also somewhat inefficient
that even if reclaim or compaction decides not to retry, the final
compaction is still attempted. Some gfp flags combination also shortcut
these retry decisions by "goto noretry;", making it even harder to
follow.
This patch attempts to restructure the code with only minimal functional
changes. The call to the first compaction and THP-specific checks are
now placed above the retry loop, and the "noretry" direct compaction is
removed.
The initial compaction is additionally restricted only to costly orders,
as we can expect smaller orders to be held back by watermarks, and only
larger orders to suffer primarily from fragmentation. This better
matches the checks in reclaim's shrink_zones().
There are two other smaller functional changes. One is that the upgrade
from async migration to light sync migration will always occur after the
initial compaction. This is how it has been until recent patch "mm,
oom: protect !costly allocations some more", which introduced upgrading
the mode based on COMPACT_COMPLETE result, but kept the final compaction
always upgraded, which made it even more special. It's better to return
to the simpler handling for now, as migration modes will be further
modified later in the series.
The second change is that once both reclaim and compaction declare it's
not worth to retry the reclaim/compact loop, there is no final
compaction attempt. As argued above, this is intentional. If that
final compaction were to succeed, it would be due to a wrong retry
decision, or simply a race with somebody else freeing memory for us.
The main outcome of this patch should be simpler code. Logically, the
initial compaction without reclaim is the exceptional case to the
reclaim/compaction scheme, but prior to the patch, it was the last loop
iteration that was exceptional. Now the code matches the logic better.
The change also enable the following patches.
Link: http://lkml.kernel.org/r/20160721073614.24395-5-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:19 -06:00
|
|
|
}
|
|
|
|
}
|
2016-07-28 16:49:16 -06:00
|
|
|
|
2016-07-28 16:49:13 -06:00
|
|
|
retry:
|
2016-07-28 16:49:16 -06:00
|
|
|
/* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
|
2016-07-28 16:49:13 -06:00
|
|
|
if (gfp_mask & __GFP_KSWAPD_RECLAIM)
|
|
|
|
wake_all_kswapds(order, ac);
|
|
|
|
|
2016-07-28 16:49:16 -06:00
|
|
|
if (gfp_pfmemalloc_allowed(gfp_mask))
|
|
|
|
alloc_flags = ALLOC_NO_WATERMARKS;
|
|
|
|
|
2016-06-03 15:56:01 -06:00
|
|
|
/*
|
|
|
|
* Reset the zonelist iterators if memory policies can be ignored.
|
|
|
|
* These allocations are high priority and system rather than user
|
|
|
|
* orientated.
|
|
|
|
*/
|
2016-07-28 16:49:16 -06:00
|
|
|
if (!(alloc_flags & ALLOC_CPUSET) || (alloc_flags & ALLOC_NO_WATERMARKS)) {
|
2016-06-03 15:56:01 -06:00
|
|
|
ac->zonelist = node_zonelist(numa_node_id(), gfp_mask);
|
|
|
|
ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
|
|
|
|
ac->high_zoneidx, ac->nodemask);
|
|
|
|
}
|
|
|
|
|
2016-07-28 16:49:16 -06:00
|
|
|
/* Attempt with potentially adjusted zonelist and alloc_flags */
|
2016-07-28 16:49:13 -06:00
|
|
|
page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
|
2005-11-13 17:06:43 -07:00
|
|
|
if (page)
|
|
|
|
goto got_pg;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2015-11-06 17:28:21 -07:00
|
|
|
/* Caller is not willing to reclaim, we can't balance anything */
|
|
|
|
if (!can_direct_reclaim) {
|
2014-01-21 16:51:12 -07:00
|
|
|
/*
|
2016-01-14 16:19:05 -07:00
|
|
|
* All existing users of the __GFP_NOFAIL are blockable, so warn
|
|
|
|
* of any new users that actually allow this type of allocation
|
|
|
|
* to fail.
|
2014-01-21 16:51:12 -07:00
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL);
|
2005-04-16 16:20:36 -06:00
|
|
|
goto nopage;
|
2014-01-21 16:51:12 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-06-16 16:32:02 -06:00
|
|
|
/* Avoid recursion of direct reclaim */
|
2016-01-14 16:19:05 -07:00
|
|
|
if (current->flags & PF_MEMALLOC) {
|
|
|
|
/*
|
|
|
|
* __GFP_NOFAIL request from this context is rather bizarre
|
|
|
|
* because we cannot reclaim anything and only can loop waiting
|
|
|
|
* for somebody to do a work for us.
|
|
|
|
*/
|
|
|
|
if (WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
|
|
|
|
cond_resched();
|
|
|
|
goto retry;
|
|
|
|
}
|
2009-06-16 16:32:02 -06:00
|
|
|
goto nopage;
|
2016-01-14 16:19:05 -07:00
|
|
|
}
|
2009-06-16 16:32:02 -06:00
|
|
|
|
2009-07-29 16:02:06 -06:00
|
|
|
/* Avoid allocations with no watermarks from looping endlessly */
|
|
|
|
if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
|
|
|
|
goto nopage;
|
|
|
|
|
mm, page_alloc: restructure direct compaction handling in slowpath
The retry loop in __alloc_pages_slowpath is supposed to keep trying
reclaim and compaction (and OOM), until either the allocation succeeds,
or returns with failure. Success here is more probable when reclaim
precedes compaction, as certain watermarks have to be met for compaction
to even try, and more free pages increase the probability of compaction
success. On the other hand, starting with light async compaction (if
the watermarks allow it), can be more efficient, especially for smaller
orders, if there's enough free memory which is just fragmented.
Thus, the current code starts with compaction before reclaim, and to
make sure that the last reclaim is always followed by a final
compaction, there's another direct compaction call at the end of the
loop. This makes the code hard to follow and adds some duplicated
handling of migration_mode decisions. It's also somewhat inefficient
that even if reclaim or compaction decides not to retry, the final
compaction is still attempted. Some gfp flags combination also shortcut
these retry decisions by "goto noretry;", making it even harder to
follow.
This patch attempts to restructure the code with only minimal functional
changes. The call to the first compaction and THP-specific checks are
now placed above the retry loop, and the "noretry" direct compaction is
removed.
The initial compaction is additionally restricted only to costly orders,
as we can expect smaller orders to be held back by watermarks, and only
larger orders to suffer primarily from fragmentation. This better
matches the checks in reclaim's shrink_zones().
There are two other smaller functional changes. One is that the upgrade
from async migration to light sync migration will always occur after the
initial compaction. This is how it has been until recent patch "mm,
oom: protect !costly allocations some more", which introduced upgrading
the mode based on COMPACT_COMPLETE result, but kept the final compaction
always upgraded, which made it even more special. It's better to return
to the simpler handling for now, as migration modes will be further
modified later in the series.
The second change is that once both reclaim and compaction declare it's
not worth to retry the reclaim/compact loop, there is no final
compaction attempt. As argued above, this is intentional. If that
final compaction were to succeed, it would be due to a wrong retry
decision, or simply a race with somebody else freeing memory for us.
The main outcome of this patch should be simpler code. Logically, the
initial compaction without reclaim is the exceptional case to the
reclaim/compaction scheme, but prior to the patch, it was the last loop
iteration that was exceptional. Now the code matches the logic better.
The change also enable the following patches.
Link: http://lkml.kernel.org/r/20160721073614.24395-5-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:19 -06:00
|
|
|
|
|
|
|
/* Try direct reclaim and then allocating */
|
|
|
|
page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
|
|
|
|
&did_some_progress);
|
|
|
|
if (page)
|
|
|
|
goto got_pg;
|
|
|
|
|
|
|
|
/* Try direct compaction and then allocating */
|
2015-02-11 16:25:41 -07:00
|
|
|
page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
|
2016-07-28 16:49:28 -06:00
|
|
|
compact_priority, &compact_result);
|
2010-05-24 15:32:30 -06:00
|
|
|
if (page)
|
|
|
|
goto got_pg;
|
2014-06-04 17:08:30 -06:00
|
|
|
|
2016-05-20 17:57:06 -06:00
|
|
|
if (order && compaction_made_progress(compact_result))
|
|
|
|
compaction_retries++;
|
2014-08-06 17:07:54 -06:00
|
|
|
|
2015-06-24 17:57:21 -06:00
|
|
|
/* Do not loop if specifically requested */
|
|
|
|
if (gfp_mask & __GFP_NORETRY)
|
mm, page_alloc: restructure direct compaction handling in slowpath
The retry loop in __alloc_pages_slowpath is supposed to keep trying
reclaim and compaction (and OOM), until either the allocation succeeds,
or returns with failure. Success here is more probable when reclaim
precedes compaction, as certain watermarks have to be met for compaction
to even try, and more free pages increase the probability of compaction
success. On the other hand, starting with light async compaction (if
the watermarks allow it), can be more efficient, especially for smaller
orders, if there's enough free memory which is just fragmented.
Thus, the current code starts with compaction before reclaim, and to
make sure that the last reclaim is always followed by a final
compaction, there's another direct compaction call at the end of the
loop. This makes the code hard to follow and adds some duplicated
handling of migration_mode decisions. It's also somewhat inefficient
that even if reclaim or compaction decides not to retry, the final
compaction is still attempted. Some gfp flags combination also shortcut
these retry decisions by "goto noretry;", making it even harder to
follow.
This patch attempts to restructure the code with only minimal functional
changes. The call to the first compaction and THP-specific checks are
now placed above the retry loop, and the "noretry" direct compaction is
removed.
The initial compaction is additionally restricted only to costly orders,
as we can expect smaller orders to be held back by watermarks, and only
larger orders to suffer primarily from fragmentation. This better
matches the checks in reclaim's shrink_zones().
There are two other smaller functional changes. One is that the upgrade
from async migration to light sync migration will always occur after the
initial compaction. This is how it has been until recent patch "mm,
oom: protect !costly allocations some more", which introduced upgrading
the mode based on COMPACT_COMPLETE result, but kept the final compaction
always upgraded, which made it even more special. It's better to return
to the simpler handling for now, as migration modes will be further
modified later in the series.
The second change is that once both reclaim and compaction declare it's
not worth to retry the reclaim/compact loop, there is no final
compaction attempt. As argued above, this is intentional. If that
final compaction were to succeed, it would be due to a wrong retry
decision, or simply a race with somebody else freeing memory for us.
The main outcome of this patch should be simpler code. Logically, the
initial compaction without reclaim is the exceptional case to the
reclaim/compaction scheme, but prior to the patch, it was the last loop
iteration that was exceptional. Now the code matches the logic better.
The change also enable the following patches.
Link: http://lkml.kernel.org/r/20160721073614.24395-5-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:19 -06:00
|
|
|
goto nopage;
|
2015-06-24 17:57:21 -06:00
|
|
|
|
2016-05-20 17:57:00 -06:00
|
|
|
/*
|
|
|
|
* Do not retry costly high order allocations unless they are
|
|
|
|
* __GFP_REPEAT
|
|
|
|
*/
|
|
|
|
if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_REPEAT))
|
mm, page_alloc: restructure direct compaction handling in slowpath
The retry loop in __alloc_pages_slowpath is supposed to keep trying
reclaim and compaction (and OOM), until either the allocation succeeds,
or returns with failure. Success here is more probable when reclaim
precedes compaction, as certain watermarks have to be met for compaction
to even try, and more free pages increase the probability of compaction
success. On the other hand, starting with light async compaction (if
the watermarks allow it), can be more efficient, especially for smaller
orders, if there's enough free memory which is just fragmented.
Thus, the current code starts with compaction before reclaim, and to
make sure that the last reclaim is always followed by a final
compaction, there's another direct compaction call at the end of the
loop. This makes the code hard to follow and adds some duplicated
handling of migration_mode decisions. It's also somewhat inefficient
that even if reclaim or compaction decides not to retry, the final
compaction is still attempted. Some gfp flags combination also shortcut
these retry decisions by "goto noretry;", making it even harder to
follow.
This patch attempts to restructure the code with only minimal functional
changes. The call to the first compaction and THP-specific checks are
now placed above the retry loop, and the "noretry" direct compaction is
removed.
The initial compaction is additionally restricted only to costly orders,
as we can expect smaller orders to be held back by watermarks, and only
larger orders to suffer primarily from fragmentation. This better
matches the checks in reclaim's shrink_zones().
There are two other smaller functional changes. One is that the upgrade
from async migration to light sync migration will always occur after the
initial compaction. This is how it has been until recent patch "mm,
oom: protect !costly allocations some more", which introduced upgrading
the mode based on COMPACT_COMPLETE result, but kept the final compaction
always upgraded, which made it even more special. It's better to return
to the simpler handling for now, as migration modes will be further
modified later in the series.
The second change is that once both reclaim and compaction declare it's
not worth to retry the reclaim/compact loop, there is no final
compaction attempt. As argued above, this is intentional. If that
final compaction were to succeed, it would be due to a wrong retry
decision, or simply a race with somebody else freeing memory for us.
The main outcome of this patch should be simpler code. Logically, the
initial compaction without reclaim is the exceptional case to the
reclaim/compaction scheme, but prior to the patch, it was the last loop
iteration that was exceptional. Now the code matches the logic better.
The change also enable the following patches.
Link: http://lkml.kernel.org/r/20160721073614.24395-5-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:49:19 -06:00
|
|
|
goto nopage;
|
2016-05-20 17:57:00 -06:00
|
|
|
|
mm: consider compaction feedback also for costly allocation
PAGE_ALLOC_COSTLY_ORDER retry logic is mostly handled inside
should_reclaim_retry currently where we decide to not retry after at
least order worth of pages were reclaimed or the watermark check for at
least one zone would succeed after reclaiming all pages if the reclaim
hasn't made any progress. Compaction feedback is mostly ignored and we
just try to make sure that the compaction did at least something before
giving up.
The first condition was added by a41f24ea9fd6 ("page allocator: smarter
retry of costly-order allocations) and it assumed that lumpy reclaim
could have created a page of the sufficient order. Lumpy reclaim, has
been removed quite some time ago so the assumption doesn't hold anymore.
Remove the check for the number of reclaimed pages and rely on the
compaction feedback solely. should_reclaim_retry now only makes sure
that we keep retrying reclaim for high order pages only if they are
hidden by watermaks so order-0 reclaim makes really sense.
should_compact_retry now keeps retrying even for the costly allocations.
The number of retries is reduced wrt. !costly requests because they are
less important and harder to grant and so their pressure shouldn't cause
contention for other requests or cause an over reclaim. We also do not
reset no_progress_loops for costly request to make sure we do not keep
reclaiming too agressively.
This has been tested by running a process which fragments memory:
- compact memory
- mmap large portion of the memory (1920M on 2GRAM machine with 2G
of swapspace)
- MADV_DONTNEED single page in PAGE_SIZE*((1UL<<MAX_ORDER)-1)
steps until certain amount of memory is freed (250M in my test)
and reduce the step to (step / 2) + 1 after reaching the end of
the mapping
- then run a script which populates the page cache 2G (MemTotal)
from /dev/zero to a new file
And then tries to allocate
nr_hugepages=$(awk '/MemAvailable/{printf "%d\n", $2/(2*1024)}' /proc/meminfo)
huge pages.
root@test1:~# echo 1 > /proc/sys/vm/overcommit_memory;echo 1 > /proc/sys/vm/compact_memory; ./fragment-mem-and-run /root/alloc_hugepages.sh 1920M 250M
Node 0, zone DMA 31 28 31 10 2 0 2 1 2 3 1
Node 0, zone DMA32 437 319 171 50 28 25 20 16 16 14 437
* This is the /proc/buddyinfo after the compaction
Done fragmenting. size=2013265920 freed=262144000
Node 0, zone DMA 165 48 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 35109 14575 185 51 41 12 6 0 0 0 0
* /proc/buddyinfo after memory got fragmented
Executing "/root/alloc_hugepages.sh"
Eating some pagecache
508623+0 records in
508623+0 records out
2083319808 bytes (2.1 GB) copied, 11.7292 s, 178 MB/s
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 111 344 153 20 24 10 3 0 0 0 0
* /proc/buddyinfo after page cache got eaten
Trying to allocate 129
129
* 129 hugepages requested and all of them granted.
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 127 97 30 99 11 6 2 1 4 0 0
* /proc/buddyinfo after hugetlb allocation.
10 runs will behave as follows:
Trying to allocate 130
130
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 132
132
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
So basically 100% success for all 10 attempts.
Without the patch numbers looked much worse:
Trying to allocate 128
12
--
Trying to allocate 129
14
--
Trying to allocate 129
7
--
Trying to allocate 129
16
--
Trying to allocate 129
30
--
Trying to allocate 129
38
--
Trying to allocate 129
19
--
Trying to allocate 129
37
--
Trying to allocate 129
28
--
Trying to allocate 129
37
Just for completness the base kernel without oom detection rework looks
as follows:
Trying to allocate 127
30
--
Trying to allocate 129
12
--
Trying to allocate 129
52
--
Trying to allocate 128
32
--
Trying to allocate 129
12
--
Trying to allocate 129
10
--
Trying to allocate 129
32
--
Trying to allocate 128
14
--
Trying to allocate 128
16
--
Trying to allocate 129
8
As we can see the success rate is much more volatile and smaller without
this patch. So the patch not only makes the retry logic for costly
requests more sensible the success rate is even higher.
Signed-off-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 17:57:09 -06:00
|
|
|
/*
|
|
|
|
* Costly allocations might have made a progress but this doesn't mean
|
|
|
|
* their order will become available due to high fragmentation so
|
|
|
|
* always increment the no progress counter for them
|
|
|
|
*/
|
|
|
|
if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
|
2016-05-20 17:57:00 -06:00
|
|
|
no_progress_loops = 0;
|
mm: consider compaction feedback also for costly allocation
PAGE_ALLOC_COSTLY_ORDER retry logic is mostly handled inside
should_reclaim_retry currently where we decide to not retry after at
least order worth of pages were reclaimed or the watermark check for at
least one zone would succeed after reclaiming all pages if the reclaim
hasn't made any progress. Compaction feedback is mostly ignored and we
just try to make sure that the compaction did at least something before
giving up.
The first condition was added by a41f24ea9fd6 ("page allocator: smarter
retry of costly-order allocations) and it assumed that lumpy reclaim
could have created a page of the sufficient order. Lumpy reclaim, has
been removed quite some time ago so the assumption doesn't hold anymore.
Remove the check for the number of reclaimed pages and rely on the
compaction feedback solely. should_reclaim_retry now only makes sure
that we keep retrying reclaim for high order pages only if they are
hidden by watermaks so order-0 reclaim makes really sense.
should_compact_retry now keeps retrying even for the costly allocations.
The number of retries is reduced wrt. !costly requests because they are
less important and harder to grant and so their pressure shouldn't cause
contention for other requests or cause an over reclaim. We also do not
reset no_progress_loops for costly request to make sure we do not keep
reclaiming too agressively.
This has been tested by running a process which fragments memory:
- compact memory
- mmap large portion of the memory (1920M on 2GRAM machine with 2G
of swapspace)
- MADV_DONTNEED single page in PAGE_SIZE*((1UL<<MAX_ORDER)-1)
steps until certain amount of memory is freed (250M in my test)
and reduce the step to (step / 2) + 1 after reaching the end of
the mapping
- then run a script which populates the page cache 2G (MemTotal)
from /dev/zero to a new file
And then tries to allocate
nr_hugepages=$(awk '/MemAvailable/{printf "%d\n", $2/(2*1024)}' /proc/meminfo)
huge pages.
root@test1:~# echo 1 > /proc/sys/vm/overcommit_memory;echo 1 > /proc/sys/vm/compact_memory; ./fragment-mem-and-run /root/alloc_hugepages.sh 1920M 250M
Node 0, zone DMA 31 28 31 10 2 0 2 1 2 3 1
Node 0, zone DMA32 437 319 171 50 28 25 20 16 16 14 437
* This is the /proc/buddyinfo after the compaction
Done fragmenting. size=2013265920 freed=262144000
Node 0, zone DMA 165 48 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 35109 14575 185 51 41 12 6 0 0 0 0
* /proc/buddyinfo after memory got fragmented
Executing "/root/alloc_hugepages.sh"
Eating some pagecache
508623+0 records in
508623+0 records out
2083319808 bytes (2.1 GB) copied, 11.7292 s, 178 MB/s
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 111 344 153 20 24 10 3 0 0 0 0
* /proc/buddyinfo after page cache got eaten
Trying to allocate 129
129
* 129 hugepages requested and all of them granted.
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 127 97 30 99 11 6 2 1 4 0 0
* /proc/buddyinfo after hugetlb allocation.
10 runs will behave as follows:
Trying to allocate 130
130
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 132
132
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
So basically 100% success for all 10 attempts.
Without the patch numbers looked much worse:
Trying to allocate 128
12
--
Trying to allocate 129
14
--
Trying to allocate 129
7
--
Trying to allocate 129
16
--
Trying to allocate 129
30
--
Trying to allocate 129
38
--
Trying to allocate 129
19
--
Trying to allocate 129
37
--
Trying to allocate 129
28
--
Trying to allocate 129
37
Just for completness the base kernel without oom detection rework looks
as follows:
Trying to allocate 127
30
--
Trying to allocate 129
12
--
Trying to allocate 129
52
--
Trying to allocate 128
32
--
Trying to allocate 129
12
--
Trying to allocate 129
10
--
Trying to allocate 129
32
--
Trying to allocate 128
14
--
Trying to allocate 128
16
--
Trying to allocate 129
8
As we can see the success rate is much more volatile and smaller without
this patch. So the patch not only makes the retry logic for costly
requests more sensible the success rate is even higher.
Signed-off-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 17:57:09 -06:00
|
|
|
else
|
2016-05-20 17:57:00 -06:00
|
|
|
no_progress_loops++;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2016-05-20 17:57:00 -06:00
|
|
|
if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
|
mm: consider compaction feedback also for costly allocation
PAGE_ALLOC_COSTLY_ORDER retry logic is mostly handled inside
should_reclaim_retry currently where we decide to not retry after at
least order worth of pages were reclaimed or the watermark check for at
least one zone would succeed after reclaiming all pages if the reclaim
hasn't made any progress. Compaction feedback is mostly ignored and we
just try to make sure that the compaction did at least something before
giving up.
The first condition was added by a41f24ea9fd6 ("page allocator: smarter
retry of costly-order allocations) and it assumed that lumpy reclaim
could have created a page of the sufficient order. Lumpy reclaim, has
been removed quite some time ago so the assumption doesn't hold anymore.
Remove the check for the number of reclaimed pages and rely on the
compaction feedback solely. should_reclaim_retry now only makes sure
that we keep retrying reclaim for high order pages only if they are
hidden by watermaks so order-0 reclaim makes really sense.
should_compact_retry now keeps retrying even for the costly allocations.
The number of retries is reduced wrt. !costly requests because they are
less important and harder to grant and so their pressure shouldn't cause
contention for other requests or cause an over reclaim. We also do not
reset no_progress_loops for costly request to make sure we do not keep
reclaiming too agressively.
This has been tested by running a process which fragments memory:
- compact memory
- mmap large portion of the memory (1920M on 2GRAM machine with 2G
of swapspace)
- MADV_DONTNEED single page in PAGE_SIZE*((1UL<<MAX_ORDER)-1)
steps until certain amount of memory is freed (250M in my test)
and reduce the step to (step / 2) + 1 after reaching the end of
the mapping
- then run a script which populates the page cache 2G (MemTotal)
from /dev/zero to a new file
And then tries to allocate
nr_hugepages=$(awk '/MemAvailable/{printf "%d\n", $2/(2*1024)}' /proc/meminfo)
huge pages.
root@test1:~# echo 1 > /proc/sys/vm/overcommit_memory;echo 1 > /proc/sys/vm/compact_memory; ./fragment-mem-and-run /root/alloc_hugepages.sh 1920M 250M
Node 0, zone DMA 31 28 31 10 2 0 2 1 2 3 1
Node 0, zone DMA32 437 319 171 50 28 25 20 16 16 14 437
* This is the /proc/buddyinfo after the compaction
Done fragmenting. size=2013265920 freed=262144000
Node 0, zone DMA 165 48 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 35109 14575 185 51 41 12 6 0 0 0 0
* /proc/buddyinfo after memory got fragmented
Executing "/root/alloc_hugepages.sh"
Eating some pagecache
508623+0 records in
508623+0 records out
2083319808 bytes (2.1 GB) copied, 11.7292 s, 178 MB/s
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 111 344 153 20 24 10 3 0 0 0 0
* /proc/buddyinfo after page cache got eaten
Trying to allocate 129
129
* 129 hugepages requested and all of them granted.
Node 0, zone DMA 3 5 3 1 2 0 2 2 2 2 0
Node 0, zone DMA32 127 97 30 99 11 6 2 1 4 0 0
* /proc/buddyinfo after hugetlb allocation.
10 runs will behave as follows:
Trying to allocate 130
130
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
--
Trying to allocate 132
132
--
Trying to allocate 129
129
--
Trying to allocate 128
128
--
Trying to allocate 129
129
So basically 100% success for all 10 attempts.
Without the patch numbers looked much worse:
Trying to allocate 128
12
--
Trying to allocate 129
14
--
Trying to allocate 129
7
--
Trying to allocate 129
16
--
Trying to allocate 129
30
--
Trying to allocate 129
38
--
Trying to allocate 129
19
--
Trying to allocate 129
37
--
Trying to allocate 129
28
--
Trying to allocate 129
37
Just for completness the base kernel without oom detection rework looks
as follows:
Trying to allocate 127
30
--
Trying to allocate 129
12
--
Trying to allocate 129
52
--
Trying to allocate 128
32
--
Trying to allocate 129
12
--
Trying to allocate 129
10
--
Trying to allocate 129
32
--
Trying to allocate 128
14
--
Trying to allocate 128
16
--
Trying to allocate 129
8
As we can see the success rate is much more volatile and smaller without
this patch. So the patch not only makes the retry logic for costly
requests more sensible the success rate is even higher.
Signed-off-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-20 17:57:09 -06:00
|
|
|
did_some_progress > 0, no_progress_loops))
|
2016-05-20 17:57:00 -06:00
|
|
|
goto retry;
|
|
|
|
|
2016-05-20 17:57:06 -06:00
|
|
|
/*
|
|
|
|
* It doesn't make any sense to retry for the compaction if the order-0
|
|
|
|
* reclaim is not able to make any progress because the current
|
|
|
|
* implementation of the compaction depends on the sufficient amount
|
|
|
|
* of free memory (see __compaction_suitable)
|
|
|
|
*/
|
|
|
|
if (did_some_progress > 0 &&
|
2016-05-20 17:57:12 -06:00
|
|
|
should_compact_retry(ac, order, alloc_flags,
|
2016-07-28 16:49:28 -06:00
|
|
|
compact_result, &compact_priority,
|
2016-05-20 17:57:12 -06:00
|
|
|
compaction_retries))
|
2016-05-20 17:57:06 -06:00
|
|
|
goto retry;
|
|
|
|
|
2015-06-24 17:57:21 -06:00
|
|
|
/* Reclaim has failed us, start killing things */
|
|
|
|
page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
|
|
|
|
if (page)
|
|
|
|
goto got_pg;
|
|
|
|
|
|
|
|
/* Retry as long as the OOM killer is making progress */
|
2016-05-20 17:57:00 -06:00
|
|
|
if (did_some_progress) {
|
|
|
|
no_progress_loops = 0;
|
2015-06-24 17:57:21 -06:00
|
|
|
goto retry;
|
2016-05-20 17:57:00 -06:00
|
|
|
}
|
2015-06-24 17:57:21 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
nopage:
|
2011-05-24 18:12:16 -06:00
|
|
|
warn_alloc_failed(gfp_mask, order, NULL);
|
2005-04-16 16:20:36 -06:00
|
|
|
got_pg:
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
return page;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-06-16 16:31:57 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This is the 'heart' of the zoned buddy allocator.
|
|
|
|
*/
|
|
|
|
struct page *
|
|
|
|
__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
|
|
|
|
struct zonelist *zonelist, nodemask_t *nodemask)
|
|
|
|
{
|
2016-05-19 18:13:50 -06:00
|
|
|
struct page *page;
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
unsigned int cpuset_mems_cookie;
|
2016-07-28 16:46:50 -06:00
|
|
|
unsigned int alloc_flags = ALLOC_WMARK_LOW;
|
2016-05-19 18:13:56 -06:00
|
|
|
gfp_t alloc_mask = gfp_mask; /* The gfp_t that was actually used for allocation */
|
2015-02-11 16:25:41 -07:00
|
|
|
struct alloc_context ac = {
|
|
|
|
.high_zoneidx = gfp_zone(gfp_mask),
|
2016-05-19 18:13:30 -06:00
|
|
|
.zonelist = zonelist,
|
2015-02-11 16:25:41 -07:00
|
|
|
.nodemask = nodemask,
|
|
|
|
.migratetype = gfpflags_to_migratetype(gfp_mask),
|
|
|
|
};
|
2009-06-16 16:31:57 -06:00
|
|
|
|
2016-05-19 18:13:30 -06:00
|
|
|
if (cpusets_enabled()) {
|
2016-05-19 18:13:56 -06:00
|
|
|
alloc_mask |= __GFP_HARDWALL;
|
2016-05-19 18:13:30 -06:00
|
|
|
alloc_flags |= ALLOC_CPUSET;
|
|
|
|
if (!ac.nodemask)
|
|
|
|
ac.nodemask = &cpuset_current_mems_allowed;
|
|
|
|
}
|
|
|
|
|
2009-06-17 21:24:12 -06:00
|
|
|
gfp_mask &= gfp_allowed_mask;
|
|
|
|
|
2009-06-16 16:31:57 -06:00
|
|
|
lockdep_trace_alloc(gfp_mask);
|
|
|
|
|
2015-11-06 17:28:21 -07:00
|
|
|
might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
|
2009-06-16 16:31:57 -06:00
|
|
|
|
|
|
|
if (should_fail_alloc_page(gfp_mask, order))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check the zones suitable for the gfp_mask contain at least one
|
|
|
|
* valid zone. It's possible to have an empty zonelist as a result
|
mm: remove GFP_THISNODE
NOTE: this is not about __GFP_THISNODE, this is only about GFP_THISNODE.
GFP_THISNODE is a secret combination of gfp bits that have different
behavior than expected. It is a combination of __GFP_THISNODE,
__GFP_NORETRY, and __GFP_NOWARN and is special-cased in the page
allocator slowpath to fail without trying reclaim even though it may be
used in combination with __GFP_WAIT.
An example of the problem this creates: commit e97ca8e5b864 ("mm: fix
GFP_THISNODE callers and clarify") fixed up many users of GFP_THISNODE
that really just wanted __GFP_THISNODE. The problem doesn't end there,
however, because even it was a no-op for alloc_misplaced_dst_page(),
which also sets __GFP_NORETRY and __GFP_NOWARN, and
migrate_misplaced_transhuge_page(), where __GFP_NORETRY and __GFP_NOWAIT
is set in GFP_TRANSHUGE. Converting GFP_THISNODE to __GFP_THISNODE is a
no-op in these cases since the page allocator special-cases
__GFP_THISNODE && __GFP_NORETRY && __GFP_NOWARN.
It's time to just remove GFP_THISNODE entirely. We leave __GFP_THISNODE
to restrict an allocation to a local node, but remove GFP_THISNODE and
its obscurity. Instead, we require that a caller clear __GFP_WAIT if it
wants to avoid reclaim.
This allows the aforementioned functions to actually reclaim as they
should. It also enables any future callers that want to do
__GFP_THISNODE but also __GFP_NORETRY && __GFP_NOWARN to reclaim. The
rule is simple: if you don't want to reclaim, then don't set __GFP_WAIT.
Aside: ovs_flow_stats_update() really wants to avoid reclaim as well, so
it is unchanged.
Signed-off-by: David Rientjes <rientjes@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Acked-by: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Pravin Shelar <pshelar@nicira.com>
Cc: Jarno Rajahalme <jrajahalme@nicira.com>
Cc: Li Zefan <lizefan@huawei.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-14 16:46:55 -06:00
|
|
|
* of __GFP_THISNODE and a memoryless node
|
2009-06-16 16:31:57 -06:00
|
|
|
*/
|
|
|
|
if (unlikely(!zonelist->_zonerefs->zone))
|
|
|
|
return NULL;
|
|
|
|
|
2015-02-11 16:25:41 -07:00
|
|
|
if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE)
|
2014-10-09 16:26:51 -06:00
|
|
|
alloc_flags |= ALLOC_CMA;
|
|
|
|
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
retry_cpuset:
|
2014-04-03 15:47:24 -06:00
|
|
|
cpuset_mems_cookie = read_mems_allowed_begin();
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
|
2015-11-06 17:28:12 -07:00
|
|
|
/* Dirty zone balancing only done in the fast path */
|
|
|
|
ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE);
|
|
|
|
|
2016-06-03 15:56:01 -06:00
|
|
|
/*
|
|
|
|
* The preferred zone is used for statistics but crucially it is
|
|
|
|
* also used as the starting point for the zonelist iterator. It
|
|
|
|
* may get reset for allocations that ignore memory policies.
|
|
|
|
*/
|
2016-05-19 18:14:10 -06:00
|
|
|
ac.preferred_zoneref = first_zones_zonelist(ac.zonelist,
|
|
|
|
ac.high_zoneidx, ac.nodemask);
|
|
|
|
if (!ac.preferred_zoneref) {
|
2016-05-19 18:13:50 -06:00
|
|
|
page = NULL;
|
2016-05-19 18:14:01 -06:00
|
|
|
goto no_zone;
|
2016-05-19 18:13:50 -06:00
|
|
|
}
|
|
|
|
|
2009-06-16 16:31:59 -06:00
|
|
|
/* First allocation attempt */
|
2015-02-11 16:25:41 -07:00
|
|
|
page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
|
2016-05-19 18:14:01 -06:00
|
|
|
if (likely(page))
|
|
|
|
goto out;
|
2009-06-16 16:31:57 -06:00
|
|
|
|
2016-05-19 18:14:01 -06:00
|
|
|
/*
|
|
|
|
* Runtime PM, block IO and its error handling path can deadlock
|
|
|
|
* because I/O on the device might not complete.
|
|
|
|
*/
|
|
|
|
alloc_mask = memalloc_noio_flags(gfp_mask);
|
|
|
|
ac.spread_dirty_pages = false;
|
2015-02-11 16:25:07 -07:00
|
|
|
|
2016-05-19 18:14:44 -06:00
|
|
|
/*
|
|
|
|
* Restore the original nodemask if it was potentially replaced with
|
|
|
|
* &cpuset_current_mems_allowed to optimize the fast-path attempt.
|
|
|
|
*/
|
|
|
|
if (cpusets_enabled())
|
|
|
|
ac.nodemask = nodemask;
|
2016-05-19 18:14:01 -06:00
|
|
|
page = __alloc_pages_slowpath(alloc_mask, order, &ac);
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
|
2016-05-19 18:14:01 -06:00
|
|
|
no_zone:
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
/*
|
|
|
|
* When updating a task's mems_allowed, it is possible to race with
|
|
|
|
* parallel threads in such a way that an allocation can fail while
|
|
|
|
* the mask is being updated. If a page allocation is about to fail,
|
|
|
|
* check if the cpuset changed during allocation and if so, retry.
|
|
|
|
*/
|
2016-05-19 18:13:56 -06:00
|
|
|
if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) {
|
|
|
|
alloc_mask = gfp_mask;
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
goto retry_cpuset;
|
2016-05-19 18:13:56 -06:00
|
|
|
}
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
|
2016-05-19 18:14:01 -06:00
|
|
|
out:
|
mm: memcontrol: only mark charged pages with PageKmemcg
To distinguish non-slab pages charged to kmemcg we mark them PageKmemcg,
which sets page->_mapcount to -512. Currently, we set/clear PageKmemcg
in __alloc_pages_nodemask()/free_pages_prepare() for any page allocated
with __GFP_ACCOUNT, including those that aren't actually charged to any
cgroup, i.e. allocated from the root cgroup context. To avoid overhead
in case cgroups are not used, we only do that if memcg_kmem_enabled() is
true. The latter is set iff there are kmem-enabled memory cgroups
(online or offline). The root cgroup is not considered kmem-enabled.
As a result, if a page is allocated with __GFP_ACCOUNT for the root
cgroup when there are kmem-enabled memory cgroups and is freed after all
kmem-enabled memory cgroups were removed, e.g.
# no memory cgroups has been created yet, create one
mkdir /sys/fs/cgroup/memory/test
# run something allocating pages with __GFP_ACCOUNT, e.g.
# a program using pipe
dmesg | tail
# remove the memory cgroup
rmdir /sys/fs/cgroup/memory/test
we'll get bad page state bug complaining about page->_mapcount != -1:
BUG: Bad page state in process swapper/0 pfn:1fd945c
page:ffffea007f651700 count:0 mapcount:-511 mapping: (null) index:0x0
flags: 0x1000000000000000()
To avoid that, let's mark with PageKmemcg only those pages that are
actually charged to and hence pin a non-root memory cgroup.
Fixes: 4949148ad433 ("mm: charge/uncharge kmemcg from generic page allocator paths")
Reported-and-tested-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-08-08 14:03:12 -06:00
|
|
|
if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
|
|
|
|
unlikely(memcg_kmem_charge(page, gfp_mask, order) != 0)) {
|
|
|
|
__free_pages(page, order);
|
|
|
|
page = NULL;
|
mm: charge/uncharge kmemcg from generic page allocator paths
Currently, to charge a non-slab allocation to kmemcg one has to use
alloc_kmem_pages helper with __GFP_ACCOUNT flag. A page allocated with
this helper should finally be freed using free_kmem_pages, otherwise it
won't be uncharged.
This API suits its current users fine, but it turns out to be impossible
to use along with page reference counting, i.e. when an allocation is
supposed to be freed with put_page, as it is the case with pipe or unix
socket buffers.
To overcome this limitation, this patch moves charging/uncharging to
generic page allocator paths, i.e. to __alloc_pages_nodemask and
free_pages_prepare, and zaps alloc/free_kmem_pages helpers. This way,
one can use any of the available page allocation functions to get the
allocated page charged to kmemcg - it's enough to pass __GFP_ACCOUNT,
just like in case of kmalloc and friends. A charged page will be
automatically uncharged on free.
To make it possible, we need to mark pages charged to kmemcg somehow.
To avoid introducing a new page flag, we make use of page->_mapcount for
marking such pages. Since pages charged to kmemcg are not supposed to
be mapped to userspace, it should work just fine. There are other
(ab)users of page->_mapcount - buddy and balloon pages - but we don't
conflict with them.
In case kmemcg is compiled out or not used at runtime, this patch
introduces no overhead to generic page allocator paths. If kmemcg is
used, it will be plus one gfp flags check on alloc and plus one
page->_mapcount check on free, which shouldn't hurt performance, because
the data accessed are hot.
Link: http://lkml.kernel.org/r/a9736d856f895bcb465d9f257b54efe32eda6f99.1464079538.git.vdavydov@virtuozzo.com
Signed-off-by: Vladimir Davydov <vdavydov@virtuozzo.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-26 16:24:24 -06:00
|
|
|
}
|
|
|
|
|
2016-05-19 18:14:01 -06:00
|
|
|
if (kmemcheck_enabled && page)
|
|
|
|
kmemcheck_pagealloc_alloc(page, order, gfp_mask);
|
|
|
|
|
|
|
|
trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
|
|
|
|
|
2009-06-16 16:31:57 -06:00
|
|
|
return page;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-06-16 16:31:52 -06:00
|
|
|
EXPORT_SYMBOL(__alloc_pages_nodemask);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Common helper functions.
|
|
|
|
*/
|
2008-02-04 23:29:26 -07:00
|
|
|
unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2009-09-21 18:01:47 -06:00
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* __get_free_pages() returns a 32-bit address, which cannot represent
|
|
|
|
* a highmem page
|
|
|
|
*/
|
|
|
|
VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
page = alloc_pages(gfp_mask, order);
|
|
|
|
if (!page)
|
|
|
|
return 0;
|
|
|
|
return (unsigned long) page_address(page);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__get_free_pages);
|
|
|
|
|
2008-02-04 23:29:26 -07:00
|
|
|
unsigned long get_zeroed_page(gfp_t gfp_mask)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2009-09-21 18:01:47 -06:00
|
|
|
return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(get_zeroed_page);
|
|
|
|
|
2008-02-04 23:29:26 -07:00
|
|
|
void __free_pages(struct page *page, unsigned int order)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2005-10-29 19:16:12 -06:00
|
|
|
if (put_page_testzero(page)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
if (order == 0)
|
2014-06-04 17:10:22 -06:00
|
|
|
free_hot_cold_page(page, false);
|
2005-04-16 16:20:36 -06:00
|
|
|
else
|
|
|
|
__free_pages_ok(page, order);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(__free_pages);
|
|
|
|
|
2008-02-04 23:29:26 -07:00
|
|
|
void free_pages(unsigned long addr, unsigned int order)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
if (addr != 0) {
|
2006-09-26 00:30:55 -06:00
|
|
|
VM_BUG_ON(!virt_addr_valid((void *)addr));
|
2005-04-16 16:20:36 -06:00
|
|
|
__free_pages(virt_to_page((void *)addr), order);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(free_pages);
|
|
|
|
|
2015-05-06 22:11:57 -06:00
|
|
|
/*
|
|
|
|
* Page Fragment:
|
|
|
|
* An arbitrary-length arbitrary-offset area of memory which resides
|
|
|
|
* within a 0 or higher order page. Multiple fragments within that page
|
|
|
|
* are individually refcounted, in the page's reference counter.
|
|
|
|
*
|
|
|
|
* The page_frag functions below provide a simple allocation framework for
|
|
|
|
* page fragments. This is used by the network stack and network device
|
|
|
|
* drivers to provide a backing region of memory for use as either an
|
|
|
|
* sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
|
|
|
|
*/
|
|
|
|
static struct page *__page_frag_refill(struct page_frag_cache *nc,
|
|
|
|
gfp_t gfp_mask)
|
|
|
|
{
|
|
|
|
struct page *page = NULL;
|
|
|
|
gfp_t gfp = gfp_mask;
|
|
|
|
|
|
|
|
#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
|
|
|
|
gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
|
|
|
|
__GFP_NOMEMALLOC;
|
|
|
|
page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
|
|
|
|
PAGE_FRAG_CACHE_MAX_ORDER);
|
|
|
|
nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
|
|
|
|
#endif
|
|
|
|
if (unlikely(!page))
|
|
|
|
page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
|
|
|
|
|
|
|
|
nc->va = page ? page_address(page) : NULL;
|
|
|
|
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
|
|
|
void *__alloc_page_frag(struct page_frag_cache *nc,
|
|
|
|
unsigned int fragsz, gfp_t gfp_mask)
|
|
|
|
{
|
|
|
|
unsigned int size = PAGE_SIZE;
|
|
|
|
struct page *page;
|
|
|
|
int offset;
|
|
|
|
|
|
|
|
if (unlikely(!nc->va)) {
|
|
|
|
refill:
|
|
|
|
page = __page_frag_refill(nc, gfp_mask);
|
|
|
|
if (!page)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
|
|
|
|
/* if size can vary use size else just use PAGE_SIZE */
|
|
|
|
size = nc->size;
|
|
|
|
#endif
|
|
|
|
/* Even if we own the page, we do not use atomic_set().
|
|
|
|
* This would break get_page_unless_zero() users.
|
|
|
|
*/
|
2016-03-17 15:19:26 -06:00
|
|
|
page_ref_add(page, size - 1);
|
2015-05-06 22:11:57 -06:00
|
|
|
|
|
|
|
/* reset page count bias and offset to start of new frag */
|
2015-08-21 15:11:51 -06:00
|
|
|
nc->pfmemalloc = page_is_pfmemalloc(page);
|
2015-05-06 22:11:57 -06:00
|
|
|
nc->pagecnt_bias = size;
|
|
|
|
nc->offset = size;
|
|
|
|
}
|
|
|
|
|
|
|
|
offset = nc->offset - fragsz;
|
|
|
|
if (unlikely(offset < 0)) {
|
|
|
|
page = virt_to_page(nc->va);
|
|
|
|
|
2016-03-17 15:19:26 -06:00
|
|
|
if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
|
2015-05-06 22:11:57 -06:00
|
|
|
goto refill;
|
|
|
|
|
|
|
|
#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
|
|
|
|
/* if size can vary use size else just use PAGE_SIZE */
|
|
|
|
size = nc->size;
|
|
|
|
#endif
|
|
|
|
/* OK, page count is 0, we can safely set it */
|
2016-03-17 15:19:26 -06:00
|
|
|
set_page_count(page, size);
|
2015-05-06 22:11:57 -06:00
|
|
|
|
|
|
|
/* reset page count bias and offset to start of new frag */
|
|
|
|
nc->pagecnt_bias = size;
|
|
|
|
offset = size - fragsz;
|
|
|
|
}
|
|
|
|
|
|
|
|
nc->pagecnt_bias--;
|
|
|
|
nc->offset = offset;
|
|
|
|
|
|
|
|
return nc->va + offset;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__alloc_page_frag);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Frees a page fragment allocated out of either a compound or order 0 page.
|
|
|
|
*/
|
|
|
|
void __free_page_frag(void *addr)
|
|
|
|
{
|
|
|
|
struct page *page = virt_to_head_page(addr);
|
|
|
|
|
|
|
|
if (unlikely(put_page_testzero(page)))
|
|
|
|
__free_pages_ok(page, compound_order(page));
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__free_page_frag);
|
|
|
|
|
2015-11-06 17:29:57 -07:00
|
|
|
static void *make_alloc_exact(unsigned long addr, unsigned int order,
|
|
|
|
size_t size)
|
2011-05-11 16:13:34 -06:00
|
|
|
{
|
|
|
|
if (addr) {
|
|
|
|
unsigned long alloc_end = addr + (PAGE_SIZE << order);
|
|
|
|
unsigned long used = addr + PAGE_ALIGN(size);
|
|
|
|
|
|
|
|
split_page(virt_to_page((void *)addr), order);
|
|
|
|
while (used < alloc_end) {
|
|
|
|
free_page(used);
|
|
|
|
used += PAGE_SIZE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return (void *)addr;
|
|
|
|
}
|
|
|
|
|
2008-07-23 22:28:11 -06:00
|
|
|
/**
|
|
|
|
* alloc_pages_exact - allocate an exact number physically-contiguous pages.
|
|
|
|
* @size: the number of bytes to allocate
|
|
|
|
* @gfp_mask: GFP flags for the allocation
|
|
|
|
*
|
|
|
|
* This function is similar to alloc_pages(), except that it allocates the
|
|
|
|
* minimum number of pages to satisfy the request. alloc_pages() can only
|
|
|
|
* allocate memory in power-of-two pages.
|
|
|
|
*
|
|
|
|
* This function is also limited by MAX_ORDER.
|
|
|
|
*
|
|
|
|
* Memory allocated by this function must be released by free_pages_exact().
|
|
|
|
*/
|
|
|
|
void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
|
|
|
|
{
|
|
|
|
unsigned int order = get_order(size);
|
|
|
|
unsigned long addr;
|
|
|
|
|
|
|
|
addr = __get_free_pages(gfp_mask, order);
|
2011-05-11 16:13:34 -06:00
|
|
|
return make_alloc_exact(addr, order, size);
|
2008-07-23 22:28:11 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(alloc_pages_exact);
|
|
|
|
|
2011-05-11 16:13:34 -06:00
|
|
|
/**
|
|
|
|
* alloc_pages_exact_nid - allocate an exact number of physically-contiguous
|
|
|
|
* pages on a node.
|
2011-05-16 14:16:54 -06:00
|
|
|
* @nid: the preferred node ID where memory should be allocated
|
2011-05-11 16:13:34 -06:00
|
|
|
* @size: the number of bytes to allocate
|
|
|
|
* @gfp_mask: GFP flags for the allocation
|
|
|
|
*
|
|
|
|
* Like alloc_pages_exact(), but try to allocate on node nid first before falling
|
|
|
|
* back.
|
|
|
|
*/
|
2014-08-06 17:04:59 -06:00
|
|
|
void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
|
2011-05-11 16:13:34 -06:00
|
|
|
{
|
2015-11-06 17:29:57 -07:00
|
|
|
unsigned int order = get_order(size);
|
2011-05-11 16:13:34 -06:00
|
|
|
struct page *p = alloc_pages_node(nid, gfp_mask, order);
|
|
|
|
if (!p)
|
|
|
|
return NULL;
|
|
|
|
return make_alloc_exact((unsigned long)page_address(p), order, size);
|
|
|
|
}
|
|
|
|
|
2008-07-23 22:28:11 -06:00
|
|
|
/**
|
|
|
|
* free_pages_exact - release memory allocated via alloc_pages_exact()
|
|
|
|
* @virt: the value returned by alloc_pages_exact.
|
|
|
|
* @size: size of allocation, same value as passed to alloc_pages_exact().
|
|
|
|
*
|
|
|
|
* Release the memory allocated by a previous call to alloc_pages_exact.
|
|
|
|
*/
|
|
|
|
void free_pages_exact(void *virt, size_t size)
|
|
|
|
{
|
|
|
|
unsigned long addr = (unsigned long)virt;
|
|
|
|
unsigned long end = addr + PAGE_ALIGN(size);
|
|
|
|
|
|
|
|
while (addr < end) {
|
|
|
|
free_page(addr);
|
|
|
|
addr += PAGE_SIZE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(free_pages_exact);
|
|
|
|
|
2013-02-22 17:35:54 -07:00
|
|
|
/**
|
|
|
|
* nr_free_zone_pages - count number of pages beyond high watermark
|
|
|
|
* @offset: The zone index of the highest zone
|
|
|
|
*
|
|
|
|
* nr_free_zone_pages() counts the number of counts pages which are beyond the
|
|
|
|
* high watermark within all zones at or below a given zone index. For each
|
|
|
|
* zone, the number of pages is calculated as:
|
2013-07-03 16:03:04 -06:00
|
|
|
* managed_pages - high_pages
|
2013-02-22 17:35:54 -07:00
|
|
|
*/
|
2013-02-22 17:35:43 -07:00
|
|
|
static unsigned long nr_free_zone_pages(int offset)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-04-28 03:12:17 -06:00
|
|
|
struct zoneref *z;
|
2008-04-28 03:12:16 -06:00
|
|
|
struct zone *zone;
|
|
|
|
|
2005-07-29 23:59:18 -06:00
|
|
|
/* Just pick one node, since fallback list is circular */
|
2013-02-22 17:35:43 -07:00
|
|
|
unsigned long sum = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-04-28 03:12:14 -06:00
|
|
|
struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-04-28 03:12:16 -06:00
|
|
|
for_each_zone_zonelist(zone, z, zonelist, offset) {
|
2013-02-22 17:33:52 -07:00
|
|
|
unsigned long size = zone->managed_pages;
|
2009-06-16 16:32:12 -06:00
|
|
|
unsigned long high = high_wmark_pages(zone);
|
2005-07-29 23:59:18 -06:00
|
|
|
if (size > high)
|
|
|
|
sum += size - high;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
return sum;
|
|
|
|
}
|
|
|
|
|
2013-02-22 17:35:54 -07:00
|
|
|
/**
|
|
|
|
* nr_free_buffer_pages - count number of pages beyond high watermark
|
|
|
|
*
|
|
|
|
* nr_free_buffer_pages() counts the number of pages which are beyond the high
|
|
|
|
* watermark within ZONE_DMA and ZONE_NORMAL.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2013-02-22 17:35:43 -07:00
|
|
|
unsigned long nr_free_buffer_pages(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2005-10-21 00:55:38 -06:00
|
|
|
return nr_free_zone_pages(gfp_zone(GFP_USER));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2007-07-17 05:04:39 -06:00
|
|
|
EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-02-22 17:35:54 -07:00
|
|
|
/**
|
|
|
|
* nr_free_pagecache_pages - count number of pages beyond high watermark
|
|
|
|
*
|
|
|
|
* nr_free_pagecache_pages() counts the number of pages which are beyond the
|
|
|
|
* high watermark within all zones.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2013-02-22 17:35:43 -07:00
|
|
|
unsigned long nr_free_pagecache_pages(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-07-17 05:03:12 -06:00
|
|
|
return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2006-09-27 02:50:06 -06:00
|
|
|
|
|
|
|
static inline void show_node(struct zone *zone)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2012-12-11 17:00:29 -07:00
|
|
|
if (IS_ENABLED(CONFIG_NUMA))
|
2006-12-06 21:33:03 -07:00
|
|
|
printk("Node %d ", zone_to_nid(zone));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2016-03-17 15:19:05 -06:00
|
|
|
long si_mem_available(void)
|
|
|
|
{
|
|
|
|
long available;
|
|
|
|
unsigned long pagecache;
|
|
|
|
unsigned long wmark_low = 0;
|
|
|
|
unsigned long pages[NR_LRU_LISTS];
|
|
|
|
struct zone *zone;
|
|
|
|
int lru;
|
|
|
|
|
|
|
|
for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
|
2016-08-11 16:32:57 -06:00
|
|
|
pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
|
2016-03-17 15:19:05 -06:00
|
|
|
|
|
|
|
for_each_zone(zone)
|
|
|
|
wmark_low += zone->watermark[WMARK_LOW];
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Estimate the amount of memory available for userspace allocations,
|
|
|
|
* without causing swapping.
|
|
|
|
*/
|
|
|
|
available = global_page_state(NR_FREE_PAGES) - totalreserve_pages;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Not all the page cache can be freed, otherwise the system will
|
|
|
|
* start swapping. Assume at least half of the page cache, or the
|
|
|
|
* low watermark worth of cache, needs to stay.
|
|
|
|
*/
|
|
|
|
pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
|
|
|
|
pagecache -= min(pagecache / 2, wmark_low);
|
|
|
|
available += pagecache;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Part of the reclaimable slab consists of items that are in use,
|
|
|
|
* and cannot be freed. Cap this estimate at the low watermark.
|
|
|
|
*/
|
|
|
|
available += global_page_state(NR_SLAB_RECLAIMABLE) -
|
|
|
|
min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low);
|
|
|
|
|
|
|
|
if (available < 0)
|
|
|
|
available = 0;
|
|
|
|
return available;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(si_mem_available);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
void si_meminfo(struct sysinfo *val)
|
|
|
|
{
|
|
|
|
val->totalram = totalram_pages;
|
2016-07-28 16:46:20 -06:00
|
|
|
val->sharedram = global_node_page_state(NR_SHMEM);
|
2007-02-10 02:43:02 -07:00
|
|
|
val->freeram = global_page_state(NR_FREE_PAGES);
|
2005-04-16 16:20:36 -06:00
|
|
|
val->bufferram = nr_blockdev_pages();
|
|
|
|
val->totalhigh = totalhigh_pages;
|
|
|
|
val->freehigh = nr_free_highpages();
|
|
|
|
val->mem_unit = PAGE_SIZE;
|
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(si_meminfo);
|
|
|
|
|
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
void si_meminfo_node(struct sysinfo *val, int nid)
|
|
|
|
{
|
2013-07-03 16:03:27 -06:00
|
|
|
int zone_type; /* needs to be signed */
|
|
|
|
unsigned long managed_pages = 0;
|
2016-05-19 18:12:23 -06:00
|
|
|
unsigned long managed_highpages = 0;
|
|
|
|
unsigned long free_highpages = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
pg_data_t *pgdat = NODE_DATA(nid);
|
|
|
|
|
2013-07-03 16:03:27 -06:00
|
|
|
for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
|
|
|
|
managed_pages += pgdat->node_zones[zone_type].managed_pages;
|
|
|
|
val->totalram = managed_pages;
|
2016-07-28 16:46:20 -06:00
|
|
|
val->sharedram = node_page_state(pgdat, NR_SHMEM);
|
mm, vmstat: add infrastructure for per-node vmstats
Patchset: "Move LRU page reclaim from zones to nodes v9"
This series moves LRUs from the zones to the node. While this is a
current rebase, the test results were based on mmotm as of June 23rd.
Conceptually, this series is simple but there are a lot of details.
Some of the broad motivations for this are;
1. The residency of a page partially depends on what zone the page was
allocated from. This is partially combatted by the fair zone allocation
policy but that is a partial solution that introduces overhead in the
page allocator paths.
2. Currently, reclaim on node 0 behaves slightly different to node 1. For
example, direct reclaim scans in zonelist order and reclaims even if
the zone is over the high watermark regardless of the age of pages
in that LRU. Kswapd on the other hand starts reclaim on the highest
unbalanced zone. A difference in distribution of file/anon pages due
to when they were allocated results can result in a difference in
again. While the fair zone allocation policy mitigates some of the
problems here, the page reclaim results on a multi-zone node will
always be different to a single-zone node.
it was scheduled on as a result.
3. kswapd and the page allocator scan zones in the opposite order to
avoid interfering with each other but it's sensitive to timing. This
mitigates the page allocator using pages that were allocated very recently
in the ideal case but it's sensitive to timing. When kswapd is allocating
from lower zones then it's great but during the rebalancing of the highest
zone, the page allocator and kswapd interfere with each other. It's worse
if the highest zone is small and difficult to balance.
4. slab shrinkers are node-based which makes it harder to identify the exact
relationship between slab reclaim and LRU reclaim.
The reason we have zone-based reclaim is that we used to have
large highmem zones in common configurations and it was necessary
to quickly find ZONE_NORMAL pages for reclaim. Today, this is much
less of a concern as machines with lots of memory will (or should) use
64-bit kernels. Combinations of 32-bit hardware and 64-bit hardware are
rare. Machines that do use highmem should have relatively low highmem:lowmem
ratios than we worried about in the past.
Conceptually, moving to node LRUs should be easier to understand. The
page allocator plays fewer tricks to game reclaim and reclaim behaves
similarly on all nodes.
The series has been tested on a 16 core UMA machine and a 2-socket 48
core NUMA machine. The UMA results are presented in most cases as the NUMA
machine behaved similarly.
pagealloc
---------
This is a microbenchmark that shows the benefit of removing the fair zone
allocation policy. It was tested uip to order-4 but only orders 0 and 1 are
shown as the other orders were comparable.
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 nodelru-v9
Min total-odr0-1 490.00 ( 0.00%) 457.00 ( 6.73%)
Min total-odr0-2 347.00 ( 0.00%) 329.00 ( 5.19%)
Min total-odr0-4 288.00 ( 0.00%) 273.00 ( 5.21%)
Min total-odr0-8 251.00 ( 0.00%) 239.00 ( 4.78%)
Min total-odr0-16 234.00 ( 0.00%) 222.00 ( 5.13%)
Min total-odr0-32 223.00 ( 0.00%) 211.00 ( 5.38%)
Min total-odr0-64 217.00 ( 0.00%) 208.00 ( 4.15%)
Min total-odr0-128 214.00 ( 0.00%) 204.00 ( 4.67%)
Min total-odr0-256 250.00 ( 0.00%) 230.00 ( 8.00%)
Min total-odr0-512 271.00 ( 0.00%) 269.00 ( 0.74%)
Min total-odr0-1024 291.00 ( 0.00%) 282.00 ( 3.09%)
Min total-odr0-2048 303.00 ( 0.00%) 296.00 ( 2.31%)
Min total-odr0-4096 311.00 ( 0.00%) 309.00 ( 0.64%)
Min total-odr0-8192 316.00 ( 0.00%) 314.00 ( 0.63%)
Min total-odr0-16384 317.00 ( 0.00%) 315.00 ( 0.63%)
Min total-odr1-1 742.00 ( 0.00%) 712.00 ( 4.04%)
Min total-odr1-2 562.00 ( 0.00%) 530.00 ( 5.69%)
Min total-odr1-4 457.00 ( 0.00%) 433.00 ( 5.25%)
Min total-odr1-8 411.00 ( 0.00%) 381.00 ( 7.30%)
Min total-odr1-16 381.00 ( 0.00%) 356.00 ( 6.56%)
Min total-odr1-32 372.00 ( 0.00%) 346.00 ( 6.99%)
Min total-odr1-64 372.00 ( 0.00%) 343.00 ( 7.80%)
Min total-odr1-128 375.00 ( 0.00%) 351.00 ( 6.40%)
Min total-odr1-256 379.00 ( 0.00%) 351.00 ( 7.39%)
Min total-odr1-512 385.00 ( 0.00%) 355.00 ( 7.79%)
Min total-odr1-1024 386.00 ( 0.00%) 358.00 ( 7.25%)
Min total-odr1-2048 390.00 ( 0.00%) 362.00 ( 7.18%)
Min total-odr1-4096 390.00 ( 0.00%) 362.00 ( 7.18%)
Min total-odr1-8192 388.00 ( 0.00%) 363.00 ( 6.44%)
This shows a steady improvement throughout. The primary benefit is from
reduced system CPU usage which is obvious from the overall times;
4.7.0-rc4 4.7.0-rc4
mmotm-20160623nodelru-v8
User 189.19 191.80
System 2604.45 2533.56
Elapsed 2855.30 2786.39
The vmstats also showed that the fair zone allocation policy was definitely
removed as can be seen here;
4.7.0-rc3 4.7.0-rc3
mmotm-20160623 nodelru-v8
DMA32 allocs 28794729769 0
Normal allocs 48432501431 77227309877
Movable allocs 0 0
tiobench on ext4
----------------
tiobench is a benchmark that artifically benefits if old pages remain resident
while new pages get reclaimed. The fair zone allocation policy mitigates this
problem so pages age fairly. While the benchmark has problems, it is important
that tiobench performance remains constant as it implies that page aging
problems that the fair zone allocation policy fixes are not re-introduced.
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 nodelru-v9
Min PotentialReadSpeed 89.65 ( 0.00%) 90.21 ( 0.62%)
Min SeqRead-MB/sec-1 82.68 ( 0.00%) 82.01 ( -0.81%)
Min SeqRead-MB/sec-2 72.76 ( 0.00%) 72.07 ( -0.95%)
Min SeqRead-MB/sec-4 75.13 ( 0.00%) 74.92 ( -0.28%)
Min SeqRead-MB/sec-8 64.91 ( 0.00%) 65.19 ( 0.43%)
Min SeqRead-MB/sec-16 62.24 ( 0.00%) 62.22 ( -0.03%)
Min RandRead-MB/sec-1 0.88 ( 0.00%) 0.88 ( 0.00%)
Min RandRead-MB/sec-2 0.95 ( 0.00%) 0.92 ( -3.16%)
Min RandRead-MB/sec-4 1.43 ( 0.00%) 1.34 ( -6.29%)
Min RandRead-MB/sec-8 1.61 ( 0.00%) 1.60 ( -0.62%)
Min RandRead-MB/sec-16 1.80 ( 0.00%) 1.90 ( 5.56%)
Min SeqWrite-MB/sec-1 76.41 ( 0.00%) 76.85 ( 0.58%)
Min SeqWrite-MB/sec-2 74.11 ( 0.00%) 73.54 ( -0.77%)
Min SeqWrite-MB/sec-4 80.05 ( 0.00%) 80.13 ( 0.10%)
Min SeqWrite-MB/sec-8 72.88 ( 0.00%) 73.20 ( 0.44%)
Min SeqWrite-MB/sec-16 75.91 ( 0.00%) 76.44 ( 0.70%)
Min RandWrite-MB/sec-1 1.18 ( 0.00%) 1.14 ( -3.39%)
Min RandWrite-MB/sec-2 1.02 ( 0.00%) 1.03 ( 0.98%)
Min RandWrite-MB/sec-4 1.05 ( 0.00%) 0.98 ( -6.67%)
Min RandWrite-MB/sec-8 0.89 ( 0.00%) 0.92 ( 3.37%)
Min RandWrite-MB/sec-16 0.92 ( 0.00%) 0.93 ( 1.09%)
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 approx-v9
User 645.72 525.90
System 403.85 331.75
Elapsed 6795.36 6783.67
This shows that the series has little or not impact on tiobench which is
desirable and a reduction in system CPU usage. It indicates that the fair
zone allocation policy was removed in a manner that didn't reintroduce
one class of page aging bug. There were only minor differences in overall
reclaim activity
4.7.0-rc4 4.7.0-rc4
mmotm-20160623nodelru-v8
Minor Faults 645838 647465
Major Faults 573 640
Swap Ins 0 0
Swap Outs 0 0
DMA allocs 0 0
DMA32 allocs 46041453 44190646
Normal allocs 78053072 79887245
Movable allocs 0 0
Allocation stalls 24 67
Stall zone DMA 0 0
Stall zone DMA32 0 0
Stall zone Normal 0 2
Stall zone HighMem 0 0
Stall zone Movable 0 65
Direct pages scanned 10969 30609
Kswapd pages scanned 93375144 93492094
Kswapd pages reclaimed 93372243 93489370
Direct pages reclaimed 10969 30609
Kswapd efficiency 99% 99%
Kswapd velocity 13741.015 13781.934
Direct efficiency 100% 100%
Direct velocity 1.614 4.512
Percentage direct scans 0% 0%
kswapd activity was roughly comparable. There were differences in direct
reclaim activity but negligible in the context of the overall workload
(velocity of 4 pages per second with the patches applied, 1.6 pages per
second in the baseline kernel).
pgbench read-only large configuration on ext4
---------------------------------------------
pgbench is a database benchmark that can be sensitive to page reclaim
decisions. This also checks if removing the fair zone allocation policy
is safe
pgbench Transactions
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 nodelru-v8
Hmean 1 188.26 ( 0.00%) 189.78 ( 0.81%)
Hmean 5 330.66 ( 0.00%) 328.69 ( -0.59%)
Hmean 12 370.32 ( 0.00%) 380.72 ( 2.81%)
Hmean 21 368.89 ( 0.00%) 369.00 ( 0.03%)
Hmean 30 382.14 ( 0.00%) 360.89 ( -5.56%)
Hmean 32 428.87 ( 0.00%) 432.96 ( 0.95%)
Negligible differences again. As with tiobench, overall reclaim activity
was comparable.
bonnie++ on ext4
----------------
No interesting performance difference, negligible differences on reclaim
stats.
paralleldd on ext4
------------------
This workload uses varying numbers of dd instances to read large amounts of
data from disk.
4.7.0-rc3 4.7.0-rc3
mmotm-20160623 nodelru-v9
Amean Elapsd-1 186.04 ( 0.00%) 189.41 ( -1.82%)
Amean Elapsd-3 192.27 ( 0.00%) 191.38 ( 0.46%)
Amean Elapsd-5 185.21 ( 0.00%) 182.75 ( 1.33%)
Amean Elapsd-7 183.71 ( 0.00%) 182.11 ( 0.87%)
Amean Elapsd-12 180.96 ( 0.00%) 181.58 ( -0.35%)
Amean Elapsd-16 181.36 ( 0.00%) 183.72 ( -1.30%)
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 nodelru-v9
User 1548.01 1552.44
System 8609.71 8515.08
Elapsed 3587.10 3594.54
There is little or no change in performance but some drop in system CPU usage.
4.7.0-rc3 4.7.0-rc3
mmotm-20160623 nodelru-v9
Minor Faults 362662 367360
Major Faults 1204 1143
Swap Ins 22 0
Swap Outs 2855 1029
DMA allocs 0 0
DMA32 allocs 31409797 28837521
Normal allocs 46611853 49231282
Movable allocs 0 0
Direct pages scanned 0 0
Kswapd pages scanned 40845270 40869088
Kswapd pages reclaimed 40830976 40855294
Direct pages reclaimed 0 0
Kswapd efficiency 99% 99%
Kswapd velocity 11386.711 11369.769
Direct efficiency 100% 100%
Direct velocity 0.000 0.000
Percentage direct scans 0% 0%
Page writes by reclaim 2855 1029
Page writes file 0 0
Page writes anon 2855 1029
Page reclaim immediate 771 1628
Sector Reads 293312636 293536360
Sector Writes 18213568 18186480
Page rescued immediate 0 0
Slabs scanned 128257 132747
Direct inode steals 181 56
Kswapd inode steals 59 1131
It basically shows that kswapd was active at roughly the same rate in
both kernels. There was also comparable slab scanning activity and direct
reclaim was avoided in both cases. There appears to be a large difference
in numbers of inodes reclaimed but the workload has few active inodes and
is likely a timing artifact.
stutter
-------
stutter simulates a simple workload. One part uses a lot of anonymous
memory, a second measures mmap latency and a third copies a large file.
The primary metric is checking for mmap latency.
stutter
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 nodelru-v8
Min mmap 16.6283 ( 0.00%) 13.4258 ( 19.26%)
1st-qrtle mmap 54.7570 ( 0.00%) 34.9121 ( 36.24%)
2nd-qrtle mmap 57.3163 ( 0.00%) 46.1147 ( 19.54%)
3rd-qrtle mmap 58.9976 ( 0.00%) 47.1882 ( 20.02%)
Max-90% mmap 59.7433 ( 0.00%) 47.4453 ( 20.58%)
Max-93% mmap 60.1298 ( 0.00%) 47.6037 ( 20.83%)
Max-95% mmap 73.4112 ( 0.00%) 82.8719 (-12.89%)
Max-99% mmap 92.8542 ( 0.00%) 88.8870 ( 4.27%)
Max mmap 1440.6569 ( 0.00%) 121.4201 ( 91.57%)
Mean mmap 59.3493 ( 0.00%) 42.2991 ( 28.73%)
Best99%Mean mmap 57.2121 ( 0.00%) 41.8207 ( 26.90%)
Best95%Mean mmap 55.9113 ( 0.00%) 39.9620 ( 28.53%)
Best90%Mean mmap 55.6199 ( 0.00%) 39.3124 ( 29.32%)
Best50%Mean mmap 53.2183 ( 0.00%) 33.1307 ( 37.75%)
Best10%Mean mmap 45.9842 ( 0.00%) 20.4040 ( 55.63%)
Best5%Mean mmap 43.2256 ( 0.00%) 17.9654 ( 58.44%)
Best1%Mean mmap 32.9388 ( 0.00%) 16.6875 ( 49.34%)
This shows a number of improvements with the worst-case outlier greatly
improved.
Some of the vmstats are interesting
4.7.0-rc4 4.7.0-rc4
mmotm-20160623nodelru-v8
Swap Ins 163 502
Swap Outs 0 0
DMA allocs 0 0
DMA32 allocs 618719206 1381662383
Normal allocs 891235743 564138421
Movable allocs 0 0
Allocation stalls 2603 1
Direct pages scanned 216787 2
Kswapd pages scanned 50719775 41778378
Kswapd pages reclaimed 41541765 41777639
Direct pages reclaimed 209159 0
Kswapd efficiency 81% 99%
Kswapd velocity 16859.554 14329.059
Direct efficiency 96% 0%
Direct velocity 72.061 0.001
Percentage direct scans 0% 0%
Page writes by reclaim 6215049 0
Page writes file 6215049 0
Page writes anon 0 0
Page reclaim immediate 70673 90
Sector Reads 81940800 81680456
Sector Writes 100158984 98816036
Page rescued immediate 0 0
Slabs scanned 1366954 22683
While this is not guaranteed in all cases, this particular test showed
a large reduction in direct reclaim activity. It's also worth noting
that no page writes were issued from reclaim context.
This series is not without its hazards. There are at least three areas
that I'm concerned with even though I could not reproduce any problems in
that area.
1. Reclaim/compaction is going to be affected because the amount of reclaim is
no longer targetted at a specific zone. Compaction works on a per-zone basis
so there is no guarantee that reclaiming a few THP's worth page pages will
have a positive impact on compaction success rates.
2. The Slab/LRU reclaim ratio is affected because the frequency the shrinkers
are called is now different. This may or may not be a problem but if it
is, it'll be because shrinkers are not called enough and some balancing
is required.
3. The anon/file reclaim ratio may be affected. Pages about to be dirtied are
distributed between zones and the fair zone allocation policy used to do
something very similar for anon. The distribution is now different but not
necessarily in any way that matters but it's still worth bearing in mind.
VM statistic counters for reclaim decisions are zone-based. If the kernel
is to reclaim on a per-node basis then we need to track per-node
statistics but there is no infrastructure for that. The most notable
change is that the old node_page_state is renamed to
sum_zone_node_page_state. The new node_page_state takes a pglist_data and
uses per-node stats but none exist yet. There is some renaming such as
vm_stat to vm_zone_stat and the addition of vm_node_stat and the renaming
of mod_state to mod_zone_state. Otherwise, this is mostly a mechanical
patch with no functional change. There is a lot of similarity between the
node and zone helpers which is unfortunate but there was no obvious way of
reusing the code and maintaining type safety.
Link: http://lkml.kernel.org/r/1467970510-21195-2-git-send-email-mgorman@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Rik van Riel <riel@surriel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:45:24 -06:00
|
|
|
val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
|
2006-09-26 00:31:12 -06:00
|
|
|
#ifdef CONFIG_HIGHMEM
|
2016-05-19 18:12:23 -06:00
|
|
|
for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
|
|
|
|
struct zone *zone = &pgdat->node_zones[zone_type];
|
|
|
|
|
|
|
|
if (is_highmem(zone)) {
|
|
|
|
managed_highpages += zone->managed_pages;
|
|
|
|
free_highpages += zone_page_state(zone, NR_FREE_PAGES);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
val->totalhigh = managed_highpages;
|
|
|
|
val->freehigh = free_highpages;
|
2006-09-26 00:31:12 -06:00
|
|
|
#else
|
2016-05-19 18:12:23 -06:00
|
|
|
val->totalhigh = managed_highpages;
|
|
|
|
val->freehigh = free_highpages;
|
2006-09-26 00:31:12 -06:00
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
val->mem_unit = PAGE_SIZE;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2011-03-22 17:30:46 -06:00
|
|
|
/*
|
2011-05-24 18:11:16 -06:00
|
|
|
* Determine whether the node should be displayed or not, depending on whether
|
|
|
|
* SHOW_MEM_FILTER_NODES was passed to show_free_areas().
|
2011-03-22 17:30:46 -06:00
|
|
|
*/
|
2011-05-24 18:11:16 -06:00
|
|
|
bool skip_free_areas_node(unsigned int flags, int nid)
|
2011-03-22 17:30:46 -06:00
|
|
|
{
|
|
|
|
bool ret = false;
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
unsigned int cpuset_mems_cookie;
|
2011-03-22 17:30:46 -06:00
|
|
|
|
|
|
|
if (!(flags & SHOW_MEM_FILTER_NODES))
|
|
|
|
goto out;
|
|
|
|
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
do {
|
2014-04-03 15:47:24 -06:00
|
|
|
cpuset_mems_cookie = read_mems_allowed_begin();
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
ret = !node_isset(nid, cpuset_current_mems_allowed);
|
2014-04-03 15:47:24 -06:00
|
|
|
} while (read_mems_allowed_retry(cpuset_mems_cookie));
|
2011-03-22 17:30:46 -06:00
|
|
|
out:
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#define K(x) ((x) << (PAGE_SHIFT-10))
|
|
|
|
|
2012-12-11 17:00:24 -07:00
|
|
|
static void show_migration_types(unsigned char type)
|
|
|
|
{
|
|
|
|
static const char types[MIGRATE_TYPES] = {
|
|
|
|
[MIGRATE_UNMOVABLE] = 'U',
|
|
|
|
[MIGRATE_MOVABLE] = 'M',
|
2015-12-11 14:40:29 -07:00
|
|
|
[MIGRATE_RECLAIMABLE] = 'E',
|
|
|
|
[MIGRATE_HIGHATOMIC] = 'H',
|
2012-12-11 17:00:24 -07:00
|
|
|
#ifdef CONFIG_CMA
|
|
|
|
[MIGRATE_CMA] = 'C',
|
|
|
|
#endif
|
2013-02-22 17:33:58 -07:00
|
|
|
#ifdef CONFIG_MEMORY_ISOLATION
|
2012-12-11 17:00:24 -07:00
|
|
|
[MIGRATE_ISOLATE] = 'I',
|
2013-02-22 17:33:58 -07:00
|
|
|
#endif
|
2012-12-11 17:00:24 -07:00
|
|
|
};
|
|
|
|
char tmp[MIGRATE_TYPES + 1];
|
|
|
|
char *p = tmp;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < MIGRATE_TYPES; i++) {
|
|
|
|
if (type & (1 << i))
|
|
|
|
*p++ = types[i];
|
|
|
|
}
|
|
|
|
|
|
|
|
*p = '\0';
|
|
|
|
printk("(%s) ", tmp);
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* Show free area list (used inside shift_scroll-lock stuff)
|
|
|
|
* We also calculate the percentage fragmentation. We do this by counting the
|
|
|
|
* memory on each free list with the exception of the first item on the list.
|
2015-04-14 16:45:30 -06:00
|
|
|
*
|
|
|
|
* Bits in @filter:
|
|
|
|
* SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
|
|
|
|
* cpuset.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2011-05-24 18:11:16 -06:00
|
|
|
void show_free_areas(unsigned int filter)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2015-04-14 16:45:30 -06:00
|
|
|
unsigned long free_pcp = 0;
|
[PATCH] Condense output of show_free_areas()
On larger systems, the amount of output dumped on the console when you do
SysRq-M is beyond insane. This patch is trying to reduce it somewhat as
even with the smaller NUMA systems that have hit the desktop this seems to
be a fair thing to do.
The philosophy I have taken is as follows:
1) If a zone is empty, don't tell, we don't need yet another line
telling us so. The information is available since one can look up
the fact how many zones were initialized in the first place.
2) Put as much information on a line is possible, if it can be done
in one line, rahter than two, then do it in one. I tried to format
the temperature stuff for easy reading.
Change show_free_areas() to not print lines for empty zones. If no zone
output is printed, the zone is empty. This reduces the number of lines
dumped to the console in sysrq on a large system by several thousand lines.
Change the zone temperature printouts to use one line per CPU instead of
two lines (one hot, one cold). On a 1024 CPU, 1024 node system, this
reduces the console output by over a million lines of output.
While this is a bigger problem on large NUMA systems, it is also applicable
to smaller desktop sized and mid range NUMA systems.
Old format:
Mem-info:
Node 0 DMA per-cpu:
cpu 0 hot: high 42, batch 7 used:24
cpu 0 cold: high 14, batch 3 used:1
cpu 1 hot: high 42, batch 7 used:34
cpu 1 cold: high 14, batch 3 used:0
cpu 2 hot: high 42, batch 7 used:0
cpu 2 cold: high 14, batch 3 used:0
cpu 3 hot: high 42, batch 7 used:0
cpu 3 cold: high 14, batch 3 used:0
cpu 4 hot: high 42, batch 7 used:0
cpu 4 cold: high 14, batch 3 used:0
cpu 5 hot: high 42, batch 7 used:0
cpu 5 cold: high 14, batch 3 used:0
cpu 6 hot: high 42, batch 7 used:0
cpu 6 cold: high 14, batch 3 used:0
cpu 7 hot: high 42, batch 7 used:0
cpu 7 cold: high 14, batch 3 used:0
Node 0 DMA32 per-cpu: empty
Node 0 Normal per-cpu: empty
Node 0 HighMem per-cpu: empty
Node 1 DMA per-cpu:
[snip]
Free pages: 5410688kB (0kB HighMem)
Active:9536 inactive:4261 dirty:6 writeback:0 unstable:0 free:338168 slab:1931 mapped:1900 pagetables:208
Node 0 DMA free:1676304kB min:3264kB low:4080kB high:4896kB active:128048kB inactive:61568kB present:1970880kB pages_scanned:0 all_unreclaimable? no
lowmem_reserve[]: 0 0 0 0
Node 0 DMA32 free:0kB min:0kB low:0kB high:0kB active:0kB inactive:0kB present:0kB pages_scanned:0 all_unreclaimable? no
lowmem_reserve[]: 0 0 0 0
Node 0 Normal free:0kB min:0kB low:0kB high:0kB active:0kB inactive:0kB present:0kB pages_scanned:0 all_unreclaimable? no
lowmem_reserve[]: 0 0 0 0
Node 0 HighMem free:0kB min:512kB low:512kB high:512kB active:0kB inactive:0kB present:0kB pages_scanned:0 all_unreclaimable? no
lowmem_reserve[]: 0 0 0 0
Node 1 DMA free:1951728kB min:3280kB low:4096kB high:4912kB active:5632kB inactive:1504kB present:1982464kB pages_scanned:0 all_unreclaimable? no
lowmem_reserve[]: 0 0 0 0
....
New format:
Mem-info:
Node 0 DMA per-cpu:
CPU 0: Hot: hi: 42, btch: 7 usd: 41 Cold: hi: 14, btch: 3 usd: 2
CPU 1: Hot: hi: 42, btch: 7 usd: 40 Cold: hi: 14, btch: 3 usd: 1
CPU 2: Hot: hi: 42, btch: 7 usd: 0 Cold: hi: 14, btch: 3 usd: 0
CPU 3: Hot: hi: 42, btch: 7 usd: 0 Cold: hi: 14, btch: 3 usd: 0
CPU 4: Hot: hi: 42, btch: 7 usd: 0 Cold: hi: 14, btch: 3 usd: 0
CPU 5: Hot: hi: 42, btch: 7 usd: 0 Cold: hi: 14, btch: 3 usd: 0
CPU 6: Hot: hi: 42, btch: 7 usd: 0 Cold: hi: 14, btch: 3 usd: 0
CPU 7: Hot: hi: 42, btch: 7 usd: 0 Cold: hi: 14, btch: 3 usd: 0
Node 1 DMA per-cpu:
[snip]
Free pages: 5411088kB (0kB HighMem)
Active:9558 inactive:4233 dirty:6 writeback:0 unstable:0 free:338193 slab:1942 mapped:1918 pagetables:208
Node 0 DMA free:1677648kB min:3264kB low:4080kB high:4896kB active:129296kB inactive:58864kB present:1970880kB pages_scanned:0 all_unreclaimable? no
lowmem_reserve[]: 0 0 0 0
Node 1 DMA free:1948448kB min:3280kB low:4096kB high:4912kB active:6864kB inactive:3536kB present:1982464kB pages_scanned:0 all_unreclaimable? no
lowmem_reserve[]: 0 0 0 0
Signed-off-by: Jes Sorensen <jes@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:50:05 -06:00
|
|
|
int cpu;
|
2005-04-16 16:20:36 -06:00
|
|
|
struct zone *zone;
|
2016-07-28 16:45:31 -06:00
|
|
|
pg_data_t *pgdat;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-03-31 16:19:31 -06:00
|
|
|
for_each_populated_zone(zone) {
|
2011-05-24 18:11:16 -06:00
|
|
|
if (skip_free_areas_node(filter, zone_to_nid(zone)))
|
2011-03-22 17:30:46 -06:00
|
|
|
continue;
|
2015-04-14 16:45:30 -06:00
|
|
|
|
2015-04-14 16:45:32 -06:00
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2009-09-21 18:01:37 -06:00
|
|
|
printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
|
|
|
|
" active_file:%lu inactive_file:%lu isolated_file:%lu\n"
|
2015-04-14 16:45:30 -06:00
|
|
|
" unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
|
|
|
|
" slab_reclaimable:%lu slab_unreclaimable:%lu\n"
|
2012-10-08 17:32:02 -06:00
|
|
|
" mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
|
2015-04-14 16:45:30 -06:00
|
|
|
" free:%lu free_pcp:%lu free_cma:%lu\n",
|
2016-07-28 16:45:31 -06:00
|
|
|
global_node_page_state(NR_ACTIVE_ANON),
|
|
|
|
global_node_page_state(NR_INACTIVE_ANON),
|
|
|
|
global_node_page_state(NR_ISOLATED_ANON),
|
|
|
|
global_node_page_state(NR_ACTIVE_FILE),
|
|
|
|
global_node_page_state(NR_INACTIVE_FILE),
|
|
|
|
global_node_page_state(NR_ISOLATED_FILE),
|
|
|
|
global_node_page_state(NR_UNEVICTABLE),
|
2016-07-28 16:46:20 -06:00
|
|
|
global_node_page_state(NR_FILE_DIRTY),
|
|
|
|
global_node_page_state(NR_WRITEBACK),
|
|
|
|
global_node_page_state(NR_UNSTABLE_NFS),
|
2009-09-21 18:01:29 -06:00
|
|
|
global_page_state(NR_SLAB_RECLAIMABLE),
|
|
|
|
global_page_state(NR_SLAB_UNRECLAIMABLE),
|
2016-07-28 16:46:14 -06:00
|
|
|
global_node_page_state(NR_FILE_MAPPED),
|
2016-07-28 16:46:20 -06:00
|
|
|
global_node_page_state(NR_SHMEM),
|
2007-02-08 15:20:40 -07:00
|
|
|
global_page_state(NR_PAGETABLE),
|
2012-10-08 17:32:02 -06:00
|
|
|
global_page_state(NR_BOUNCE),
|
2015-04-14 16:45:30 -06:00
|
|
|
global_page_state(NR_FREE_PAGES),
|
|
|
|
free_pcp,
|
2012-10-08 17:32:02 -06:00
|
|
|
global_page_state(NR_FREE_CMA_PAGES));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2016-07-28 16:45:31 -06:00
|
|
|
for_each_online_pgdat(pgdat) {
|
|
|
|
printk("Node %d"
|
|
|
|
" active_anon:%lukB"
|
|
|
|
" inactive_anon:%lukB"
|
|
|
|
" active_file:%lukB"
|
|
|
|
" inactive_file:%lukB"
|
|
|
|
" unevictable:%lukB"
|
|
|
|
" isolated(anon):%lukB"
|
|
|
|
" isolated(file):%lukB"
|
2016-07-28 16:46:14 -06:00
|
|
|
" mapped:%lukB"
|
2016-07-28 16:46:20 -06:00
|
|
|
" dirty:%lukB"
|
|
|
|
" writeback:%lukB"
|
|
|
|
" shmem:%lukB"
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
" shmem_thp: %lukB"
|
|
|
|
" shmem_pmdmapped: %lukB"
|
|
|
|
" anon_thp: %lukB"
|
|
|
|
#endif
|
|
|
|
" writeback_tmp:%lukB"
|
|
|
|
" unstable:%lukB"
|
2016-07-28 16:47:14 -06:00
|
|
|
" pages_scanned:%lu"
|
2016-07-28 16:45:31 -06:00
|
|
|
" all_unreclaimable? %s"
|
|
|
|
"\n",
|
|
|
|
pgdat->node_id,
|
|
|
|
K(node_page_state(pgdat, NR_ACTIVE_ANON)),
|
|
|
|
K(node_page_state(pgdat, NR_INACTIVE_ANON)),
|
|
|
|
K(node_page_state(pgdat, NR_ACTIVE_FILE)),
|
|
|
|
K(node_page_state(pgdat, NR_INACTIVE_FILE)),
|
|
|
|
K(node_page_state(pgdat, NR_UNEVICTABLE)),
|
|
|
|
K(node_page_state(pgdat, NR_ISOLATED_ANON)),
|
|
|
|
K(node_page_state(pgdat, NR_ISOLATED_FILE)),
|
2016-07-28 16:46:14 -06:00
|
|
|
K(node_page_state(pgdat, NR_FILE_MAPPED)),
|
2016-07-28 16:46:20 -06:00
|
|
|
K(node_page_state(pgdat, NR_FILE_DIRTY)),
|
|
|
|
K(node_page_state(pgdat, NR_WRITEBACK)),
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
|
|
|
|
K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
|
|
|
|
* HPAGE_PMD_NR),
|
|
|
|
K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
|
|
|
|
#endif
|
|
|
|
K(node_page_state(pgdat, NR_SHMEM)),
|
|
|
|
K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
|
|
|
|
K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
|
2016-07-28 16:47:14 -06:00
|
|
|
node_page_state(pgdat, NR_PAGES_SCANNED),
|
2016-07-28 16:45:31 -06:00
|
|
|
!pgdat_reclaimable(pgdat) ? "yes" : "no");
|
|
|
|
}
|
|
|
|
|
2009-03-31 16:19:31 -06:00
|
|
|
for_each_populated_zone(zone) {
|
2005-04-16 16:20:36 -06:00
|
|
|
int i;
|
|
|
|
|
2011-05-24 18:11:16 -06:00
|
|
|
if (skip_free_areas_node(filter, zone_to_nid(zone)))
|
2011-03-22 17:30:46 -06:00
|
|
|
continue;
|
2015-04-14 16:45:30 -06:00
|
|
|
|
|
|
|
free_pcp = 0;
|
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
show_node(zone);
|
|
|
|
printk("%s"
|
|
|
|
" free:%lukB"
|
|
|
|
" min:%lukB"
|
|
|
|
" low:%lukB"
|
|
|
|
" high:%lukB"
|
mm: add per-zone lru list stat
When I did stress test with hackbench, I got OOM message frequently
which didn't ever happen in zone-lru.
gfp_mask=0x26004c0(GFP_KERNEL|__GFP_REPEAT|__GFP_NOTRACK), order=0
..
..
__alloc_pages_nodemask+0xe52/0xe60
? new_slab+0x39c/0x3b0
new_slab+0x39c/0x3b0
___slab_alloc.constprop.87+0x6da/0x840
? __alloc_skb+0x3c/0x260
? _raw_spin_unlock_irq+0x27/0x60
? trace_hardirqs_on_caller+0xec/0x1b0
? finish_task_switch+0xa6/0x220
? poll_select_copy_remaining+0x140/0x140
__slab_alloc.isra.81.constprop.86+0x40/0x6d
? __alloc_skb+0x3c/0x260
kmem_cache_alloc+0x22c/0x260
? __alloc_skb+0x3c/0x260
__alloc_skb+0x3c/0x260
alloc_skb_with_frags+0x4e/0x1a0
sock_alloc_send_pskb+0x16a/0x1b0
? wait_for_unix_gc+0x31/0x90
? alloc_set_pte+0x2ad/0x310
unix_stream_sendmsg+0x28d/0x340
sock_sendmsg+0x2d/0x40
sock_write_iter+0x6c/0xc0
__vfs_write+0xc0/0x120
vfs_write+0x9b/0x1a0
? __might_fault+0x49/0xa0
SyS_write+0x44/0x90
do_fast_syscall_32+0xa6/0x1e0
sysenter_past_esp+0x45/0x74
Mem-Info:
active_anon:104698 inactive_anon:105791 isolated_anon:192
active_file:433 inactive_file:283 isolated_file:22
unevictable:0 dirty:0 writeback:296 unstable:0
slab_reclaimable:6389 slab_unreclaimable:78927
mapped:474 shmem:0 pagetables:101426 bounce:0
free:10518 free_pcp:334 free_cma:0
Node 0 active_anon:418792kB inactive_anon:423164kB active_file:1732kB inactive_file:1132kB unevictable:0kB isolated(anon):768kB isolated(file):88kB mapped:1896kB dirty:0kB writeback:1184kB shmem:0kB writeback_tmp:0kB unstable:0kB pages_scanned:1478632 all_unreclaimable? yes
DMA free:3304kB min:68kB low:84kB high:100kB present:15992kB managed:15916kB mlocked:0kB slab_reclaimable:0kB slab_unreclaimable:4088kB kernel_stack:0kB pagetables:2480kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB
lowmem_reserve[]: 0 809 1965 1965
Normal free:3436kB min:3604kB low:4504kB high:5404kB present:897016kB managed:858460kB mlocked:0kB slab_reclaimable:25556kB slab_unreclaimable:311712kB kernel_stack:164608kB pagetables:30844kB bounce:0kB free_pcp:620kB local_pcp:104kB free_cma:0kB
lowmem_reserve[]: 0 0 9247 9247
HighMem free:33808kB min:512kB low:1796kB high:3080kB present:1183736kB managed:1183736kB mlocked:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:372252kB bounce:0kB free_pcp:428kB local_pcp:72kB free_cma:0kB
lowmem_reserve[]: 0 0 0 0
DMA: 2*4kB (UM) 2*8kB (UM) 0*16kB 1*32kB (U) 1*64kB (U) 2*128kB (UM) 1*256kB (U) 1*512kB (M) 0*1024kB 1*2048kB (U) 0*4096kB = 3192kB
Normal: 33*4kB (MH) 79*8kB (ME) 11*16kB (M) 4*32kB (M) 2*64kB (ME) 2*128kB (EH) 7*256kB (EH) 0*512kB 0*1024kB 0*2048kB 0*4096kB = 3244kB
HighMem: 2590*4kB (UM) 1568*8kB (UM) 491*16kB (UM) 60*32kB (UM) 6*64kB (M) 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 33064kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
25121 total pagecache pages
24160 pages in swap cache
Swap cache stats: add 86371, delete 62211, find 42865/60187
Free swap = 4015560kB
Total swap = 4192252kB
524186 pages RAM
295934 pages HighMem/MovableOnly
9658 pages reserved
0 pages cma reserved
The order-0 allocation for normal zone failed while there are a lot of
reclaimable memory(i.e., anonymous memory with free swap). I wanted to
analyze the problem but it was hard because we removed per-zone lru stat
so I couldn't know how many of anonymous memory there are in normal/dma
zone.
When we investigate OOM problem, reclaimable memory count is crucial
stat to find a problem. Without it, it's hard to parse the OOM message
so I believe we should keep it.
With per-zone lru stat,
gfp_mask=0x26004c0(GFP_KERNEL|__GFP_REPEAT|__GFP_NOTRACK), order=0
Mem-Info:
active_anon:101103 inactive_anon:102219 isolated_anon:0
active_file:503 inactive_file:544 isolated_file:0
unevictable:0 dirty:0 writeback:34 unstable:0
slab_reclaimable:6298 slab_unreclaimable:74669
mapped:863 shmem:0 pagetables:100998 bounce:0
free:23573 free_pcp:1861 free_cma:0
Node 0 active_anon:404412kB inactive_anon:409040kB active_file:2012kB inactive_file:2176kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:3452kB dirty:0kB writeback:136kB shmem:0kB writeback_tmp:0kB unstable:0kB pages_scanned:1320845 all_unreclaimable? yes
DMA free:3296kB min:68kB low:84kB high:100kB active_anon:5540kB inactive_anon:0kB active_file:0kB inactive_file:0kB present:15992kB managed:15916kB mlocked:0kB slab_reclaimable:248kB slab_unreclaimable:2628kB kernel_stack:792kB pagetables:2316kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB
lowmem_reserve[]: 0 809 1965 1965
Normal free:3600kB min:3604kB low:4504kB high:5404kB active_anon:86304kB inactive_anon:0kB active_file:160kB inactive_file:376kB present:897016kB managed:858524kB mlocked:0kB slab_reclaimable:24944kB slab_unreclaimable:296048kB kernel_stack:163832kB pagetables:35892kB bounce:0kB free_pcp:3076kB local_pcp:656kB free_cma:0kB
lowmem_reserve[]: 0 0 9247 9247
HighMem free:86156kB min:512kB low:1796kB high:3080kB active_anon:312852kB inactive_anon:410024kB active_file:1924kB inactive_file:2012kB present:1183736kB managed:1183736kB mlocked:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:365784kB bounce:0kB free_pcp:3868kB local_pcp:720kB free_cma:0kB
lowmem_reserve[]: 0 0 0 0
DMA: 8*4kB (UM) 8*8kB (UM) 4*16kB (M) 2*32kB (UM) 2*64kB (UM) 1*128kB (M) 3*256kB (UME) 2*512kB (UE) 1*1024kB (E) 0*2048kB 0*4096kB = 3296kB
Normal: 240*4kB (UME) 160*8kB (UME) 23*16kB (ME) 3*32kB (UE) 3*64kB (UME) 2*128kB (ME) 1*256kB (U) 0*512kB 0*1024kB 0*2048kB 0*4096kB = 3408kB
HighMem: 10942*4kB (UM) 3102*8kB (UM) 866*16kB (UM) 76*32kB (UM) 11*64kB (UM) 4*128kB (UM) 1*256kB (M) 0*512kB 0*1024kB 0*2048kB 0*4096kB = 86344kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
54409 total pagecache pages
53215 pages in swap cache
Swap cache stats: add 300982, delete 247765, find 157978/226539
Free swap = 3803244kB
Total swap = 4192252kB
524186 pages RAM
295934 pages HighMem/MovableOnly
9642 pages reserved
0 pages cma reserved
With that, we can see normal zone has a 86M reclaimable memory so we can
know something goes wrong(I will fix the problem in next patch) in
reclaim.
[mgorman@techsingularity.net: rename zone LRU stats in /proc/vmstat]
Link: http://lkml.kernel.org/r/20160725072300.GK10438@techsingularity.net
Link: http://lkml.kernel.org/r/1469110261-7365-2-git-send-email-mgorman@techsingularity.net
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:47:26 -06:00
|
|
|
" active_anon:%lukB"
|
|
|
|
" inactive_anon:%lukB"
|
|
|
|
" active_file:%lukB"
|
|
|
|
" inactive_file:%lukB"
|
|
|
|
" unevictable:%lukB"
|
2016-07-28 16:47:31 -06:00
|
|
|
" writepending:%lukB"
|
2005-04-16 16:20:36 -06:00
|
|
|
" present:%lukB"
|
2012-12-12 14:52:12 -07:00
|
|
|
" managed:%lukB"
|
2009-09-21 18:01:30 -06:00
|
|
|
" mlocked:%lukB"
|
|
|
|
" slab_reclaimable:%lukB"
|
|
|
|
" slab_unreclaimable:%lukB"
|
2009-09-21 18:01:32 -06:00
|
|
|
" kernel_stack:%lukB"
|
2009-09-21 18:01:30 -06:00
|
|
|
" pagetables:%lukB"
|
|
|
|
" bounce:%lukB"
|
2015-04-14 16:45:30 -06:00
|
|
|
" free_pcp:%lukB"
|
|
|
|
" local_pcp:%ukB"
|
2012-10-08 17:32:02 -06:00
|
|
|
" free_cma:%lukB"
|
2005-04-16 16:20:36 -06:00
|
|
|
"\n",
|
|
|
|
zone->name,
|
mm: page allocator: adjust the per-cpu counter threshold when memory is low
Commit aa45484 ("calculate a better estimate of NR_FREE_PAGES when memory
is low") noted that watermarks were based on the vmstat NR_FREE_PAGES. To
avoid synchronization overhead, these counters are maintained on a per-cpu
basis and drained both periodically and when a threshold is above a
threshold. On large CPU systems, the difference between the estimate and
real value of NR_FREE_PAGES can be very high. The system can get into a
case where pages are allocated far below the min watermark potentially
causing livelock issues. The commit solved the problem by taking a better
reading of NR_FREE_PAGES when memory was low.
Unfortately, as reported by Shaohua Li this accurate reading can consume a
large amount of CPU time on systems with many sockets due to cache line
bouncing. This patch takes a different approach. For large machines
where counter drift might be unsafe and while kswapd is awake, the per-cpu
thresholds for the target pgdat are reduced to limit the level of drift to
what should be a safe level. This incurs a performance penalty in heavy
memory pressure by a factor that depends on the workload and the machine
but the machine should function correctly without accidentally exhausting
all memory on a node. There is an additional cost when kswapd wakes and
sleeps but the event is not expected to be frequent - in Shaohua's test
case, there was one recorded sleep and wake event at least.
To ensure that kswapd wakes up, a safe version of zone_watermark_ok() is
introduced that takes a more accurate reading of NR_FREE_PAGES when called
from wakeup_kswapd, when deciding whether it is really safe to go back to
sleep in sleeping_prematurely() and when deciding if a zone is really
balanced or not in balance_pgdat(). We are still using an expensive
function but limiting how often it is called.
When the test case is reproduced, the time spent in the watermark
functions is reduced. The following report is on the percentage of time
spent cumulatively spent in the functions zone_nr_free_pages(),
zone_watermark_ok(), __zone_watermark_ok(), zone_watermark_ok_safe(),
zone_page_state_snapshot(), zone_page_state().
vanilla 11.6615%
disable-threshold 0.2584%
David said:
: We had to pull aa454840 "mm: page allocator: calculate a better estimate
: of NR_FREE_PAGES when memory is low and kswapd is awake" from 2.6.36
: internally because tests showed that it would cause the machine to stall
: as the result of heavy kswapd activity. I merged it back with this fix as
: it is pending in the -mm tree and it solves the issue we were seeing, so I
: definitely think this should be pushed to -stable (and I would seriously
: consider it for 2.6.37 inclusion even at this late date).
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Reported-by: Shaohua Li <shaohua.li@intel.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Tested-by: Nicolas Bareil <nico@chdir.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: <stable@kernel.org> [2.6.37.1, 2.6.36.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 16:45:41 -07:00
|
|
|
K(zone_page_state(zone, NR_FREE_PAGES)),
|
2009-06-16 16:32:12 -06:00
|
|
|
K(min_wmark_pages(zone)),
|
|
|
|
K(low_wmark_pages(zone)),
|
|
|
|
K(high_wmark_pages(zone)),
|
mm: add per-zone lru list stat
When I did stress test with hackbench, I got OOM message frequently
which didn't ever happen in zone-lru.
gfp_mask=0x26004c0(GFP_KERNEL|__GFP_REPEAT|__GFP_NOTRACK), order=0
..
..
__alloc_pages_nodemask+0xe52/0xe60
? new_slab+0x39c/0x3b0
new_slab+0x39c/0x3b0
___slab_alloc.constprop.87+0x6da/0x840
? __alloc_skb+0x3c/0x260
? _raw_spin_unlock_irq+0x27/0x60
? trace_hardirqs_on_caller+0xec/0x1b0
? finish_task_switch+0xa6/0x220
? poll_select_copy_remaining+0x140/0x140
__slab_alloc.isra.81.constprop.86+0x40/0x6d
? __alloc_skb+0x3c/0x260
kmem_cache_alloc+0x22c/0x260
? __alloc_skb+0x3c/0x260
__alloc_skb+0x3c/0x260
alloc_skb_with_frags+0x4e/0x1a0
sock_alloc_send_pskb+0x16a/0x1b0
? wait_for_unix_gc+0x31/0x90
? alloc_set_pte+0x2ad/0x310
unix_stream_sendmsg+0x28d/0x340
sock_sendmsg+0x2d/0x40
sock_write_iter+0x6c/0xc0
__vfs_write+0xc0/0x120
vfs_write+0x9b/0x1a0
? __might_fault+0x49/0xa0
SyS_write+0x44/0x90
do_fast_syscall_32+0xa6/0x1e0
sysenter_past_esp+0x45/0x74
Mem-Info:
active_anon:104698 inactive_anon:105791 isolated_anon:192
active_file:433 inactive_file:283 isolated_file:22
unevictable:0 dirty:0 writeback:296 unstable:0
slab_reclaimable:6389 slab_unreclaimable:78927
mapped:474 shmem:0 pagetables:101426 bounce:0
free:10518 free_pcp:334 free_cma:0
Node 0 active_anon:418792kB inactive_anon:423164kB active_file:1732kB inactive_file:1132kB unevictable:0kB isolated(anon):768kB isolated(file):88kB mapped:1896kB dirty:0kB writeback:1184kB shmem:0kB writeback_tmp:0kB unstable:0kB pages_scanned:1478632 all_unreclaimable? yes
DMA free:3304kB min:68kB low:84kB high:100kB present:15992kB managed:15916kB mlocked:0kB slab_reclaimable:0kB slab_unreclaimable:4088kB kernel_stack:0kB pagetables:2480kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB
lowmem_reserve[]: 0 809 1965 1965
Normal free:3436kB min:3604kB low:4504kB high:5404kB present:897016kB managed:858460kB mlocked:0kB slab_reclaimable:25556kB slab_unreclaimable:311712kB kernel_stack:164608kB pagetables:30844kB bounce:0kB free_pcp:620kB local_pcp:104kB free_cma:0kB
lowmem_reserve[]: 0 0 9247 9247
HighMem free:33808kB min:512kB low:1796kB high:3080kB present:1183736kB managed:1183736kB mlocked:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:372252kB bounce:0kB free_pcp:428kB local_pcp:72kB free_cma:0kB
lowmem_reserve[]: 0 0 0 0
DMA: 2*4kB (UM) 2*8kB (UM) 0*16kB 1*32kB (U) 1*64kB (U) 2*128kB (UM) 1*256kB (U) 1*512kB (M) 0*1024kB 1*2048kB (U) 0*4096kB = 3192kB
Normal: 33*4kB (MH) 79*8kB (ME) 11*16kB (M) 4*32kB (M) 2*64kB (ME) 2*128kB (EH) 7*256kB (EH) 0*512kB 0*1024kB 0*2048kB 0*4096kB = 3244kB
HighMem: 2590*4kB (UM) 1568*8kB (UM) 491*16kB (UM) 60*32kB (UM) 6*64kB (M) 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 33064kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
25121 total pagecache pages
24160 pages in swap cache
Swap cache stats: add 86371, delete 62211, find 42865/60187
Free swap = 4015560kB
Total swap = 4192252kB
524186 pages RAM
295934 pages HighMem/MovableOnly
9658 pages reserved
0 pages cma reserved
The order-0 allocation for normal zone failed while there are a lot of
reclaimable memory(i.e., anonymous memory with free swap). I wanted to
analyze the problem but it was hard because we removed per-zone lru stat
so I couldn't know how many of anonymous memory there are in normal/dma
zone.
When we investigate OOM problem, reclaimable memory count is crucial
stat to find a problem. Without it, it's hard to parse the OOM message
so I believe we should keep it.
With per-zone lru stat,
gfp_mask=0x26004c0(GFP_KERNEL|__GFP_REPEAT|__GFP_NOTRACK), order=0
Mem-Info:
active_anon:101103 inactive_anon:102219 isolated_anon:0
active_file:503 inactive_file:544 isolated_file:0
unevictable:0 dirty:0 writeback:34 unstable:0
slab_reclaimable:6298 slab_unreclaimable:74669
mapped:863 shmem:0 pagetables:100998 bounce:0
free:23573 free_pcp:1861 free_cma:0
Node 0 active_anon:404412kB inactive_anon:409040kB active_file:2012kB inactive_file:2176kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:3452kB dirty:0kB writeback:136kB shmem:0kB writeback_tmp:0kB unstable:0kB pages_scanned:1320845 all_unreclaimable? yes
DMA free:3296kB min:68kB low:84kB high:100kB active_anon:5540kB inactive_anon:0kB active_file:0kB inactive_file:0kB present:15992kB managed:15916kB mlocked:0kB slab_reclaimable:248kB slab_unreclaimable:2628kB kernel_stack:792kB pagetables:2316kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB
lowmem_reserve[]: 0 809 1965 1965
Normal free:3600kB min:3604kB low:4504kB high:5404kB active_anon:86304kB inactive_anon:0kB active_file:160kB inactive_file:376kB present:897016kB managed:858524kB mlocked:0kB slab_reclaimable:24944kB slab_unreclaimable:296048kB kernel_stack:163832kB pagetables:35892kB bounce:0kB free_pcp:3076kB local_pcp:656kB free_cma:0kB
lowmem_reserve[]: 0 0 9247 9247
HighMem free:86156kB min:512kB low:1796kB high:3080kB active_anon:312852kB inactive_anon:410024kB active_file:1924kB inactive_file:2012kB present:1183736kB managed:1183736kB mlocked:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:365784kB bounce:0kB free_pcp:3868kB local_pcp:720kB free_cma:0kB
lowmem_reserve[]: 0 0 0 0
DMA: 8*4kB (UM) 8*8kB (UM) 4*16kB (M) 2*32kB (UM) 2*64kB (UM) 1*128kB (M) 3*256kB (UME) 2*512kB (UE) 1*1024kB (E) 0*2048kB 0*4096kB = 3296kB
Normal: 240*4kB (UME) 160*8kB (UME) 23*16kB (ME) 3*32kB (UE) 3*64kB (UME) 2*128kB (ME) 1*256kB (U) 0*512kB 0*1024kB 0*2048kB 0*4096kB = 3408kB
HighMem: 10942*4kB (UM) 3102*8kB (UM) 866*16kB (UM) 76*32kB (UM) 11*64kB (UM) 4*128kB (UM) 1*256kB (M) 0*512kB 0*1024kB 0*2048kB 0*4096kB = 86344kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
54409 total pagecache pages
53215 pages in swap cache
Swap cache stats: add 300982, delete 247765, find 157978/226539
Free swap = 3803244kB
Total swap = 4192252kB
524186 pages RAM
295934 pages HighMem/MovableOnly
9642 pages reserved
0 pages cma reserved
With that, we can see normal zone has a 86M reclaimable memory so we can
know something goes wrong(I will fix the problem in next patch) in
reclaim.
[mgorman@techsingularity.net: rename zone LRU stats in /proc/vmstat]
Link: http://lkml.kernel.org/r/20160725072300.GK10438@techsingularity.net
Link: http://lkml.kernel.org/r/1469110261-7365-2-git-send-email-mgorman@techsingularity.net
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:47:26 -06:00
|
|
|
K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
|
|
|
|
K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
|
|
|
|
K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
|
|
|
|
K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
|
|
|
|
K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
|
2016-07-28 16:47:31 -06:00
|
|
|
K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
|
2005-04-16 16:20:36 -06:00
|
|
|
K(zone->present_pages),
|
2012-12-12 14:52:12 -07:00
|
|
|
K(zone->managed_pages),
|
2009-09-21 18:01:30 -06:00
|
|
|
K(zone_page_state(zone, NR_MLOCK)),
|
|
|
|
K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
|
|
|
|
K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
|
2016-07-28 16:48:14 -06:00
|
|
|
zone_page_state(zone, NR_KERNEL_STACK_KB),
|
2009-09-21 18:01:30 -06:00
|
|
|
K(zone_page_state(zone, NR_PAGETABLE)),
|
|
|
|
K(zone_page_state(zone, NR_BOUNCE)),
|
2015-04-14 16:45:30 -06:00
|
|
|
K(free_pcp),
|
|
|
|
K(this_cpu_read(zone->pageset->pcp.count)),
|
2016-07-28 16:47:14 -06:00
|
|
|
K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
|
2005-04-16 16:20:36 -06:00
|
|
|
printk("lowmem_reserve[]:");
|
|
|
|
for (i = 0; i < MAX_NR_ZONES; i++)
|
2014-08-06 17:07:14 -06:00
|
|
|
printk(" %ld", zone->lowmem_reserve[i]);
|
2005-04-16 16:20:36 -06:00
|
|
|
printk("\n");
|
|
|
|
}
|
|
|
|
|
2009-03-31 16:19:31 -06:00
|
|
|
for_each_populated_zone(zone) {
|
2015-11-06 17:29:57 -07:00
|
|
|
unsigned int order;
|
|
|
|
unsigned long nr[MAX_ORDER], flags, total = 0;
|
2012-12-11 17:00:24 -07:00
|
|
|
unsigned char types[MAX_ORDER];
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2011-05-24 18:11:16 -06:00
|
|
|
if (skip_free_areas_node(filter, zone_to_nid(zone)))
|
2011-03-22 17:30:46 -06:00
|
|
|
continue;
|
2005-04-16 16:20:36 -06:00
|
|
|
show_node(zone);
|
|
|
|
printk("%s: ", zone->name);
|
|
|
|
|
|
|
|
spin_lock_irqsave(&zone->lock, flags);
|
|
|
|
for (order = 0; order < MAX_ORDER; order++) {
|
2012-12-11 17:00:24 -07:00
|
|
|
struct free_area *area = &zone->free_area[order];
|
|
|
|
int type;
|
|
|
|
|
|
|
|
nr[order] = area->nr_free;
|
2006-06-23 03:03:50 -06:00
|
|
|
total += nr[order] << order;
|
2012-12-11 17:00:24 -07:00
|
|
|
|
|
|
|
types[order] = 0;
|
|
|
|
for (type = 0; type < MIGRATE_TYPES; type++) {
|
|
|
|
if (!list_empty(&area->free_list[type]))
|
|
|
|
types[order] |= 1 << type;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
spin_unlock_irqrestore(&zone->lock, flags);
|
2012-12-11 17:00:24 -07:00
|
|
|
for (order = 0; order < MAX_ORDER; order++) {
|
2006-06-23 03:03:50 -06:00
|
|
|
printk("%lu*%lukB ", nr[order], K(1UL) << order);
|
2012-12-11 17:00:24 -07:00
|
|
|
if (nr[order])
|
|
|
|
show_migration_types(types[order]);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
printk("= %lukB\n", K(total));
|
|
|
|
}
|
|
|
|
|
2013-04-29 16:07:48 -06:00
|
|
|
hugetlb_show_meminfo();
|
|
|
|
|
2016-07-28 16:46:20 -06:00
|
|
|
printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
|
2008-02-04 23:29:30 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
show_swap_cache_info();
|
|
|
|
}
|
|
|
|
|
2008-04-28 03:12:18 -06:00
|
|
|
static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
|
|
|
|
{
|
|
|
|
zoneref->zone = zone;
|
|
|
|
zoneref->zone_idx = zone_idx(zone);
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* Builds allocation fallback zone lists.
|
2006-01-06 01:11:16 -07:00
|
|
|
*
|
|
|
|
* Add all populated zones of a node to the zonelist.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2007-07-16 00:38:01 -06:00
|
|
|
static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
|
2013-07-08 17:00:06 -06:00
|
|
|
int nr_zones)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-01-06 01:11:16 -07:00
|
|
|
struct zone *zone;
|
2013-07-08 17:00:06 -06:00
|
|
|
enum zone_type zone_type = MAX_NR_ZONES;
|
2006-01-06 01:11:18 -07:00
|
|
|
|
|
|
|
do {
|
2006-09-26 00:31:18 -06:00
|
|
|
zone_type--;
|
2006-01-06 01:11:19 -07:00
|
|
|
zone = pgdat->node_zones + zone_type;
|
2006-01-06 01:11:16 -07:00
|
|
|
if (populated_zone(zone)) {
|
2008-04-28 03:12:17 -06:00
|
|
|
zoneref_set_zone(zone,
|
|
|
|
&zonelist->_zonerefs[nr_zones++]);
|
2006-01-06 01:11:19 -07:00
|
|
|
check_highest_zone(zone_type);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2006-09-26 00:31:18 -06:00
|
|
|
} while (zone_type);
|
2013-07-08 17:00:06 -06:00
|
|
|
|
2006-01-06 01:11:19 -07:00
|
|
|
return nr_zones;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2007-07-16 00:38:01 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* zonelist_order:
|
|
|
|
* 0 = automatic detection of better ordering.
|
|
|
|
* 1 = order by ([node] distance, -zonetype)
|
|
|
|
* 2 = order by (-zonetype, [node] distance)
|
|
|
|
*
|
|
|
|
* If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
|
|
|
|
* the same zonelist. So only NUMA can configure this param.
|
|
|
|
*/
|
|
|
|
#define ZONELIST_ORDER_DEFAULT 0
|
|
|
|
#define ZONELIST_ORDER_NODE 1
|
|
|
|
#define ZONELIST_ORDER_ZONE 2
|
|
|
|
|
|
|
|
/* zonelist order in the kernel.
|
|
|
|
* set_zonelist_order() will set this to NODE or ZONE.
|
|
|
|
*/
|
|
|
|
static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
|
|
|
|
static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
|
|
|
|
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
2007-07-16 00:38:01 -06:00
|
|
|
/* The value user specified ....changed by config */
|
|
|
|
static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
|
|
|
|
/* string for sysctl */
|
|
|
|
#define NUMA_ZONELIST_ORDER_LEN 16
|
|
|
|
char numa_zonelist_order[16] = "default";
|
|
|
|
|
|
|
|
/*
|
|
|
|
* interface for configure zonelist ordering.
|
|
|
|
* command line option "numa_zonelist_order"
|
|
|
|
* = "[dD]efault - default, automatic configuration.
|
|
|
|
* = "[nN]ode - order by node locality, then by zone within node
|
|
|
|
* = "[zZ]one - order by zone, then by locality within zone
|
|
|
|
*/
|
|
|
|
|
|
|
|
static int __parse_numa_zonelist_order(char *s)
|
|
|
|
{
|
|
|
|
if (*s == 'd' || *s == 'D') {
|
|
|
|
user_zonelist_order = ZONELIST_ORDER_DEFAULT;
|
|
|
|
} else if (*s == 'n' || *s == 'N') {
|
|
|
|
user_zonelist_order = ZONELIST_ORDER_NODE;
|
|
|
|
} else if (*s == 'z' || *s == 'Z') {
|
|
|
|
user_zonelist_order = ZONELIST_ORDER_ZONE;
|
|
|
|
} else {
|
2016-03-17 15:19:50 -06:00
|
|
|
pr_warn("Ignoring invalid numa_zonelist_order value: %s\n", s);
|
2007-07-16 00:38:01 -06:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static __init int setup_numa_zonelist_order(char *s)
|
|
|
|
{
|
2011-01-13 16:46:26 -07:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!s)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
ret = __parse_numa_zonelist_order(s);
|
|
|
|
if (ret == 0)
|
|
|
|
strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN);
|
|
|
|
|
|
|
|
return ret;
|
2007-07-16 00:38:01 -06:00
|
|
|
}
|
|
|
|
early_param("numa_zonelist_order", setup_numa_zonelist_order);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* sysctl handler for numa_zonelist_order
|
|
|
|
*/
|
2014-06-06 15:38:09 -06:00
|
|
|
int numa_zonelist_order_handler(struct ctl_table *table, int write,
|
2009-09-23 16:57:19 -06:00
|
|
|
void __user *buffer, size_t *length,
|
2007-07-16 00:38:01 -06:00
|
|
|
loff_t *ppos)
|
|
|
|
{
|
|
|
|
char saved_string[NUMA_ZONELIST_ORDER_LEN];
|
|
|
|
int ret;
|
2009-12-23 13:00:47 -07:00
|
|
|
static DEFINE_MUTEX(zl_order_mutex);
|
2007-07-16 00:38:01 -06:00
|
|
|
|
2009-12-23 13:00:47 -07:00
|
|
|
mutex_lock(&zl_order_mutex);
|
2013-07-03 16:02:35 -06:00
|
|
|
if (write) {
|
|
|
|
if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
strcpy(saved_string, (char *)table->data);
|
|
|
|
}
|
2009-09-23 16:57:19 -06:00
|
|
|
ret = proc_dostring(table, write, buffer, length, ppos);
|
2007-07-16 00:38:01 -06:00
|
|
|
if (ret)
|
2009-12-23 13:00:47 -07:00
|
|
|
goto out;
|
2007-07-16 00:38:01 -06:00
|
|
|
if (write) {
|
|
|
|
int oldval = user_zonelist_order;
|
2013-07-03 16:02:35 -06:00
|
|
|
|
|
|
|
ret = __parse_numa_zonelist_order((char *)table->data);
|
|
|
|
if (ret) {
|
2007-07-16 00:38:01 -06:00
|
|
|
/*
|
|
|
|
* bogus value. restore saved string
|
|
|
|
*/
|
2013-07-03 16:02:35 -06:00
|
|
|
strncpy((char *)table->data, saved_string,
|
2007-07-16 00:38:01 -06:00
|
|
|
NUMA_ZONELIST_ORDER_LEN);
|
|
|
|
user_zonelist_order = oldval;
|
2010-05-24 15:32:52 -06:00
|
|
|
} else if (oldval != user_zonelist_order) {
|
|
|
|
mutex_lock(&zonelists_mutex);
|
2012-07-31 17:43:28 -06:00
|
|
|
build_all_zonelists(NULL, NULL);
|
2010-05-24 15:32:52 -06:00
|
|
|
mutex_unlock(&zonelists_mutex);
|
|
|
|
}
|
2007-07-16 00:38:01 -06:00
|
|
|
}
|
2009-12-23 13:00:47 -07:00
|
|
|
out:
|
|
|
|
mutex_unlock(&zl_order_mutex);
|
|
|
|
return ret;
|
2007-07-16 00:38:01 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2009-06-16 16:32:15 -06:00
|
|
|
#define MAX_NODE_LOAD (nr_online_nodes)
|
2007-07-16 00:38:01 -06:00
|
|
|
static int node_load[MAX_NUMNODES];
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/**
|
2005-05-01 09:59:25 -06:00
|
|
|
* find_next_best_node - find the next node that should appear in a given node's fallback list
|
2005-04-16 16:20:36 -06:00
|
|
|
* @node: node whose fallback list we're appending
|
|
|
|
* @used_node_mask: nodemask_t of already used nodes
|
|
|
|
*
|
|
|
|
* We use a number of factors to determine which is the next node that should
|
|
|
|
* appear on a given node's fallback list. The node should not have appeared
|
|
|
|
* already in @node's fallback list, and it should be the next closest node
|
|
|
|
* according to the distance array (which contains arbitrary distance values
|
|
|
|
* from each node to each node in the system), and should also prefer nodes
|
|
|
|
* with no CPUs, since presumably they'll have very little allocation pressure
|
|
|
|
* on them otherwise.
|
|
|
|
* It returns -1 if no node is found.
|
|
|
|
*/
|
2007-07-16 00:38:01 -06:00
|
|
|
static int find_next_best_node(int node, nodemask_t *used_node_mask)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-02-17 12:38:21 -07:00
|
|
|
int n, val;
|
2005-04-16 16:20:36 -06:00
|
|
|
int min_val = INT_MAX;
|
2013-02-22 17:35:36 -07:00
|
|
|
int best_node = NUMA_NO_NODE;
|
2009-03-12 22:19:46 -06:00
|
|
|
const struct cpumask *tmp = cpumask_of_node(0);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-02-17 12:38:21 -07:00
|
|
|
/* Use the local node if we haven't already */
|
|
|
|
if (!node_isset(node, *used_node_mask)) {
|
|
|
|
node_set(node, *used_node_mask);
|
|
|
|
return node;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-12-12 14:51:46 -07:00
|
|
|
for_each_node_state(n, N_MEMORY) {
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Don't want a node to appear more than once */
|
|
|
|
if (node_isset(n, *used_node_mask))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Use the distance array to find the distance */
|
|
|
|
val = node_distance(node, n);
|
|
|
|
|
2006-02-17 12:38:21 -07:00
|
|
|
/* Penalize nodes under us ("prefer the next node") */
|
|
|
|
val += (n < node);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* Give preference to headless and unused nodes */
|
2009-03-12 22:19:46 -06:00
|
|
|
tmp = cpumask_of_node(n);
|
|
|
|
if (!cpumask_empty(tmp))
|
2005-04-16 16:20:36 -06:00
|
|
|
val += PENALTY_FOR_NODE_WITH_CPUS;
|
|
|
|
|
|
|
|
/* Slight preference for less loaded node */
|
|
|
|
val *= (MAX_NODE_LOAD*MAX_NUMNODES);
|
|
|
|
val += node_load[n];
|
|
|
|
|
|
|
|
if (val < min_val) {
|
|
|
|
min_val = val;
|
|
|
|
best_node = n;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (best_node >= 0)
|
|
|
|
node_set(best_node, *used_node_mask);
|
|
|
|
|
|
|
|
return best_node;
|
|
|
|
}
|
|
|
|
|
2007-07-16 00:38:01 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Build zonelists ordered by node and zones within node.
|
|
|
|
* This results in maximum locality--normal zone overflows into local
|
|
|
|
* DMA zone, if any--but risks exhausting DMA zone.
|
|
|
|
*/
|
|
|
|
static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-07-16 00:38:01 -06:00
|
|
|
int j;
|
2005-04-16 16:20:36 -06:00
|
|
|
struct zonelist *zonelist;
|
2007-07-16 00:38:01 -06:00
|
|
|
|
2008-04-28 03:12:16 -06:00
|
|
|
zonelist = &pgdat->node_zonelists[0];
|
2008-04-28 03:12:17 -06:00
|
|
|
for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
|
2008-04-28 03:12:16 -06:00
|
|
|
;
|
2013-07-08 17:00:06 -06:00
|
|
|
j = build_zonelists_node(NODE_DATA(node), zonelist, j);
|
2008-04-28 03:12:17 -06:00
|
|
|
zonelist->_zonerefs[j].zone = NULL;
|
|
|
|
zonelist->_zonerefs[j].zone_idx = 0;
|
2007-07-16 00:38:01 -06:00
|
|
|
}
|
|
|
|
|
2007-10-16 02:25:37 -06:00
|
|
|
/*
|
|
|
|
* Build gfp_thisnode zonelists
|
|
|
|
*/
|
|
|
|
static void build_thisnode_zonelists(pg_data_t *pgdat)
|
|
|
|
{
|
|
|
|
int j;
|
|
|
|
struct zonelist *zonelist;
|
|
|
|
|
2008-04-28 03:12:16 -06:00
|
|
|
zonelist = &pgdat->node_zonelists[1];
|
2013-07-08 17:00:06 -06:00
|
|
|
j = build_zonelists_node(pgdat, zonelist, 0);
|
2008-04-28 03:12:17 -06:00
|
|
|
zonelist->_zonerefs[j].zone = NULL;
|
|
|
|
zonelist->_zonerefs[j].zone_idx = 0;
|
2007-10-16 02:25:37 -06:00
|
|
|
}
|
|
|
|
|
2007-07-16 00:38:01 -06:00
|
|
|
/*
|
|
|
|
* Build zonelists ordered by zone and nodes within zones.
|
|
|
|
* This results in conserving DMA zone[s] until all Normal memory is
|
|
|
|
* exhausted, but results in overflowing to remote node while memory
|
|
|
|
* may still exist in local DMA zone.
|
|
|
|
*/
|
|
|
|
static int node_order[MAX_NUMNODES];
|
|
|
|
|
|
|
|
static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
|
|
|
|
{
|
|
|
|
int pos, j, node;
|
|
|
|
int zone_type; /* needs to be signed */
|
|
|
|
struct zone *z;
|
|
|
|
struct zonelist *zonelist;
|
|
|
|
|
2008-04-28 03:12:16 -06:00
|
|
|
zonelist = &pgdat->node_zonelists[0];
|
|
|
|
pos = 0;
|
|
|
|
for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
|
|
|
|
for (j = 0; j < nr_nodes; j++) {
|
|
|
|
node = node_order[j];
|
|
|
|
z = &NODE_DATA(node)->node_zones[zone_type];
|
|
|
|
if (populated_zone(z)) {
|
2008-04-28 03:12:17 -06:00
|
|
|
zoneref_set_zone(z,
|
|
|
|
&zonelist->_zonerefs[pos++]);
|
2008-04-28 03:12:16 -06:00
|
|
|
check_highest_zone(zone_type);
|
2007-07-16 00:38:01 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2008-04-28 03:12:17 -06:00
|
|
|
zonelist->_zonerefs[pos].zone = NULL;
|
|
|
|
zonelist->_zonerefs[pos].zone_idx = 0;
|
2007-07-16 00:38:01 -06:00
|
|
|
}
|
|
|
|
|
2014-10-09 16:28:30 -06:00
|
|
|
#if defined(CONFIG_64BIT)
|
|
|
|
/*
|
|
|
|
* Devices that require DMA32/DMA are relatively rare and do not justify a
|
|
|
|
* penalty to every machine in case the specialised case applies. Default
|
|
|
|
* to Node-ordering on 64-bit NUMA machines
|
|
|
|
*/
|
|
|
|
static int default_zonelist_order(void)
|
|
|
|
{
|
|
|
|
return ZONELIST_ORDER_NODE;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
/*
|
|
|
|
* On 32-bit, the Normal zone needs to be preserved for allocations accessible
|
|
|
|
* by the kernel. If processes running on node 0 deplete the low memory zone
|
|
|
|
* then reclaim will occur more frequency increasing stalls and potentially
|
|
|
|
* be easier to OOM if a large percentage of the zone is under writeback or
|
|
|
|
* dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set.
|
|
|
|
* Hence, default to zone ordering on 32-bit.
|
|
|
|
*/
|
2007-07-16 00:38:01 -06:00
|
|
|
static int default_zonelist_order(void)
|
|
|
|
{
|
|
|
|
return ZONELIST_ORDER_ZONE;
|
|
|
|
}
|
2014-10-09 16:28:30 -06:00
|
|
|
#endif /* CONFIG_64BIT */
|
2007-07-16 00:38:01 -06:00
|
|
|
|
|
|
|
static void set_zonelist_order(void)
|
|
|
|
{
|
|
|
|
if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
|
|
|
|
current_zonelist_order = default_zonelist_order();
|
|
|
|
else
|
|
|
|
current_zonelist_order = user_zonelist_order;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void build_zonelists(pg_data_t *pgdat)
|
|
|
|
{
|
2016-01-14 16:19:00 -07:00
|
|
|
int i, node, load;
|
2005-04-16 16:20:36 -06:00
|
|
|
nodemask_t used_mask;
|
2007-07-16 00:38:01 -06:00
|
|
|
int local_node, prev_node;
|
|
|
|
struct zonelist *zonelist;
|
2015-11-06 17:29:57 -07:00
|
|
|
unsigned int order = current_zonelist_order;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* initialize zonelists */
|
2007-10-16 02:25:37 -06:00
|
|
|
for (i = 0; i < MAX_ZONELISTS; i++) {
|
2005-04-16 16:20:36 -06:00
|
|
|
zonelist = pgdat->node_zonelists + i;
|
2008-04-28 03:12:17 -06:00
|
|
|
zonelist->_zonerefs[0].zone = NULL;
|
|
|
|
zonelist->_zonerefs[0].zone_idx = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/* NUMA-aware ordering of nodes */
|
|
|
|
local_node = pgdat->node_id;
|
2009-06-16 16:32:15 -06:00
|
|
|
load = nr_online_nodes;
|
2005-04-16 16:20:36 -06:00
|
|
|
prev_node = local_node;
|
|
|
|
nodes_clear(used_mask);
|
2007-07-16 00:38:01 -06:00
|
|
|
|
|
|
|
memset(node_order, 0, sizeof(node_order));
|
2016-01-14 16:19:00 -07:00
|
|
|
i = 0;
|
2007-07-16 00:38:01 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
|
|
|
|
/*
|
|
|
|
* We don't want to pressure a particular node.
|
|
|
|
* So adding penalty to the first node in same
|
|
|
|
* distance group to make it round-robin.
|
|
|
|
*/
|
2012-10-08 17:33:24 -06:00
|
|
|
if (node_distance(local_node, node) !=
|
|
|
|
node_distance(local_node, prev_node))
|
2007-07-16 00:38:01 -06:00
|
|
|
node_load[node] = load;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
prev_node = node;
|
|
|
|
load--;
|
2007-07-16 00:38:01 -06:00
|
|
|
if (order == ZONELIST_ORDER_NODE)
|
|
|
|
build_zonelists_in_node_order(pgdat, node);
|
|
|
|
else
|
2016-01-14 16:19:00 -07:00
|
|
|
node_order[i++] = node; /* remember order */
|
2007-07-16 00:38:01 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-07-16 00:38:01 -06:00
|
|
|
if (order == ZONELIST_ORDER_ZONE) {
|
|
|
|
/* calculate node order -- i.e., DMA last! */
|
2016-01-14 16:19:00 -07:00
|
|
|
build_zonelists_in_zone_order(pgdat, i);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2007-10-16 02:25:37 -06:00
|
|
|
|
|
|
|
build_thisnode_zonelists(pgdat);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2010-05-26 15:45:00 -06:00
|
|
|
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
|
|
|
|
/*
|
|
|
|
* Return node id of node used for "local" allocations.
|
|
|
|
* I.e., first node id of first zone in arg node's generic zonelist.
|
|
|
|
* Used for initializing percpu 'numa_mem', which is used primarily
|
|
|
|
* for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
|
|
|
|
*/
|
|
|
|
int local_memory_node(int node)
|
|
|
|
{
|
2016-05-19 18:14:10 -06:00
|
|
|
struct zoneref *z;
|
2010-05-26 15:45:00 -06:00
|
|
|
|
2016-05-19 18:14:10 -06:00
|
|
|
z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
|
2010-05-26 15:45:00 -06:00
|
|
|
gfp_zone(GFP_KERNEL),
|
2016-05-19 18:14:10 -06:00
|
|
|
NULL);
|
|
|
|
return z->zone->node;
|
2010-05-26 15:45:00 -06:00
|
|
|
}
|
|
|
|
#endif
|
2007-07-16 00:38:01 -06:00
|
|
|
|
2016-08-10 17:27:49 -06:00
|
|
|
static void setup_min_unmapped_ratio(void);
|
|
|
|
static void setup_min_slab_ratio(void);
|
2005-04-16 16:20:36 -06:00
|
|
|
#else /* CONFIG_NUMA */
|
|
|
|
|
2007-07-16 00:38:01 -06:00
|
|
|
static void set_zonelist_order(void)
|
|
|
|
{
|
|
|
|
current_zonelist_order = ZONELIST_ORDER_ZONE;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void build_zonelists(pg_data_t *pgdat)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-09-26 00:31:19 -06:00
|
|
|
int node, local_node;
|
2008-04-28 03:12:16 -06:00
|
|
|
enum zone_type j;
|
|
|
|
struct zonelist *zonelist;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
local_node = pgdat->node_id;
|
|
|
|
|
2008-04-28 03:12:16 -06:00
|
|
|
zonelist = &pgdat->node_zonelists[0];
|
2013-07-08 17:00:06 -06:00
|
|
|
j = build_zonelists_node(pgdat, zonelist, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-04-28 03:12:16 -06:00
|
|
|
/*
|
|
|
|
* Now we build the zonelist so that it contains the zones
|
|
|
|
* of all the other nodes.
|
|
|
|
* We don't want to pressure a particular node, so when
|
|
|
|
* building the zones for node N, we make sure that the
|
|
|
|
* zones coming right after the local ones are those from
|
|
|
|
* node N+1 (modulo N)
|
|
|
|
*/
|
|
|
|
for (node = local_node + 1; node < MAX_NUMNODES; node++) {
|
|
|
|
if (!node_online(node))
|
|
|
|
continue;
|
2013-07-08 17:00:06 -06:00
|
|
|
j = build_zonelists_node(NODE_DATA(node), zonelist, j);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2008-04-28 03:12:16 -06:00
|
|
|
for (node = 0; node < local_node; node++) {
|
|
|
|
if (!node_online(node))
|
|
|
|
continue;
|
2013-07-08 17:00:06 -06:00
|
|
|
j = build_zonelists_node(NODE_DATA(node), zonelist, j);
|
2008-04-28 03:12:16 -06:00
|
|
|
}
|
|
|
|
|
2008-04-28 03:12:17 -06:00
|
|
|
zonelist->_zonerefs[j].zone = NULL;
|
|
|
|
zonelist->_zonerefs[j].zone_idx = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
#endif /* CONFIG_NUMA */
|
|
|
|
|
2010-01-04 23:34:51 -07:00
|
|
|
/*
|
|
|
|
* Boot pageset table. One per cpu which is going to be used for all
|
|
|
|
* zones and all nodes. The parameters will be set in such a way
|
|
|
|
* that an item put on a list will immediately be handed over to
|
|
|
|
* the buddy list. This is safe since pageset manipulation is done
|
|
|
|
* with interrupts disabled.
|
|
|
|
*
|
|
|
|
* The boot_pagesets must be kept even after bootup is complete for
|
|
|
|
* unused processors and/or zones. They do play a role for bootstrapping
|
|
|
|
* hotplugged processors.
|
|
|
|
*
|
|
|
|
* zoneinfo_show() and maybe other functions do
|
|
|
|
* not check if the processor is online before following the pageset pointer.
|
|
|
|
* Other parts of the kernel may not check if the zone is available.
|
|
|
|
*/
|
|
|
|
static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
|
|
|
|
static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
|
2010-05-24 15:32:51 -06:00
|
|
|
static void setup_zone_pageset(struct zone *zone);
|
2010-01-04 23:34:51 -07:00
|
|
|
|
2010-05-24 15:32:52 -06:00
|
|
|
/*
|
|
|
|
* Global mutex to protect against size modification of zonelists
|
|
|
|
* as well as to serialize pageset setup for the new populated zone.
|
|
|
|
*/
|
|
|
|
DEFINE_MUTEX(zonelists_mutex);
|
|
|
|
|
2008-07-28 11:16:30 -06:00
|
|
|
/* return values int ....just for stop_machine() */
|
2012-07-31 17:43:35 -06:00
|
|
|
static int __build_all_zonelists(void *data)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-06-23 03:03:11 -06:00
|
|
|
int nid;
|
2010-01-04 23:34:51 -07:00
|
|
|
int cpu;
|
2012-07-31 17:43:28 -06:00
|
|
|
pg_data_t *self = data;
|
[PATCH] memory page_alloc zonelist caching speedup
Optimize the critical zonelist scanning for free pages in the kernel memory
allocator by caching the zones that were found to be full recently, and
skipping them.
Remembers the zones in a zonelist that were short of free memory in the
last second. And it stashes a zone-to-node table in the zonelist struct,
to optimize that conversion (minimize its cache footprint.)
Recent changes:
This differs in a significant way from a similar patch that I
posted a week ago. Now, instead of having a nodemask_t of
recently full nodes, I have a bitmask of recently full zones.
This solves a problem that last weeks patch had, which on
systems with multiple zones per node (such as DMA zone) would
take seeing any of these zones full as meaning that all zones
on that node were full.
Also I changed names - from "zonelist faster" to "zonelist cache",
as that seemed to better convey what we're doing here - caching
some of the key zonelist state (for faster access.)
See below for some performance benchmark results. After all that
discussion with David on why I didn't need them, I went and got
some ;). I wanted to verify that I had not hurt the normal case
of memory allocation noticeably. At least for my one little
microbenchmark, I found (1) the normal case wasn't affected, and
(2) workloads that forced scanning across multiple nodes for
memory improved up to 10% fewer System CPU cycles and lower
elapsed clock time ('sys' and 'real'). Good. See details, below.
I didn't have the logic in get_page_from_freelist() for various
full nodes and zone reclaim failures correct. That should be
fixed up now - notice the new goto labels zonelist_scan,
this_zone_full, and try_next_zone, in get_page_from_freelist().
There are two reasons I persued this alternative, over some earlier
proposals that would have focused on optimizing the fake numa
emulation case by caching the last useful zone:
1) Contrary to what I said before, we (SGI, on large ia64 sn2 systems)
have seen real customer loads where the cost to scan the zonelist
was a problem, due to many nodes being full of memory before
we got to a node we could use. Or at least, I think we have.
This was related to me by another engineer, based on experiences
from some time past. So this is not guaranteed. Most likely, though.
The following approach should help such real numa systems just as
much as it helps fake numa systems, or any combination thereof.
2) The effort to distinguish fake from real numa, using node_distance,
so that we could cache a fake numa node and optimize choosing
it over equivalent distance fake nodes, while continuing to
properly scan all real nodes in distance order, was going to
require a nasty blob of zonelist and node distance munging.
The following approach has no new dependency on node distances or
zone sorting.
See comment in the patch below for a description of what it actually does.
Technical details of note (or controversy):
- See the use of "zlc_active" and "did_zlc_setup" below, to delay
adding any work for this new mechanism until we've looked at the
first zone in zonelist. I figured the odds of the first zone
having the memory we needed were high enough that we should just
look there, first, then get fancy only if we need to keep looking.
- Some odd hackery was needed to add items to struct zonelist, while
not tripping up the custom zonelists built by the mm/mempolicy.c
code for MPOL_BIND. My usual wordy comments below explain this.
Search for "MPOL_BIND".
- Some per-node data in the struct zonelist is now modified frequently,
with no locking. Multiple CPU cores on a node could hit and mangle
this data. The theory is that this is just performance hint data,
and the memory allocator will work just fine despite any such mangling.
The fields at risk are the struct 'zonelist_cache' fields 'fullzones'
(a bitmask) and 'last_full_zap' (unsigned long jiffies). It should
all be self correcting after at most a one second delay.
- This still does a linear scan of the same lengths as before. All
I've optimized is making the scan faster, not algorithmically
shorter. It is now able to scan a compact array of 'unsigned
short' in the case of many full nodes, so one cache line should
cover quite a few nodes, rather than each node hitting another
one or two new and distinct cache lines.
- If both Andi and Nick don't find this too complicated, I will be
(pleasantly) flabbergasted.
- I removed the comment claiming we only use one cachline's worth of
zonelist. We seem, at least in the fake numa case, to have put the
lie to that claim.
- I pay no attention to the various watermarks and such in this performance
hint. A node could be marked full for one watermark, and then skipped
over when searching for a page using a different watermark. I think
that's actually quite ok, as it will tend to slightly increase the
spreading of memory over other nodes, away from a memory stressed node.
===============
Performance - some benchmark results and analysis:
This benchmark runs a memory hog program that uses multiple
threads to touch alot of memory as quickly as it can.
Multiple runs were made, touching 12, 38, 64 or 90 GBytes out of
the total 96 GBytes on the system, and using 1, 19, 37, or 55
threads (on a 56 CPU system.) System, user and real (elapsed)
timings were recorded for each run, shown in units of seconds,
in the table below.
Two kernels were tested - 2.6.18-mm3 and the same kernel with
this zonelist caching patch added. The table also shows the
percentage improvement the zonelist caching sys time is over
(lower than) the stock *-mm kernel.
number 2.6.18-mm3 zonelist-cache delta (< 0 good) percent
GBs N ------------ -------------- ---------------- systime
mem threads sys user real sys user real sys user real better
12 1 153 24 177 151 24 176 -2 0 -1 1%
12 19 99 22 8 99 22 8 0 0 0 0%
12 37 111 25 6 112 25 6 1 0 0 -0%
12 55 115 25 5 110 23 5 -5 -2 0 4%
38 1 502 74 576 497 73 570 -5 -1 -6 0%
38 19 426 78 48 373 76 39 -53 -2 -9 12%
38 37 544 83 36 547 82 36 3 -1 0 -0%
38 55 501 77 23 511 80 24 10 3 1 -1%
64 1 917 125 1042 890 124 1014 -27 -1 -28 2%
64 19 1118 138 119 965 141 103 -153 3 -16 13%
64 37 1202 151 94 1136 150 81 -66 -1 -13 5%
64 55 1118 141 61 1072 140 58 -46 -1 -3 4%
90 1 1342 177 1519 1275 174 1450 -67 -3 -69 4%
90 19 2392 199 192 2116 189 176 -276 -10 -16 11%
90 37 3313 238 175 2972 225 145 -341 -13 -30 10%
90 55 1948 210 104 1843 213 100 -105 3 -4 5%
Notes:
1) This test ran a memory hog program that started a specified number N of
threads, and had each thread allocate and touch 1/N'th of
the total memory to be used in the test run in a single loop,
writing a constant word to memory, one store every 4096 bytes.
Watching this test during some earlier trial runs, I would see
each of these threads sit down on one CPU and stay there, for
the remainder of the pass, a different CPU for each thread.
2) The 'real' column is not comparable to the 'sys' or 'user' columns.
The 'real' column is seconds wall clock time elapsed, from beginning
to end of that test pass. The 'sys' and 'user' columns are total
CPU seconds spent on that test pass. For a 19 thread test run,
for example, the sum of 'sys' and 'user' could be up to 19 times the
number of 'real' elapsed wall clock seconds.
3) Tests were run on a fresh, single-user boot, to minimize the amount
of memory already in use at the start of the test, and to minimize
the amount of background activity that might interfere.
4) Tests were done on a 56 CPU, 28 Node system with 96 GBytes of RAM.
5) Notice that the 'real' time gets large for the single thread runs, even
though the measured 'sys' and 'user' times are modest. I'm not sure what
that means - probably something to do with it being slow for one thread to
be accessing memory along ways away. Perhaps the fake numa system, running
ostensibly the same workload, would not show this substantial degradation
of 'real' time for one thread on many nodes -- lets hope not.
6) The high thread count passes (one thread per CPU - on 55 of 56 CPUs)
ran quite efficiently, as one might expect. Each pair of threads needed
to allocate and touch the memory on the node the two threads shared, a
pleasantly parallizable workload.
7) The intermediate thread count passes, when asking for alot of memory forcing
them to go to a few neighboring nodes, improved the most with this zonelist
caching patch.
Conclusions:
* This zonelist cache patch probably makes little difference one way or the
other for most workloads on real numa hardware, if those workloads avoid
heavy off node allocations.
* For memory intensive workloads requiring substantial off-node allocations
on real numa hardware, this patch improves both kernel and elapsed timings
up to ten per-cent.
* For fake numa systems, I'm optimistic, but will have to leave that up to
Rohit Seth to actually test (once I get him a 2.6.18 backport.)
Signed-off-by: Paul Jackson <pj@sgi.com>
Cc: Rohit Seth <rohitseth@google.com>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Cc: David Rientjes <rientjes@cs.washington.edu>
Cc: Paul Menage <menage@google.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-06 21:31:48 -07:00
|
|
|
|
2009-08-18 15:11:19 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
memset(node_load, 0, sizeof(node_load));
|
|
|
|
#endif
|
2012-07-31 17:43:28 -06:00
|
|
|
|
|
|
|
if (self && !node_online(self->node_id)) {
|
|
|
|
build_zonelists(self);
|
|
|
|
}
|
|
|
|
|
[PATCH] memory page_alloc zonelist caching speedup
Optimize the critical zonelist scanning for free pages in the kernel memory
allocator by caching the zones that were found to be full recently, and
skipping them.
Remembers the zones in a zonelist that were short of free memory in the
last second. And it stashes a zone-to-node table in the zonelist struct,
to optimize that conversion (minimize its cache footprint.)
Recent changes:
This differs in a significant way from a similar patch that I
posted a week ago. Now, instead of having a nodemask_t of
recently full nodes, I have a bitmask of recently full zones.
This solves a problem that last weeks patch had, which on
systems with multiple zones per node (such as DMA zone) would
take seeing any of these zones full as meaning that all zones
on that node were full.
Also I changed names - from "zonelist faster" to "zonelist cache",
as that seemed to better convey what we're doing here - caching
some of the key zonelist state (for faster access.)
See below for some performance benchmark results. After all that
discussion with David on why I didn't need them, I went and got
some ;). I wanted to verify that I had not hurt the normal case
of memory allocation noticeably. At least for my one little
microbenchmark, I found (1) the normal case wasn't affected, and
(2) workloads that forced scanning across multiple nodes for
memory improved up to 10% fewer System CPU cycles and lower
elapsed clock time ('sys' and 'real'). Good. See details, below.
I didn't have the logic in get_page_from_freelist() for various
full nodes and zone reclaim failures correct. That should be
fixed up now - notice the new goto labels zonelist_scan,
this_zone_full, and try_next_zone, in get_page_from_freelist().
There are two reasons I persued this alternative, over some earlier
proposals that would have focused on optimizing the fake numa
emulation case by caching the last useful zone:
1) Contrary to what I said before, we (SGI, on large ia64 sn2 systems)
have seen real customer loads where the cost to scan the zonelist
was a problem, due to many nodes being full of memory before
we got to a node we could use. Or at least, I think we have.
This was related to me by another engineer, based on experiences
from some time past. So this is not guaranteed. Most likely, though.
The following approach should help such real numa systems just as
much as it helps fake numa systems, or any combination thereof.
2) The effort to distinguish fake from real numa, using node_distance,
so that we could cache a fake numa node and optimize choosing
it over equivalent distance fake nodes, while continuing to
properly scan all real nodes in distance order, was going to
require a nasty blob of zonelist and node distance munging.
The following approach has no new dependency on node distances or
zone sorting.
See comment in the patch below for a description of what it actually does.
Technical details of note (or controversy):
- See the use of "zlc_active" and "did_zlc_setup" below, to delay
adding any work for this new mechanism until we've looked at the
first zone in zonelist. I figured the odds of the first zone
having the memory we needed were high enough that we should just
look there, first, then get fancy only if we need to keep looking.
- Some odd hackery was needed to add items to struct zonelist, while
not tripping up the custom zonelists built by the mm/mempolicy.c
code for MPOL_BIND. My usual wordy comments below explain this.
Search for "MPOL_BIND".
- Some per-node data in the struct zonelist is now modified frequently,
with no locking. Multiple CPU cores on a node could hit and mangle
this data. The theory is that this is just performance hint data,
and the memory allocator will work just fine despite any such mangling.
The fields at risk are the struct 'zonelist_cache' fields 'fullzones'
(a bitmask) and 'last_full_zap' (unsigned long jiffies). It should
all be self correcting after at most a one second delay.
- This still does a linear scan of the same lengths as before. All
I've optimized is making the scan faster, not algorithmically
shorter. It is now able to scan a compact array of 'unsigned
short' in the case of many full nodes, so one cache line should
cover quite a few nodes, rather than each node hitting another
one or two new and distinct cache lines.
- If both Andi and Nick don't find this too complicated, I will be
(pleasantly) flabbergasted.
- I removed the comment claiming we only use one cachline's worth of
zonelist. We seem, at least in the fake numa case, to have put the
lie to that claim.
- I pay no attention to the various watermarks and such in this performance
hint. A node could be marked full for one watermark, and then skipped
over when searching for a page using a different watermark. I think
that's actually quite ok, as it will tend to slightly increase the
spreading of memory over other nodes, away from a memory stressed node.
===============
Performance - some benchmark results and analysis:
This benchmark runs a memory hog program that uses multiple
threads to touch alot of memory as quickly as it can.
Multiple runs were made, touching 12, 38, 64 or 90 GBytes out of
the total 96 GBytes on the system, and using 1, 19, 37, or 55
threads (on a 56 CPU system.) System, user and real (elapsed)
timings were recorded for each run, shown in units of seconds,
in the table below.
Two kernels were tested - 2.6.18-mm3 and the same kernel with
this zonelist caching patch added. The table also shows the
percentage improvement the zonelist caching sys time is over
(lower than) the stock *-mm kernel.
number 2.6.18-mm3 zonelist-cache delta (< 0 good) percent
GBs N ------------ -------------- ---------------- systime
mem threads sys user real sys user real sys user real better
12 1 153 24 177 151 24 176 -2 0 -1 1%
12 19 99 22 8 99 22 8 0 0 0 0%
12 37 111 25 6 112 25 6 1 0 0 -0%
12 55 115 25 5 110 23 5 -5 -2 0 4%
38 1 502 74 576 497 73 570 -5 -1 -6 0%
38 19 426 78 48 373 76 39 -53 -2 -9 12%
38 37 544 83 36 547 82 36 3 -1 0 -0%
38 55 501 77 23 511 80 24 10 3 1 -1%
64 1 917 125 1042 890 124 1014 -27 -1 -28 2%
64 19 1118 138 119 965 141 103 -153 3 -16 13%
64 37 1202 151 94 1136 150 81 -66 -1 -13 5%
64 55 1118 141 61 1072 140 58 -46 -1 -3 4%
90 1 1342 177 1519 1275 174 1450 -67 -3 -69 4%
90 19 2392 199 192 2116 189 176 -276 -10 -16 11%
90 37 3313 238 175 2972 225 145 -341 -13 -30 10%
90 55 1948 210 104 1843 213 100 -105 3 -4 5%
Notes:
1) This test ran a memory hog program that started a specified number N of
threads, and had each thread allocate and touch 1/N'th of
the total memory to be used in the test run in a single loop,
writing a constant word to memory, one store every 4096 bytes.
Watching this test during some earlier trial runs, I would see
each of these threads sit down on one CPU and stay there, for
the remainder of the pass, a different CPU for each thread.
2) The 'real' column is not comparable to the 'sys' or 'user' columns.
The 'real' column is seconds wall clock time elapsed, from beginning
to end of that test pass. The 'sys' and 'user' columns are total
CPU seconds spent on that test pass. For a 19 thread test run,
for example, the sum of 'sys' and 'user' could be up to 19 times the
number of 'real' elapsed wall clock seconds.
3) Tests were run on a fresh, single-user boot, to minimize the amount
of memory already in use at the start of the test, and to minimize
the amount of background activity that might interfere.
4) Tests were done on a 56 CPU, 28 Node system with 96 GBytes of RAM.
5) Notice that the 'real' time gets large for the single thread runs, even
though the measured 'sys' and 'user' times are modest. I'm not sure what
that means - probably something to do with it being slow for one thread to
be accessing memory along ways away. Perhaps the fake numa system, running
ostensibly the same workload, would not show this substantial degradation
of 'real' time for one thread on many nodes -- lets hope not.
6) The high thread count passes (one thread per CPU - on 55 of 56 CPUs)
ran quite efficiently, as one might expect. Each pair of threads needed
to allocate and touch the memory on the node the two threads shared, a
pleasantly parallizable workload.
7) The intermediate thread count passes, when asking for alot of memory forcing
them to go to a few neighboring nodes, improved the most with this zonelist
caching patch.
Conclusions:
* This zonelist cache patch probably makes little difference one way or the
other for most workloads on real numa hardware, if those workloads avoid
heavy off node allocations.
* For memory intensive workloads requiring substantial off-node allocations
on real numa hardware, this patch improves both kernel and elapsed timings
up to ten per-cent.
* For fake numa systems, I'm optimistic, but will have to leave that up to
Rohit Seth to actually test (once I get him a 2.6.18 backport.)
Signed-off-by: Paul Jackson <pj@sgi.com>
Cc: Rohit Seth <rohitseth@google.com>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Cc: David Rientjes <rientjes@cs.washington.edu>
Cc: Paul Menage <menage@google.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-06 21:31:48 -07:00
|
|
|
for_each_online_node(nid) {
|
2007-10-16 02:25:29 -06:00
|
|
|
pg_data_t *pgdat = NODE_DATA(nid);
|
|
|
|
|
|
|
|
build_zonelists(pgdat);
|
[PATCH] memory page_alloc zonelist caching speedup
Optimize the critical zonelist scanning for free pages in the kernel memory
allocator by caching the zones that were found to be full recently, and
skipping them.
Remembers the zones in a zonelist that were short of free memory in the
last second. And it stashes a zone-to-node table in the zonelist struct,
to optimize that conversion (minimize its cache footprint.)
Recent changes:
This differs in a significant way from a similar patch that I
posted a week ago. Now, instead of having a nodemask_t of
recently full nodes, I have a bitmask of recently full zones.
This solves a problem that last weeks patch had, which on
systems with multiple zones per node (such as DMA zone) would
take seeing any of these zones full as meaning that all zones
on that node were full.
Also I changed names - from "zonelist faster" to "zonelist cache",
as that seemed to better convey what we're doing here - caching
some of the key zonelist state (for faster access.)
See below for some performance benchmark results. After all that
discussion with David on why I didn't need them, I went and got
some ;). I wanted to verify that I had not hurt the normal case
of memory allocation noticeably. At least for my one little
microbenchmark, I found (1) the normal case wasn't affected, and
(2) workloads that forced scanning across multiple nodes for
memory improved up to 10% fewer System CPU cycles and lower
elapsed clock time ('sys' and 'real'). Good. See details, below.
I didn't have the logic in get_page_from_freelist() for various
full nodes and zone reclaim failures correct. That should be
fixed up now - notice the new goto labels zonelist_scan,
this_zone_full, and try_next_zone, in get_page_from_freelist().
There are two reasons I persued this alternative, over some earlier
proposals that would have focused on optimizing the fake numa
emulation case by caching the last useful zone:
1) Contrary to what I said before, we (SGI, on large ia64 sn2 systems)
have seen real customer loads where the cost to scan the zonelist
was a problem, due to many nodes being full of memory before
we got to a node we could use. Or at least, I think we have.
This was related to me by another engineer, based on experiences
from some time past. So this is not guaranteed. Most likely, though.
The following approach should help such real numa systems just as
much as it helps fake numa systems, or any combination thereof.
2) The effort to distinguish fake from real numa, using node_distance,
so that we could cache a fake numa node and optimize choosing
it over equivalent distance fake nodes, while continuing to
properly scan all real nodes in distance order, was going to
require a nasty blob of zonelist and node distance munging.
The following approach has no new dependency on node distances or
zone sorting.
See comment in the patch below for a description of what it actually does.
Technical details of note (or controversy):
- See the use of "zlc_active" and "did_zlc_setup" below, to delay
adding any work for this new mechanism until we've looked at the
first zone in zonelist. I figured the odds of the first zone
having the memory we needed were high enough that we should just
look there, first, then get fancy only if we need to keep looking.
- Some odd hackery was needed to add items to struct zonelist, while
not tripping up the custom zonelists built by the mm/mempolicy.c
code for MPOL_BIND. My usual wordy comments below explain this.
Search for "MPOL_BIND".
- Some per-node data in the struct zonelist is now modified frequently,
with no locking. Multiple CPU cores on a node could hit and mangle
this data. The theory is that this is just performance hint data,
and the memory allocator will work just fine despite any such mangling.
The fields at risk are the struct 'zonelist_cache' fields 'fullzones'
(a bitmask) and 'last_full_zap' (unsigned long jiffies). It should
all be self correcting after at most a one second delay.
- This still does a linear scan of the same lengths as before. All
I've optimized is making the scan faster, not algorithmically
shorter. It is now able to scan a compact array of 'unsigned
short' in the case of many full nodes, so one cache line should
cover quite a few nodes, rather than each node hitting another
one or two new and distinct cache lines.
- If both Andi and Nick don't find this too complicated, I will be
(pleasantly) flabbergasted.
- I removed the comment claiming we only use one cachline's worth of
zonelist. We seem, at least in the fake numa case, to have put the
lie to that claim.
- I pay no attention to the various watermarks and such in this performance
hint. A node could be marked full for one watermark, and then skipped
over when searching for a page using a different watermark. I think
that's actually quite ok, as it will tend to slightly increase the
spreading of memory over other nodes, away from a memory stressed node.
===============
Performance - some benchmark results and analysis:
This benchmark runs a memory hog program that uses multiple
threads to touch alot of memory as quickly as it can.
Multiple runs were made, touching 12, 38, 64 or 90 GBytes out of
the total 96 GBytes on the system, and using 1, 19, 37, or 55
threads (on a 56 CPU system.) System, user and real (elapsed)
timings were recorded for each run, shown in units of seconds,
in the table below.
Two kernels were tested - 2.6.18-mm3 and the same kernel with
this zonelist caching patch added. The table also shows the
percentage improvement the zonelist caching sys time is over
(lower than) the stock *-mm kernel.
number 2.6.18-mm3 zonelist-cache delta (< 0 good) percent
GBs N ------------ -------------- ---------------- systime
mem threads sys user real sys user real sys user real better
12 1 153 24 177 151 24 176 -2 0 -1 1%
12 19 99 22 8 99 22 8 0 0 0 0%
12 37 111 25 6 112 25 6 1 0 0 -0%
12 55 115 25 5 110 23 5 -5 -2 0 4%
38 1 502 74 576 497 73 570 -5 -1 -6 0%
38 19 426 78 48 373 76 39 -53 -2 -9 12%
38 37 544 83 36 547 82 36 3 -1 0 -0%
38 55 501 77 23 511 80 24 10 3 1 -1%
64 1 917 125 1042 890 124 1014 -27 -1 -28 2%
64 19 1118 138 119 965 141 103 -153 3 -16 13%
64 37 1202 151 94 1136 150 81 -66 -1 -13 5%
64 55 1118 141 61 1072 140 58 -46 -1 -3 4%
90 1 1342 177 1519 1275 174 1450 -67 -3 -69 4%
90 19 2392 199 192 2116 189 176 -276 -10 -16 11%
90 37 3313 238 175 2972 225 145 -341 -13 -30 10%
90 55 1948 210 104 1843 213 100 -105 3 -4 5%
Notes:
1) This test ran a memory hog program that started a specified number N of
threads, and had each thread allocate and touch 1/N'th of
the total memory to be used in the test run in a single loop,
writing a constant word to memory, one store every 4096 bytes.
Watching this test during some earlier trial runs, I would see
each of these threads sit down on one CPU and stay there, for
the remainder of the pass, a different CPU for each thread.
2) The 'real' column is not comparable to the 'sys' or 'user' columns.
The 'real' column is seconds wall clock time elapsed, from beginning
to end of that test pass. The 'sys' and 'user' columns are total
CPU seconds spent on that test pass. For a 19 thread test run,
for example, the sum of 'sys' and 'user' could be up to 19 times the
number of 'real' elapsed wall clock seconds.
3) Tests were run on a fresh, single-user boot, to minimize the amount
of memory already in use at the start of the test, and to minimize
the amount of background activity that might interfere.
4) Tests were done on a 56 CPU, 28 Node system with 96 GBytes of RAM.
5) Notice that the 'real' time gets large for the single thread runs, even
though the measured 'sys' and 'user' times are modest. I'm not sure what
that means - probably something to do with it being slow for one thread to
be accessing memory along ways away. Perhaps the fake numa system, running
ostensibly the same workload, would not show this substantial degradation
of 'real' time for one thread on many nodes -- lets hope not.
6) The high thread count passes (one thread per CPU - on 55 of 56 CPUs)
ran quite efficiently, as one might expect. Each pair of threads needed
to allocate and touch the memory on the node the two threads shared, a
pleasantly parallizable workload.
7) The intermediate thread count passes, when asking for alot of memory forcing
them to go to a few neighboring nodes, improved the most with this zonelist
caching patch.
Conclusions:
* This zonelist cache patch probably makes little difference one way or the
other for most workloads on real numa hardware, if those workloads avoid
heavy off node allocations.
* For memory intensive workloads requiring substantial off-node allocations
on real numa hardware, this patch improves both kernel and elapsed timings
up to ten per-cent.
* For fake numa systems, I'm optimistic, but will have to leave that up to
Rohit Seth to actually test (once I get him a 2.6.18 backport.)
Signed-off-by: Paul Jackson <pj@sgi.com>
Cc: Rohit Seth <rohitseth@google.com>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Cc: David Rientjes <rientjes@cs.washington.edu>
Cc: Paul Menage <menage@google.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-06 21:31:48 -07:00
|
|
|
}
|
2010-01-04 23:34:51 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize the boot_pagesets that are going to be used
|
|
|
|
* for bootstrapping processors. The real pagesets for
|
|
|
|
* each zone will be allocated later when the per cpu
|
|
|
|
* allocator is available.
|
|
|
|
*
|
|
|
|
* boot_pagesets are used also for bootstrapping offline
|
|
|
|
* cpus if the system is already booted because the pagesets
|
|
|
|
* are needed to initialize allocators on a specific cpu too.
|
|
|
|
* F.e. the percpu allocator needs the page allocator which
|
|
|
|
* needs the percpu allocator in order to allocate its pagesets
|
|
|
|
* (a chicken-egg dilemma).
|
|
|
|
*/
|
2010-05-26 15:45:00 -06:00
|
|
|
for_each_possible_cpu(cpu) {
|
2010-01-04 23:34:51 -07:00
|
|
|
setup_pageset(&per_cpu(boot_pageset, cpu), 0);
|
|
|
|
|
2010-05-26 15:45:00 -06:00
|
|
|
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
|
|
|
|
/*
|
|
|
|
* We now know the "local memory node" for each node--
|
|
|
|
* i.e., the node of the first zone in the generic zonelist.
|
|
|
|
* Set up numa_mem percpu variable for on-line cpus. During
|
|
|
|
* boot, only the boot cpu should be on-line; we'll init the
|
|
|
|
* secondary cpus' numa_mem as they come on-line. During
|
|
|
|
* node/memory hotplug, we'll fixup all on-line cpus.
|
|
|
|
*/
|
|
|
|
if (cpu_online(cpu))
|
|
|
|
set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2006-06-23 03:03:11 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-02-12 16:00:06 -07:00
|
|
|
static noinline void __init
|
|
|
|
build_all_zonelists_init(void)
|
|
|
|
{
|
|
|
|
__build_all_zonelists(NULL);
|
|
|
|
mminit_verify_zonelist();
|
|
|
|
cpuset_init_current_mems_allowed();
|
|
|
|
}
|
|
|
|
|
2010-05-24 15:32:52 -06:00
|
|
|
/*
|
|
|
|
* Called with zonelists_mutex held always
|
|
|
|
* unless system_state == SYSTEM_BOOTING.
|
2015-02-12 16:00:06 -07:00
|
|
|
*
|
|
|
|
* __ref due to (1) call of __meminit annotated setup_zone_pageset
|
|
|
|
* [we're only called with non-NULL zone through __meminit paths] and
|
|
|
|
* (2) call of __init annotated helper build_all_zonelists_init
|
|
|
|
* [protected by SYSTEM_BOOTING].
|
2010-05-24 15:32:52 -06:00
|
|
|
*/
|
2012-07-31 17:43:28 -06:00
|
|
|
void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone)
|
2006-06-23 03:03:11 -06:00
|
|
|
{
|
2007-07-16 00:38:01 -06:00
|
|
|
set_zonelist_order();
|
|
|
|
|
2006-06-23 03:03:11 -06:00
|
|
|
if (system_state == SYSTEM_BOOTING) {
|
2015-02-12 16:00:06 -07:00
|
|
|
build_all_zonelists_init();
|
2006-06-23 03:03:11 -06:00
|
|
|
} else {
|
2010-11-24 13:57:09 -07:00
|
|
|
#ifdef CONFIG_MEMORY_HOTPLUG
|
2012-07-31 17:43:28 -06:00
|
|
|
if (zone)
|
|
|
|
setup_zone_pageset(zone);
|
2010-11-24 13:57:09 -07:00
|
|
|
#endif
|
2013-07-03 16:01:36 -06:00
|
|
|
/* we have to stop all cpus to guarantee there is no user
|
|
|
|
of zonelist */
|
2012-07-31 17:43:28 -06:00
|
|
|
stop_machine(__build_all_zonelists, pgdat, NULL);
|
2006-06-23 03:03:11 -06:00
|
|
|
/* cpuset refresh routine should be here */
|
|
|
|
}
|
2006-06-23 03:03:47 -06:00
|
|
|
vm_total_pages = nr_free_pagecache_pages();
|
2007-10-16 02:25:54 -06:00
|
|
|
/*
|
|
|
|
* Disable grouping by mobility if the number of pages in the
|
|
|
|
* system is too low to allow the mechanism to work. It would be
|
|
|
|
* more accurate, but expensive to check per-zone. This check is
|
|
|
|
* made on memory-hotadd so a system can start with mobility
|
|
|
|
* disabled and enable it later
|
|
|
|
*/
|
2007-10-16 02:26:01 -06:00
|
|
|
if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
|
2007-10-16 02:25:54 -06:00
|
|
|
page_group_by_mobility_disabled = 1;
|
|
|
|
else
|
|
|
|
page_group_by_mobility_disabled = 0;
|
|
|
|
|
2016-03-17 15:19:47 -06:00
|
|
|
pr_info("Built %i zonelists in %s order, mobility grouping %s. Total pages: %ld\n",
|
|
|
|
nr_online_nodes,
|
|
|
|
zonelist_order_name[current_zonelist_order],
|
|
|
|
page_group_by_mobility_disabled ? "off" : "on",
|
|
|
|
vm_total_pages);
|
2007-07-16 00:38:01 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
2014-12-10 16:42:53 -07:00
|
|
|
pr_info("Policy zone: %s\n", zone_names[policy_zone]);
|
2007-07-16 00:38:01 -06:00
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Helper functions to size the waitqueue hash table.
|
|
|
|
* Essentially these want to choose hash table sizes sufficiently
|
|
|
|
* large so that collisions trying to wait on pages are rare.
|
|
|
|
* But in fact, the number of active page waitqueues on typical
|
|
|
|
* systems is ridiculously low, less than 200. So this is even
|
|
|
|
* conservative, even though it seems large.
|
|
|
|
*
|
|
|
|
* The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
|
|
|
|
* waitqueues, i.e. the size of the waitq table given the number of pages.
|
|
|
|
*/
|
|
|
|
#define PAGES_PER_WAITQUEUE 256
|
|
|
|
|
2006-06-23 03:03:10 -06:00
|
|
|
#ifndef CONFIG_MEMORY_HOTPLUG
|
2006-06-23 03:03:08 -06:00
|
|
|
static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
unsigned long size = 1;
|
|
|
|
|
|
|
|
pages /= PAGES_PER_WAITQUEUE;
|
|
|
|
|
|
|
|
while (size < pages)
|
|
|
|
size <<= 1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Once we have dozens or even hundreds of threads sleeping
|
|
|
|
* on IO we've got bigger problems than wait queue collision.
|
|
|
|
* Limit the size of the wait table to a reasonable size.
|
|
|
|
*/
|
|
|
|
size = min(size, 4096UL);
|
|
|
|
|
|
|
|
return max(size, 4UL);
|
|
|
|
}
|
2006-06-23 03:03:10 -06:00
|
|
|
#else
|
|
|
|
/*
|
|
|
|
* A zone's size might be changed by hot-add, so it is not possible to determine
|
|
|
|
* a suitable size for its wait_table. So we use the maximum size now.
|
|
|
|
*
|
|
|
|
* The max wait table size = 4096 x sizeof(wait_queue_head_t). ie:
|
|
|
|
*
|
|
|
|
* i386 (preemption config) : 4096 x 16 = 64Kbyte.
|
|
|
|
* ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
|
|
|
|
* ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte.
|
|
|
|
*
|
|
|
|
* The maximum entries are prepared when a zone's memory is (512K + 256) pages
|
|
|
|
* or more by the traditional way. (See above). It equals:
|
|
|
|
*
|
|
|
|
* i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte.
|
|
|
|
* ia64(16K page size) : = ( 8G + 4M)byte.
|
|
|
|
* powerpc (64K page size) : = (32G +16M)byte.
|
|
|
|
*/
|
|
|
|
static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
|
|
|
|
{
|
|
|
|
return 4096UL;
|
|
|
|
}
|
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This is an integer logarithm so that shifts can be used later
|
|
|
|
* to extract the more random high bits from the multiplicative
|
|
|
|
* hash function before the remainder is taken.
|
|
|
|
*/
|
|
|
|
static inline unsigned long wait_table_bits(unsigned long size)
|
|
|
|
{
|
|
|
|
return ffz(~size);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initially all pages are reserved - free ones are freed
|
|
|
|
* up by free_all_bootmem() once the early boot process is
|
|
|
|
* done. Non-atomic initialization, single-pass.
|
|
|
|
*/
|
2006-01-16 23:03:44 -07:00
|
|
|
void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
|
2007-01-11 00:15:30 -07:00
|
|
|
unsigned long start_pfn, enum memmap_context context)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2016-01-15 17:56:22 -07:00
|
|
|
struct vmem_altmap *altmap = to_vmem_altmap(__pfn_to_phys(start_pfn));
|
2005-06-23 01:08:00 -06:00
|
|
|
unsigned long end_pfn = start_pfn + size;
|
2016-01-15 17:56:22 -07:00
|
|
|
pg_data_t *pgdat = NODE_DATA(nid);
|
2005-06-23 01:08:00 -06:00
|
|
|
unsigned long pfn;
|
2015-06-30 15:57:02 -06:00
|
|
|
unsigned long nr_initialised = 0;
|
2016-03-15 15:55:22 -06:00
|
|
|
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
|
|
|
|
struct memblock_region *r = NULL, *tmp;
|
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-01-06 15:40:09 -07:00
|
|
|
if (highest_memmap_pfn < end_pfn - 1)
|
|
|
|
highest_memmap_pfn = end_pfn - 1;
|
|
|
|
|
2016-01-15 17:56:22 -07:00
|
|
|
/*
|
|
|
|
* Honor reservation requested by the driver for this ZONE_DEVICE
|
|
|
|
* memory
|
|
|
|
*/
|
|
|
|
if (altmap && start_pfn == altmap->base_pfn)
|
|
|
|
start_pfn += altmap->reserve;
|
|
|
|
|
2006-01-12 02:05:24 -07:00
|
|
|
for (pfn = start_pfn; pfn < end_pfn; pfn++) {
|
2007-01-11 00:15:30 -07:00
|
|
|
/*
|
2016-03-15 15:55:25 -06:00
|
|
|
* There can be holes in boot-time mem_map[]s handed to this
|
|
|
|
* function. They do not exist on hotplugged memory.
|
2007-01-11 00:15:30 -07:00
|
|
|
*/
|
2016-03-15 15:55:25 -06:00
|
|
|
if (context != MEMMAP_EARLY)
|
|
|
|
goto not_early;
|
|
|
|
|
|
|
|
if (!early_pfn_valid(pfn))
|
|
|
|
continue;
|
|
|
|
if (!early_pfn_in_nid(pfn, nid))
|
|
|
|
continue;
|
|
|
|
if (!update_defer_init(pgdat, pfn, end_pfn, &nr_initialised))
|
|
|
|
break;
|
2016-03-15 15:55:22 -06:00
|
|
|
|
|
|
|
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
|
2016-03-15 15:55:25 -06:00
|
|
|
/*
|
|
|
|
* If not mirrored_kernelcore and ZONE_MOVABLE exists, range
|
|
|
|
* from zone_movable_pfn[nid] to end of each node should be
|
|
|
|
* ZONE_MOVABLE not ZONE_NORMAL. skip it.
|
|
|
|
*/
|
|
|
|
if (!mirrored_kernelcore && zone_movable_pfn[nid])
|
|
|
|
if (zone == ZONE_NORMAL && pfn >= zone_movable_pfn[nid])
|
|
|
|
continue;
|
2016-03-15 15:55:22 -06:00
|
|
|
|
2016-03-15 15:55:25 -06:00
|
|
|
/*
|
|
|
|
* Check given memblock attribute by firmware which can affect
|
|
|
|
* kernel memory layout. If zone==ZONE_MOVABLE but memory is
|
|
|
|
* mirrored, it's an overlapped memmap init. skip it.
|
|
|
|
*/
|
|
|
|
if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
|
|
|
|
if (!r || pfn >= memblock_region_memory_end_pfn(r)) {
|
|
|
|
for_each_memblock(memory, tmp)
|
|
|
|
if (pfn < memblock_region_memory_end_pfn(tmp))
|
|
|
|
break;
|
|
|
|
r = tmp;
|
|
|
|
}
|
|
|
|
if (pfn >= memblock_region_memory_base_pfn(r) &&
|
|
|
|
memblock_is_mirror(r)) {
|
|
|
|
/* already initialized as NORMAL */
|
|
|
|
pfn = memblock_region_memory_end_pfn(r);
|
|
|
|
continue;
|
2016-03-15 15:55:22 -06:00
|
|
|
}
|
2007-01-11 00:15:30 -07:00
|
|
|
}
|
2016-03-15 15:55:25 -06:00
|
|
|
#endif
|
2015-06-30 15:57:20 -06:00
|
|
|
|
2016-03-15 15:55:25 -06:00
|
|
|
not_early:
|
2015-06-30 15:57:20 -06:00
|
|
|
/*
|
|
|
|
* Mark the block movable so that blocks are reserved for
|
|
|
|
* movable at startup. This will force kernel allocations
|
|
|
|
* to reserve their blocks rather than leaking throughout
|
|
|
|
* the address space during boot when many long-lived
|
2015-11-06 17:28:34 -07:00
|
|
|
* kernel allocations are made.
|
2015-06-30 15:57:20 -06:00
|
|
|
*
|
|
|
|
* bitmap is created for zone's valid pfn range. but memmap
|
|
|
|
* can be created for invalid pages (for alignment)
|
|
|
|
* check here not to call set_pageblock_migratetype() against
|
|
|
|
* pfn out of zone.
|
|
|
|
*/
|
|
|
|
if (!(pfn & (pageblock_nr_pages - 1))) {
|
|
|
|
struct page *page = pfn_to_page(pfn);
|
|
|
|
|
|
|
|
__init_single_page(page, pfn, zone, nid);
|
|
|
|
set_pageblock_migratetype(page, MIGRATE_MOVABLE);
|
|
|
|
} else {
|
|
|
|
__init_single_pfn(pfn, zone, nid);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-02-04 23:29:26 -07:00
|
|
|
static void __meminit zone_init_free_lists(struct zone *zone)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2014-06-04 17:10:21 -06:00
|
|
|
unsigned int order, t;
|
2007-10-16 02:25:48 -06:00
|
|
|
for_each_migratetype_order(order, t) {
|
|
|
|
INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
|
2005-04-16 16:20:36 -06:00
|
|
|
zone->free_area[order].nr_free = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef __HAVE_ARCH_MEMMAP_INIT
|
|
|
|
#define memmap_init(size, nid, zone, start_pfn) \
|
2007-01-11 00:15:30 -07:00
|
|
|
memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
|
2014-06-23 14:22:04 -06:00
|
|
|
static int zone_batchsize(struct zone *zone)
|
2005-06-21 18:14:47 -06:00
|
|
|
{
|
nommu: clamp zone_batchsize() to 0 under NOMMU conditions
Clamp zone_batchsize() to 0 under NOMMU conditions to stop
free_hot_cold_page() from queueing and batching frees.
The problem is that under NOMMU conditions it is really important to be
able to allocate large contiguous chunks of memory, but when munmap() or
exit_mmap() releases big stretches of memory, return of these to the buddy
allocator can be deferred, and when it does finally happen, it can be in
small chunks.
Whilst the fragmentation this incurs isn't so much of a problem under MMU
conditions as userspace VM is glued together from individual pages with
the aid of the MMU, it is a real problem if there isn't an MMU.
By clamping the page freeing queue size to 0, pages are returned to the
allocator immediately, and the buddy detector is more likely to be able to
glue them together into large chunks immediately, and fragmentation is
less likely to occur.
By disabling batching of frees, and by turning off the trimming of excess
space during boot, Coldfire can manage to boot.
Reported-by: Lanttor Guo <lanttor.guo@freescale.com>
Signed-off-by: David Howells <dhowells@redhat.com>
Tested-by: Lanttor Guo <lanttor.guo@freescale.com>
Cc: Greg Ungerer <gerg@snapgear.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-05-06 17:03:03 -06:00
|
|
|
#ifdef CONFIG_MMU
|
2005-06-21 18:14:47 -06:00
|
|
|
int batch;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The per-cpu-pages pools are set to around 1000th of the
|
2005-10-29 19:15:47 -06:00
|
|
|
* size of the zone. But no more than 1/2 of a meg.
|
2005-06-21 18:14:47 -06:00
|
|
|
*
|
|
|
|
* OK, so we don't know how big the cache is. So guess.
|
|
|
|
*/
|
2013-02-22 17:33:52 -07:00
|
|
|
batch = zone->managed_pages / 1024;
|
2005-10-29 19:15:47 -06:00
|
|
|
if (batch * PAGE_SIZE > 512 * 1024)
|
|
|
|
batch = (512 * 1024) / PAGE_SIZE;
|
2005-06-21 18:14:47 -06:00
|
|
|
batch /= 4; /* We effectively *= 4 below */
|
|
|
|
if (batch < 1)
|
|
|
|
batch = 1;
|
|
|
|
|
|
|
|
/*
|
2005-12-03 19:55:25 -07:00
|
|
|
* Clamp the batch to a 2^n - 1 value. Having a power
|
|
|
|
* of 2 value was found to be more likely to have
|
|
|
|
* suboptimal cache aliasing properties in some cases.
|
2005-06-21 18:14:47 -06:00
|
|
|
*
|
2005-12-03 19:55:25 -07:00
|
|
|
* For example if 2 tasks are alternately allocating
|
|
|
|
* batches of pages, one task can end up with a lot
|
|
|
|
* of pages of one half of the possible page colors
|
|
|
|
* and the other with pages of the other colors.
|
2005-06-21 18:14:47 -06:00
|
|
|
*/
|
2009-05-06 17:03:02 -06:00
|
|
|
batch = rounddown_pow_of_two(batch + batch/2) - 1;
|
2005-10-29 19:15:47 -06:00
|
|
|
|
2005-06-21 18:14:47 -06:00
|
|
|
return batch;
|
nommu: clamp zone_batchsize() to 0 under NOMMU conditions
Clamp zone_batchsize() to 0 under NOMMU conditions to stop
free_hot_cold_page() from queueing and batching frees.
The problem is that under NOMMU conditions it is really important to be
able to allocate large contiguous chunks of memory, but when munmap() or
exit_mmap() releases big stretches of memory, return of these to the buddy
allocator can be deferred, and when it does finally happen, it can be in
small chunks.
Whilst the fragmentation this incurs isn't so much of a problem under MMU
conditions as userspace VM is glued together from individual pages with
the aid of the MMU, it is a real problem if there isn't an MMU.
By clamping the page freeing queue size to 0, pages are returned to the
allocator immediately, and the buddy detector is more likely to be able to
glue them together into large chunks immediately, and fragmentation is
less likely to occur.
By disabling batching of frees, and by turning off the trimming of excess
space during boot, Coldfire can manage to boot.
Reported-by: Lanttor Guo <lanttor.guo@freescale.com>
Signed-off-by: David Howells <dhowells@redhat.com>
Tested-by: Lanttor Guo <lanttor.guo@freescale.com>
Cc: Greg Ungerer <gerg@snapgear.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-05-06 17:03:03 -06:00
|
|
|
|
|
|
|
#else
|
|
|
|
/* The deferral and batching of frees should be suppressed under NOMMU
|
|
|
|
* conditions.
|
|
|
|
*
|
|
|
|
* The problem is that NOMMU needs to be able to allocate large chunks
|
|
|
|
* of contiguous memory as there's no hardware page translation to
|
|
|
|
* assemble apparent contiguous memory from discontiguous pages.
|
|
|
|
*
|
|
|
|
* Queueing large contiguous runs of pages for batching, however,
|
|
|
|
* causes the pages to actually be freed in smaller chunks. As there
|
|
|
|
* can be a significant delay between the individual batches being
|
|
|
|
* recycled, this leads to the once large chunks of space being
|
|
|
|
* fragmented and becoming unavailable for high-order allocations.
|
|
|
|
*/
|
|
|
|
return 0;
|
|
|
|
#endif
|
2005-06-21 18:14:47 -06:00
|
|
|
}
|
|
|
|
|
2013-07-03 16:01:31 -06:00
|
|
|
/*
|
|
|
|
* pcp->high and pcp->batch values are related and dependent on one another:
|
|
|
|
* ->batch must never be higher then ->high.
|
|
|
|
* The following function updates them in a safe manner without read side
|
|
|
|
* locking.
|
|
|
|
*
|
|
|
|
* Any new users of pcp->batch and pcp->high should ensure they can cope with
|
|
|
|
* those fields changing asynchronously (acording the the above rule).
|
|
|
|
*
|
|
|
|
* mutex_is_locked(&pcp_batch_high_lock) required when calling this function
|
|
|
|
* outside of boot time (or some other assurance that no concurrent updaters
|
|
|
|
* exist).
|
|
|
|
*/
|
|
|
|
static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
|
|
|
|
unsigned long batch)
|
|
|
|
{
|
|
|
|
/* start with a fail safe value for batch */
|
|
|
|
pcp->batch = 1;
|
|
|
|
smp_wmb();
|
|
|
|
|
|
|
|
/* Update high, then batch, in order */
|
|
|
|
pcp->high = high;
|
|
|
|
smp_wmb();
|
|
|
|
|
|
|
|
pcp->batch = batch;
|
|
|
|
}
|
|
|
|
|
2013-07-03 16:01:40 -06:00
|
|
|
/* a companion to pageset_set_high() */
|
mm/page_alloc: factor out setting of pcp->high and pcp->batch
"Problems" with the current code:
1: there is a lack of synchronization in setting ->high and ->batch in
percpu_pagelist_fraction_sysctl_handler()
2: stop_machine() in zone_pcp_update() is unnecissary.
3: zone_pcp_update() does not consider the case where
percpu_pagelist_fraction is non-zero
To fix:
1: add memory barriers, a safe ->batch value, an update side mutex when
updating ->high and ->batch, and use ACCESS_ONCE() for ->batch users
that expect a stable value.
2: avoid draining pages in zone_pcp_update(), rely upon the memory
barriers added to fix #1
3: factor out quite a few functions, and then call the appropriate one.
Note that it results in a change to the behavior of zone_pcp_update(),
which is used by memory_hotplug. I'm rather certain that I've diserned
(and preserved) the essential behavior (changing ->high and ->batch), and
only eliminated unneeded actions (draining the per cpu pages), but this
may not be the case.
Further note that the draining of pages that previously took place in
zone_pcp_update() occured after repeated draining when attempting to
offline a page, and after the offline has "succeeded". It appears that
the draining was added to zone_pcp_update() to avoid refactoring
setup_pageset() into 2 funtions.
This patch:
Creates pageset_set_batch() for use in setup_pageset().
pageset_set_batch() imitates the functionality of
setup_pagelist_highmark(), but uses the boot time
(percpu_pagelist_fraction == 0) calculations for determining ->high based
on ->batch.
Signed-off-by: Cody P Schafer <cody@linux.vnet.ibm.com>
Cc: Gilad Ben-Yossef <gilad@benyossef.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@gmail.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Pekka Enberg <penberg@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-07-03 16:01:28 -06:00
|
|
|
static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
|
|
|
|
{
|
2013-07-03 16:01:31 -06:00
|
|
|
pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
|
mm/page_alloc: factor out setting of pcp->high and pcp->batch
"Problems" with the current code:
1: there is a lack of synchronization in setting ->high and ->batch in
percpu_pagelist_fraction_sysctl_handler()
2: stop_machine() in zone_pcp_update() is unnecissary.
3: zone_pcp_update() does not consider the case where
percpu_pagelist_fraction is non-zero
To fix:
1: add memory barriers, a safe ->batch value, an update side mutex when
updating ->high and ->batch, and use ACCESS_ONCE() for ->batch users
that expect a stable value.
2: avoid draining pages in zone_pcp_update(), rely upon the memory
barriers added to fix #1
3: factor out quite a few functions, and then call the appropriate one.
Note that it results in a change to the behavior of zone_pcp_update(),
which is used by memory_hotplug. I'm rather certain that I've diserned
(and preserved) the essential behavior (changing ->high and ->batch), and
only eliminated unneeded actions (draining the per cpu pages), but this
may not be the case.
Further note that the draining of pages that previously took place in
zone_pcp_update() occured after repeated draining when attempting to
offline a page, and after the offline has "succeeded". It appears that
the draining was added to zone_pcp_update() to avoid refactoring
setup_pageset() into 2 funtions.
This patch:
Creates pageset_set_batch() for use in setup_pageset().
pageset_set_batch() imitates the functionality of
setup_pagelist_highmark(), but uses the boot time
(percpu_pagelist_fraction == 0) calculations for determining ->high based
on ->batch.
Signed-off-by: Cody P Schafer <cody@linux.vnet.ibm.com>
Cc: Gilad Ben-Yossef <gilad@benyossef.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@gmail.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Pekka Enberg <penberg@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-07-03 16:01:28 -06:00
|
|
|
}
|
|
|
|
|
2013-07-03 16:01:35 -06:00
|
|
|
static void pageset_init(struct per_cpu_pageset *p)
|
2005-06-21 18:15:00 -06:00
|
|
|
{
|
|
|
|
struct per_cpu_pages *pcp;
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
int migratetype;
|
2005-06-21 18:15:00 -06:00
|
|
|
|
2005-10-26 02:58:59 -06:00
|
|
|
memset(p, 0, sizeof(*p));
|
|
|
|
|
2008-02-04 23:29:19 -07:00
|
|
|
pcp = &p->pcp;
|
2005-06-21 18:15:00 -06:00
|
|
|
pcp->count = 0;
|
page-allocator: split per-cpu list into one-list-per-migrate-type
The following two patches remove searching in the page allocator fast-path
by maintaining multiple free-lists in the per-cpu structure. At the time
the search was introduced, increasing the per-cpu structures would waste a
lot of memory as per-cpu structures were statically allocated at
compile-time. This is no longer the case.
The patches are as follows. They are based on mmotm-2009-08-27.
Patch 1 adds multiple lists to struct per_cpu_pages, one per
migratetype that can be stored on the PCP lists.
Patch 2 notes that the pcpu drain path check empty lists multiple times. The
patch reduces the number of checks by maintaining a count of free
lists encountered. Lists containing pages will then free multiple
pages in batch
The patches were tested with kernbench, netperf udp/tcp, hackbench and
sysbench. The netperf tests were not bound to any CPU in particular and
were run such that the results should be 99% confidence that the reported
results are within 1% of the estimated mean. sysbench was run with a
postgres background and read-only tests. Similar to netperf, it was run
multiple times so that it's 99% confidence results are within 1%. The
patches were tested on x86, x86-64 and ppc64 as
x86: Intel Pentium D 3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.34% to 2.28% gain
netperf-tcp - 0.45% to 1.22% gain
hackbench - Small variances, very close to noise
sysbench - Very small gains
x86-64: AMD Phenom 9950 1.3GHz with 8G RAM (no-brand machine)
kernbench - No significant difference, variance well within noise
netperf-udp - 1.83% to 10.42% gains
netperf-tcp - No conclusive until buffer >= PAGE_SIZE
4096 +15.83%
8192 + 0.34% (not significant)
16384 + 1%
hackbench - Small gains, very close to noise
sysbench - 0.79% to 1.6% gain
ppc64: PPC970MP 2.5GHz with 10GB RAM (it's a terrasoft powerstation)
kernbench - No significant difference, variance well within noise
netperf-udp - 2-3% gain for almost all buffer sizes tested
netperf-tcp - losses on small buffers, gains on larger buffers
possibly indicates some bad caching effect.
hackbench - No significant difference
sysbench - 2-4% gain
This patch:
Currently the per-cpu page allocator searches the PCP list for pages of
the correct migrate-type to reduce the possibility of pages being
inappropriate placed from a fragmentation perspective. This search is
potentially expensive in a fast-path and undesirable. Splitting the
per-cpu list into multiple lists increases the size of a per-cpu structure
and this was potentially a major problem at the time the search was
introduced. These problem has been mitigated as now only the necessary
number of structures is allocated for the running system.
This patch replaces a list search in the per-cpu allocator with one list
per migrate type. The potential snag with this approach is when bulk
freeing pages. We round-robin free pages based on migrate type which has
little bearing on the cache hotness of the page and potentially checks
empty lists repeatedly in the event the majority of PCP pages are of one
type.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-21 18:03:19 -06:00
|
|
|
for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
|
|
|
|
INIT_LIST_HEAD(&pcp->lists[migratetype]);
|
2005-06-21 18:15:00 -06:00
|
|
|
}
|
|
|
|
|
2013-07-03 16:01:35 -06:00
|
|
|
static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
|
|
|
|
{
|
|
|
|
pageset_init(p);
|
|
|
|
pageset_set_batch(p, batch);
|
|
|
|
}
|
|
|
|
|
2006-01-08 02:00:40 -07:00
|
|
|
/*
|
2013-07-03 16:01:40 -06:00
|
|
|
* pageset_set_high() sets the high water mark for hot per_cpu_pagelist
|
2006-01-08 02:00:40 -07:00
|
|
|
* to the value high for the pageset p.
|
|
|
|
*/
|
2013-07-03 16:01:40 -06:00
|
|
|
static void pageset_set_high(struct per_cpu_pageset *p,
|
2006-01-08 02:00:40 -07:00
|
|
|
unsigned long high)
|
|
|
|
{
|
2013-07-03 16:01:31 -06:00
|
|
|
unsigned long batch = max(1UL, high / 4);
|
|
|
|
if ((high / 4) > (PAGE_SHIFT * 8))
|
|
|
|
batch = PAGE_SHIFT * 8;
|
2006-01-08 02:00:40 -07:00
|
|
|
|
2013-07-03 16:01:31 -06:00
|
|
|
pageset_update(&p->pcp, high, batch);
|
2006-01-08 02:00:40 -07:00
|
|
|
}
|
|
|
|
|
2014-06-23 14:22:04 -06:00
|
|
|
static void pageset_set_high_and_batch(struct zone *zone,
|
|
|
|
struct per_cpu_pageset *pcp)
|
2013-07-03 16:01:38 -06:00
|
|
|
{
|
|
|
|
if (percpu_pagelist_fraction)
|
2013-07-03 16:01:40 -06:00
|
|
|
pageset_set_high(pcp,
|
2013-07-03 16:01:38 -06:00
|
|
|
(zone->managed_pages /
|
|
|
|
percpu_pagelist_fraction));
|
|
|
|
else
|
|
|
|
pageset_set_batch(pcp, zone_batchsize(zone));
|
|
|
|
}
|
|
|
|
|
2013-07-03 16:01:41 -06:00
|
|
|
static void __meminit zone_pageset_init(struct zone *zone, int cpu)
|
|
|
|
{
|
|
|
|
struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
|
|
|
|
|
|
|
|
pageset_init(pcp);
|
|
|
|
pageset_set_high_and_batch(zone, pcp);
|
|
|
|
}
|
|
|
|
|
2012-07-31 17:43:35 -06:00
|
|
|
static void __meminit setup_zone_pageset(struct zone *zone)
|
2010-05-24 15:32:49 -06:00
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
zone->pageset = alloc_percpu(struct per_cpu_pageset);
|
2013-07-03 16:01:38 -06:00
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
zone_pageset_init(zone, cpu);
|
2010-05-24 15:32:49 -06:00
|
|
|
}
|
|
|
|
|
2005-06-21 18:15:00 -06:00
|
|
|
/*
|
2010-01-04 23:34:51 -07:00
|
|
|
* Allocate per cpu pagesets and initialize them.
|
|
|
|
* Before this call only boot pagesets were available.
|
2005-06-21 18:14:47 -06:00
|
|
|
*/
|
2010-01-04 23:34:51 -07:00
|
|
|
void __init setup_per_cpu_pageset(void)
|
2005-06-21 18:14:47 -06:00
|
|
|
{
|
mm: initialise per_cpu_nodestats for all online pgdats at boot
Paul Mackerras and Reza Arbab reported that machines with memoryless
nodes fail when vmstats are refreshed. Paul reported an oops as follows
Unable to handle kernel paging request for data at address 0xff7a10000
Faulting instruction address: 0xc000000000270cd0
Oops: Kernel access of bad area, sig: 11 [#1]
SMP NR_CPUS=2048 NUMA PowerNV
Modules linked in:
CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.7.0-kvm+ #118
task: c000000ff0680010 task.stack: c000000ff0704000
NIP: c000000000270cd0 LR: c000000000270ce8 CTR: 0000000000000000
REGS: c000000ff0707900 TRAP: 0300 Not tainted (4.7.0-kvm+)
MSR: 9000000102009033 <SF,HV,VEC,EE,ME,IR,DR,RI,LE,TM[E]> CR: 846b6824 XER: 20000000
CFAR: c000000000008768 DAR: 0000000ff7a10000 DSISR: 42000000 SOFTE: 1
NIP refresh_zone_stat_thresholds+0x80/0x240
LR refresh_zone_stat_thresholds+0x98/0x240
Call Trace:
refresh_zone_stat_thresholds+0xb8/0x240 (unreliable)
Both supplied potential fixes but one potentially misses checks and
another had redundant initialisations. This version initialises
per_cpu_nodestats on a per-pgdat basis instead of on a per-zone basis.
Link: http://lkml.kernel.org/r/20160804092404.GI2799@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Reported-by: Paul Mackerras <paulus@ozlabs.org>
Reported-by: Reza Arbab <arbab@linux.vnet.ibm.com>
Tested-by: Reza Arbab <arbab@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-08-04 16:31:49 -06:00
|
|
|
struct pglist_data *pgdat;
|
2010-01-04 23:34:51 -07:00
|
|
|
struct zone *zone;
|
2005-06-21 18:14:47 -06:00
|
|
|
|
2010-05-24 15:32:49 -06:00
|
|
|
for_each_populated_zone(zone)
|
|
|
|
setup_zone_pageset(zone);
|
mm: initialise per_cpu_nodestats for all online pgdats at boot
Paul Mackerras and Reza Arbab reported that machines with memoryless
nodes fail when vmstats are refreshed. Paul reported an oops as follows
Unable to handle kernel paging request for data at address 0xff7a10000
Faulting instruction address: 0xc000000000270cd0
Oops: Kernel access of bad area, sig: 11 [#1]
SMP NR_CPUS=2048 NUMA PowerNV
Modules linked in:
CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.7.0-kvm+ #118
task: c000000ff0680010 task.stack: c000000ff0704000
NIP: c000000000270cd0 LR: c000000000270ce8 CTR: 0000000000000000
REGS: c000000ff0707900 TRAP: 0300 Not tainted (4.7.0-kvm+)
MSR: 9000000102009033 <SF,HV,VEC,EE,ME,IR,DR,RI,LE,TM[E]> CR: 846b6824 XER: 20000000
CFAR: c000000000008768 DAR: 0000000ff7a10000 DSISR: 42000000 SOFTE: 1
NIP refresh_zone_stat_thresholds+0x80/0x240
LR refresh_zone_stat_thresholds+0x98/0x240
Call Trace:
refresh_zone_stat_thresholds+0xb8/0x240 (unreliable)
Both supplied potential fixes but one potentially misses checks and
another had redundant initialisations. This version initialises
per_cpu_nodestats on a per-pgdat basis instead of on a per-zone basis.
Link: http://lkml.kernel.org/r/20160804092404.GI2799@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Reported-by: Paul Mackerras <paulus@ozlabs.org>
Reported-by: Reza Arbab <arbab@linux.vnet.ibm.com>
Tested-by: Reza Arbab <arbab@linux.vnet.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-08-04 16:31:49 -06:00
|
|
|
|
|
|
|
for_each_online_pgdat(pgdat)
|
|
|
|
pgdat->per_cpu_nodestats =
|
|
|
|
alloc_percpu(struct per_cpu_nodestat);
|
2005-06-21 18:14:47 -06:00
|
|
|
}
|
|
|
|
|
2016-08-02 15:03:33 -06:00
|
|
|
static noinline __ref
|
2006-06-23 03:03:10 -06:00
|
|
|
int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
|
2005-10-29 19:16:50 -06:00
|
|
|
{
|
|
|
|
int i;
|
2006-06-23 03:03:10 -06:00
|
|
|
size_t alloc_size;
|
2005-10-29 19:16:50 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The per-page waitqueue mechanism uses hashed waitqueues
|
|
|
|
* per zone.
|
|
|
|
*/
|
2006-06-23 03:03:08 -06:00
|
|
|
zone->wait_table_hash_nr_entries =
|
|
|
|
wait_table_hash_nr_entries(zone_size_pages);
|
|
|
|
zone->wait_table_bits =
|
|
|
|
wait_table_bits(zone->wait_table_hash_nr_entries);
|
2006-06-23 03:03:10 -06:00
|
|
|
alloc_size = zone->wait_table_hash_nr_entries
|
|
|
|
* sizeof(wait_queue_head_t);
|
|
|
|
|
2008-05-23 14:04:52 -06:00
|
|
|
if (!slab_is_available()) {
|
2006-06-23 03:03:10 -06:00
|
|
|
zone->wait_table = (wait_queue_head_t *)
|
2014-01-21 16:50:25 -07:00
|
|
|
memblock_virt_alloc_node_nopanic(
|
|
|
|
alloc_size, zone->zone_pgdat->node_id);
|
2006-06-23 03:03:10 -06:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* This case means that a zone whose size was 0 gets new memory
|
|
|
|
* via memory hot-add.
|
|
|
|
* But it may be the case that a new node was hot-added. In
|
|
|
|
* this case vmalloc() will not be able to use this new node's
|
|
|
|
* memory - this wait_table must be initialized to use this new
|
|
|
|
* node itself as well.
|
|
|
|
* To use this new node's memory, further consideration will be
|
|
|
|
* necessary.
|
|
|
|
*/
|
2007-10-16 02:24:49 -06:00
|
|
|
zone->wait_table = vmalloc(alloc_size);
|
2006-06-23 03:03:10 -06:00
|
|
|
}
|
|
|
|
if (!zone->wait_table)
|
|
|
|
return -ENOMEM;
|
2005-10-29 19:16:50 -06:00
|
|
|
|
2013-09-11 15:20:34 -06:00
|
|
|
for (i = 0; i < zone->wait_table_hash_nr_entries; ++i)
|
2005-10-29 19:16:50 -06:00
|
|
|
init_waitqueue_head(zone->wait_table + i);
|
2006-06-23 03:03:10 -06:00
|
|
|
|
|
|
|
return 0;
|
2005-10-29 19:16:50 -06:00
|
|
|
}
|
|
|
|
|
2006-01-16 23:03:44 -07:00
|
|
|
static __meminit void zone_pcp_init(struct zone *zone)
|
2005-10-29 19:16:50 -06:00
|
|
|
{
|
2010-01-04 23:34:51 -07:00
|
|
|
/*
|
|
|
|
* per cpu subsystem is not up at this point. The following code
|
|
|
|
* relies on the ability of the linker to provide the
|
|
|
|
* offset of a (static) per cpu variable into the per cpu area.
|
|
|
|
*/
|
|
|
|
zone->pageset = &boot_pageset;
|
2005-10-29 19:16:50 -06:00
|
|
|
|
2013-11-12 16:07:20 -07:00
|
|
|
if (populated_zone(zone))
|
2010-01-04 23:34:51 -07:00
|
|
|
printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
|
|
|
|
zone->name, zone->present_pages,
|
|
|
|
zone_batchsize(zone));
|
2005-10-29 19:16:50 -06:00
|
|
|
}
|
|
|
|
|
2012-07-31 17:43:35 -06:00
|
|
|
int __meminit init_currently_empty_zone(struct zone *zone,
|
2006-06-23 03:03:10 -06:00
|
|
|
unsigned long zone_start_pfn,
|
2015-11-05 19:47:06 -07:00
|
|
|
unsigned long size)
|
2005-10-29 19:16:50 -06:00
|
|
|
{
|
|
|
|
struct pglist_data *pgdat = zone->zone_pgdat;
|
2006-06-23 03:03:10 -06:00
|
|
|
int ret;
|
|
|
|
ret = zone_wait_table_init(zone, size);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2005-10-29 19:16:50 -06:00
|
|
|
pgdat->nr_zones = zone_idx(zone) + 1;
|
|
|
|
|
|
|
|
zone->zone_start_pfn = zone_start_pfn;
|
|
|
|
|
2008-07-23 22:26:51 -06:00
|
|
|
mminit_dprintk(MMINIT_TRACE, "memmap_init",
|
|
|
|
"Initialising map node %d zone %lu pfns %lu -> %lu\n",
|
|
|
|
pgdat->node_id,
|
|
|
|
(unsigned long)zone_idx(zone),
|
|
|
|
zone_start_pfn, (zone_start_pfn + size));
|
|
|
|
|
2008-02-04 23:29:26 -07:00
|
|
|
zone_init_free_lists(zone);
|
2006-06-23 03:03:10 -06:00
|
|
|
|
|
|
|
return 0;
|
2005-10-29 19:16:50 -06:00
|
|
|
}
|
|
|
|
|
2011-12-08 11:22:09 -07:00
|
|
|
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
|
2015-06-30 15:56:55 -06:00
|
|
|
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
/*
|
|
|
|
* Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
|
|
|
|
*/
|
2015-06-30 15:56:55 -06:00
|
|
|
int __meminit __early_pfn_to_nid(unsigned long pfn,
|
|
|
|
struct mminit_pfnnid_cache *state)
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
{
|
2011-07-12 02:46:30 -06:00
|
|
|
unsigned long start_pfn, end_pfn;
|
2013-09-11 15:22:17 -06:00
|
|
|
int nid;
|
mm: speedup in __early_pfn_to_nid
When booting on a large memory system, the kernel spends considerable
time in memmap_init_zone() setting up memory zones. Analysis shows
significant time spent in __early_pfn_to_nid().
The routine memmap_init_zone() checks each PFN to verify the nid is
valid. __early_pfn_to_nid() sequentially scans the list of pfn ranges
to find the right range and returns the nid. This does not scale well.
On a 4 TB (single rack) system there are 308 memory ranges to scan. The
higher the PFN the more time spent sequentially spinning through memory
ranges.
Since memmap_init_zone() increments pfn, it will almost always be
looking for the same range as the previous pfn, so check that range
first. If it is in the same range, return that nid. If not, scan the
list as before.
A 4 TB (single rack) UV1 system takes 512 seconds to get through the
zone code. This performance optimization reduces the time by 189
seconds, a 36% improvement.
A 2 TB (single rack) UV2 system goes from 212.7 seconds to 99.8 seconds,
a 112.9 second (53%) reduction.
[akpm@linux-foundation.org: make the statics __meminitdata]
[akpm@linux-foundation.org: fix comment formatting]
[akpm@linux-foundation.org: fix ia64, per yinghai]
[akpm@linux-foundation.org: add missing semicolon, per Tony]
Signed-off-by: Russ Anderson <rja@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Tested-by: "Luck, Tony" <tony.luck@intel.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Lin Feng <linfeng@cn.fujitsu.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-04-29 16:07:59 -06:00
|
|
|
|
2015-06-30 15:56:55 -06:00
|
|
|
if (state->last_start <= pfn && pfn < state->last_end)
|
|
|
|
return state->last_nid;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
2013-09-11 15:22:17 -06:00
|
|
|
nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
|
|
|
|
if (nid != -1) {
|
2015-06-30 15:56:55 -06:00
|
|
|
state->last_start = start_pfn;
|
|
|
|
state->last_end = end_pfn;
|
|
|
|
state->last_nid = nid;
|
2013-09-11 15:22:17 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
return nid;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
|
|
|
#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
|
|
|
|
|
|
|
|
/**
|
2014-01-21 16:50:25 -07:00
|
|
|
* free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range
|
2006-10-04 03:15:25 -06:00
|
|
|
* @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
|
2014-01-21 16:50:25 -07:00
|
|
|
* @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*
|
2014-06-04 17:10:53 -06:00
|
|
|
* If an architecture guarantees that all ranges registered contain no holes
|
|
|
|
* and may be freed, this this function may be used instead of calling
|
|
|
|
* memblock_free_early_nid() manually.
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*/
|
2011-07-12 02:46:30 -06:00
|
|
|
void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
|
2011-02-26 05:05:43 -07:00
|
|
|
{
|
2011-07-12 02:46:30 -06:00
|
|
|
unsigned long start_pfn, end_pfn;
|
|
|
|
int i, this_nid;
|
2010-08-25 14:39:16 -06:00
|
|
|
|
2011-07-12 02:46:30 -06:00
|
|
|
for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
|
|
|
|
start_pfn = min(start_pfn, max_low_pfn);
|
|
|
|
end_pfn = min(end_pfn, max_low_pfn);
|
2010-08-25 14:39:16 -06:00
|
|
|
|
2011-07-12 02:46:30 -06:00
|
|
|
if (start_pfn < end_pfn)
|
2014-01-21 16:50:25 -07:00
|
|
|
memblock_free_early_nid(PFN_PHYS(start_pfn),
|
|
|
|
(end_pfn - start_pfn) << PAGE_SHIFT,
|
|
|
|
this_nid);
|
2010-08-25 14:39:16 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
/**
|
|
|
|
* sparse_memory_present_with_active_regions - Call memory_present for each active range
|
2006-10-04 03:15:25 -06:00
|
|
|
* @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*
|
2014-06-04 17:10:53 -06:00
|
|
|
* If an architecture guarantees that all ranges registered contain no holes and may
|
|
|
|
* be freed, this function may be used instead of calling memory_present() manually.
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*/
|
|
|
|
void __init sparse_memory_present_with_active_regions(int nid)
|
|
|
|
{
|
2011-07-12 02:46:30 -06:00
|
|
|
unsigned long start_pfn, end_pfn;
|
|
|
|
int i, this_nid;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
2011-07-12 02:46:30 -06:00
|
|
|
for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
|
|
|
|
memory_present(this_nid, start_pfn, end_pfn);
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* get_pfn_range_for_nid - Return the start and end page frames for a node
|
2006-10-04 03:15:25 -06:00
|
|
|
* @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
|
|
|
|
* @start_pfn: Passed by reference. On return, it will have the node start_pfn.
|
|
|
|
* @end_pfn: Passed by reference. On return, it will have the node end_pfn.
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*
|
|
|
|
* It returns the start and end page frame of a node based on information
|
2014-06-04 17:10:53 -06:00
|
|
|
* provided by memblock_set_node(). If called for a node
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
* with no available memory, a warning is printed and the start and end
|
2006-10-04 03:15:25 -06:00
|
|
|
* PFNs will be 0.
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*/
|
2007-05-08 01:23:07 -06:00
|
|
|
void __meminit get_pfn_range_for_nid(unsigned int nid,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
unsigned long *start_pfn, unsigned long *end_pfn)
|
|
|
|
{
|
2011-07-12 02:46:30 -06:00
|
|
|
unsigned long this_start_pfn, this_end_pfn;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
int i;
|
2011-07-12 02:46:30 -06:00
|
|
|
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*start_pfn = -1UL;
|
|
|
|
*end_pfn = 0;
|
|
|
|
|
2011-07-12 02:46:30 -06:00
|
|
|
for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
|
|
|
|
*start_pfn = min(*start_pfn, this_start_pfn);
|
|
|
|
*end_pfn = max(*end_pfn, this_end_pfn);
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
|
|
|
|
2007-10-16 02:25:37 -06:00
|
|
|
if (*start_pfn == -1UL)
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*start_pfn = 0;
|
|
|
|
}
|
|
|
|
|
2007-07-17 05:03:12 -06:00
|
|
|
/*
|
|
|
|
* This finds a zone that can be used for ZONE_MOVABLE pages. The
|
|
|
|
* assumption is made that zones within a node are ordered in monotonic
|
|
|
|
* increasing memory addresses so that the "highest" populated zone is used
|
|
|
|
*/
|
2008-07-23 22:28:12 -06:00
|
|
|
static void __init find_usable_zone_for_movable(void)
|
2007-07-17 05:03:12 -06:00
|
|
|
{
|
|
|
|
int zone_index;
|
|
|
|
for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
|
|
|
|
if (zone_index == ZONE_MOVABLE)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (arch_zone_highest_possible_pfn[zone_index] >
|
|
|
|
arch_zone_lowest_possible_pfn[zone_index])
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
VM_BUG_ON(zone_index == -1);
|
|
|
|
movable_zone = zone_index;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The zone ranges provided by the architecture do not include ZONE_MOVABLE
|
2011-03-30 19:57:33 -06:00
|
|
|
* because it is sized independent of architecture. Unlike the other zones,
|
2007-07-17 05:03:12 -06:00
|
|
|
* the starting point for ZONE_MOVABLE is not fixed. It may be different
|
|
|
|
* in each node depending on the size of each node and how evenly kernelcore
|
|
|
|
* is distributed. This helper function adjusts the zone ranges
|
|
|
|
* provided by the architecture for a given node by using the end of the
|
|
|
|
* highest usable zone for ZONE_MOVABLE. This preserves the assumption that
|
|
|
|
* zones within a node are in order of monotonic increases memory addresses
|
|
|
|
*/
|
2008-07-23 22:28:12 -06:00
|
|
|
static void __meminit adjust_zone_range_for_zone_movable(int nid,
|
2007-07-17 05:03:12 -06:00
|
|
|
unsigned long zone_type,
|
|
|
|
unsigned long node_start_pfn,
|
|
|
|
unsigned long node_end_pfn,
|
|
|
|
unsigned long *zone_start_pfn,
|
|
|
|
unsigned long *zone_end_pfn)
|
|
|
|
{
|
|
|
|
/* Only adjust if ZONE_MOVABLE is on this node */
|
|
|
|
if (zone_movable_pfn[nid]) {
|
|
|
|
/* Size ZONE_MOVABLE */
|
|
|
|
if (zone_type == ZONE_MOVABLE) {
|
|
|
|
*zone_start_pfn = zone_movable_pfn[nid];
|
|
|
|
*zone_end_pfn = min(node_end_pfn,
|
|
|
|
arch_zone_highest_possible_pfn[movable_zone]);
|
|
|
|
|
|
|
|
/* Check if this whole range is within ZONE_MOVABLE */
|
|
|
|
} else if (*zone_start_pfn >= zone_movable_pfn[nid])
|
|
|
|
*zone_start_pfn = *zone_end_pfn;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
/*
|
|
|
|
* Return the number of pages a zone spans in a node, including holes
|
|
|
|
* present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
|
|
|
|
*/
|
2007-07-16 00:38:20 -06:00
|
|
|
static unsigned long __meminit zone_spanned_pages_in_node(int nid,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
unsigned long zone_type,
|
2013-07-08 16:59:52 -06:00
|
|
|
unsigned long node_start_pfn,
|
|
|
|
unsigned long node_end_pfn,
|
2016-03-15 15:55:18 -06:00
|
|
|
unsigned long *zone_start_pfn,
|
|
|
|
unsigned long *zone_end_pfn,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
unsigned long *ignored)
|
|
|
|
{
|
2015-09-08 16:04:16 -06:00
|
|
|
/* When hotadd a new node from cpu_up(), the node should be empty */
|
2015-08-14 16:35:16 -06:00
|
|
|
if (!node_start_pfn && !node_end_pfn)
|
|
|
|
return 0;
|
|
|
|
|
2013-07-08 16:59:52 -06:00
|
|
|
/* Get the start and end of the zone */
|
2016-03-15 15:55:18 -06:00
|
|
|
*zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
|
|
|
|
*zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
|
2007-07-17 05:03:12 -06:00
|
|
|
adjust_zone_range_for_zone_movable(nid, zone_type,
|
|
|
|
node_start_pfn, node_end_pfn,
|
2016-03-15 15:55:18 -06:00
|
|
|
zone_start_pfn, zone_end_pfn);
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
|
|
|
/* Check that this node has pages within the zone's required range */
|
2016-03-15 15:55:18 -06:00
|
|
|
if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* Move the zone boundaries inside the node if necessary */
|
2016-03-15 15:55:18 -06:00
|
|
|
*zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
|
|
|
|
*zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
|
|
|
/* Return the spanned pages */
|
2016-03-15 15:55:18 -06:00
|
|
|
return *zone_end_pfn - *zone_start_pfn;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
|
2006-10-04 03:15:25 -06:00
|
|
|
* then all holes in the requested range will be accounted for.
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*/
|
x86: Fix checking of SRAT when node 0 ram is not from 0
Found one system that boot from socket1 instead of socket0, SRAT get rejected...
[ 0.000000] SRAT: Node 1 PXM 0 0-a0000
[ 0.000000] SRAT: Node 1 PXM 0 100000-80000000
[ 0.000000] SRAT: Node 1 PXM 0 100000000-2080000000
[ 0.000000] SRAT: Node 0 PXM 1 2080000000-4080000000
[ 0.000000] SRAT: Node 2 PXM 2 4080000000-6080000000
[ 0.000000] SRAT: Node 3 PXM 3 6080000000-8080000000
[ 0.000000] SRAT: Node 4 PXM 4 8080000000-a080000000
[ 0.000000] SRAT: Node 5 PXM 5 a080000000-c080000000
[ 0.000000] SRAT: Node 6 PXM 6 c080000000-e080000000
[ 0.000000] SRAT: Node 7 PXM 7 e080000000-10080000000
...
[ 0.000000] NUMA: Allocated memnodemap from 500000 - 701040
[ 0.000000] NUMA: Using 20 for the hash shift.
[ 0.000000] Adding active range (0, 0x2080000, 0x4080000) 0 entries of 3200 used
[ 0.000000] Adding active range (1, 0x0, 0x96) 1 entries of 3200 used
[ 0.000000] Adding active range (1, 0x100, 0x7f750) 2 entries of 3200 used
[ 0.000000] Adding active range (1, 0x100000, 0x2080000) 3 entries of 3200 used
[ 0.000000] Adding active range (2, 0x4080000, 0x6080000) 4 entries of 3200 used
[ 0.000000] Adding active range (3, 0x6080000, 0x8080000) 5 entries of 3200 used
[ 0.000000] Adding active range (4, 0x8080000, 0xa080000) 6 entries of 3200 used
[ 0.000000] Adding active range (5, 0xa080000, 0xc080000) 7 entries of 3200 used
[ 0.000000] Adding active range (6, 0xc080000, 0xe080000) 8 entries of 3200 used
[ 0.000000] Adding active range (7, 0xe080000, 0x10080000) 9 entries of 3200 used
[ 0.000000] SRAT: PXMs only cover 917504MB of your 1048566MB e820 RAM. Not used.
[ 0.000000] SRAT: SRAT not used.
the early_node_map is not sorted because node0 with non zero start come first.
so try to sort it right away after all regions are registered.
also fixs refression by 8716273c (x86: Export srat physical topology)
-v2: make it more solid to handle cross node case like node0 [0,4g), [8,12g) and node1 [4g, 8g), [12g, 16g)
-v3: update comments.
Reported-and-tested-by: Jens Axboe <jens.axboe@oracle.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
LKML-Reference: <4B2579D2.3010201@kernel.org>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2009-12-15 18:59:02 -07:00
|
|
|
unsigned long __meminit __absent_pages_in_range(int nid,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
unsigned long range_start_pfn,
|
|
|
|
unsigned long range_end_pfn)
|
|
|
|
{
|
2011-07-12 02:46:29 -06:00
|
|
|
unsigned long nr_absent = range_end_pfn - range_start_pfn;
|
|
|
|
unsigned long start_pfn, end_pfn;
|
|
|
|
int i;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
2011-07-12 02:46:29 -06:00
|
|
|
for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
|
|
|
|
start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
|
|
|
|
end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
|
|
|
|
nr_absent -= end_pfn - start_pfn;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
2011-07-12 02:46:29 -06:00
|
|
|
return nr_absent;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* absent_pages_in_range - Return number of page frames in holes within a range
|
|
|
|
* @start_pfn: The start PFN to start searching for holes
|
|
|
|
* @end_pfn: The end PFN to stop searching for holes
|
|
|
|
*
|
2006-10-04 03:15:25 -06:00
|
|
|
* It returns the number of pages frames in memory holes within a range.
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*/
|
|
|
|
unsigned long __init absent_pages_in_range(unsigned long start_pfn,
|
|
|
|
unsigned long end_pfn)
|
|
|
|
{
|
|
|
|
return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Return the number of page frames in holes in a zone on a node */
|
2007-07-16 00:38:20 -06:00
|
|
|
static unsigned long __meminit zone_absent_pages_in_node(int nid,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
unsigned long zone_type,
|
2013-07-08 16:59:52 -06:00
|
|
|
unsigned long node_start_pfn,
|
|
|
|
unsigned long node_end_pfn,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
unsigned long *ignored)
|
|
|
|
{
|
2011-07-12 02:46:29 -06:00
|
|
|
unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
|
|
|
|
unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
|
2006-09-27 02:49:58 -06:00
|
|
|
unsigned long zone_start_pfn, zone_end_pfn;
|
2016-03-15 15:55:22 -06:00
|
|
|
unsigned long nr_absent;
|
2006-09-27 02:49:58 -06:00
|
|
|
|
2015-09-08 16:04:16 -06:00
|
|
|
/* When hotadd a new node from cpu_up(), the node should be empty */
|
2015-08-14 16:35:16 -06:00
|
|
|
if (!node_start_pfn && !node_end_pfn)
|
|
|
|
return 0;
|
|
|
|
|
2011-07-12 02:46:29 -06:00
|
|
|
zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
|
|
|
|
zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
|
2006-09-27 02:49:58 -06:00
|
|
|
|
2007-07-17 05:03:12 -06:00
|
|
|
adjust_zone_range_for_zone_movable(nid, zone_type,
|
|
|
|
node_start_pfn, node_end_pfn,
|
|
|
|
&zone_start_pfn, &zone_end_pfn);
|
2016-03-15 15:55:22 -06:00
|
|
|
nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ZONE_MOVABLE handling.
|
|
|
|
* Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
|
|
|
|
* and vice versa.
|
|
|
|
*/
|
|
|
|
if (zone_movable_pfn[nid]) {
|
|
|
|
if (mirrored_kernelcore) {
|
|
|
|
unsigned long start_pfn, end_pfn;
|
|
|
|
struct memblock_region *r;
|
|
|
|
|
|
|
|
for_each_memblock(memory, r) {
|
|
|
|
start_pfn = clamp(memblock_region_memory_base_pfn(r),
|
|
|
|
zone_start_pfn, zone_end_pfn);
|
|
|
|
end_pfn = clamp(memblock_region_memory_end_pfn(r),
|
|
|
|
zone_start_pfn, zone_end_pfn);
|
|
|
|
|
|
|
|
if (zone_type == ZONE_MOVABLE &&
|
|
|
|
memblock_is_mirror(r))
|
|
|
|
nr_absent += end_pfn - start_pfn;
|
|
|
|
|
|
|
|
if (zone_type == ZONE_NORMAL &&
|
|
|
|
!memblock_is_mirror(r))
|
|
|
|
nr_absent += end_pfn - start_pfn;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (zone_type == ZONE_NORMAL)
|
|
|
|
nr_absent += node_end_pfn - zone_movable_pfn[nid];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nr_absent;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
2006-09-27 02:49:56 -06:00
|
|
|
|
2011-12-08 11:22:09 -07:00
|
|
|
#else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
|
2007-07-16 00:38:20 -06:00
|
|
|
static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
unsigned long zone_type,
|
2013-07-08 16:59:52 -06:00
|
|
|
unsigned long node_start_pfn,
|
|
|
|
unsigned long node_end_pfn,
|
2016-03-15 15:55:18 -06:00
|
|
|
unsigned long *zone_start_pfn,
|
|
|
|
unsigned long *zone_end_pfn,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
unsigned long *zones_size)
|
|
|
|
{
|
2016-03-15 15:55:18 -06:00
|
|
|
unsigned int zone;
|
|
|
|
|
|
|
|
*zone_start_pfn = node_start_pfn;
|
|
|
|
for (zone = 0; zone < zone_type; zone++)
|
|
|
|
*zone_start_pfn += zones_size[zone];
|
|
|
|
|
|
|
|
*zone_end_pfn = *zone_start_pfn + zones_size[zone_type];
|
|
|
|
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
return zones_size[zone_type];
|
|
|
|
}
|
|
|
|
|
2007-07-16 00:38:20 -06:00
|
|
|
static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
unsigned long zone_type,
|
2013-07-08 16:59:52 -06:00
|
|
|
unsigned long node_start_pfn,
|
|
|
|
unsigned long node_end_pfn,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
unsigned long *zholes_size)
|
|
|
|
{
|
|
|
|
if (!zholes_size)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return zholes_size[zone_type];
|
|
|
|
}
|
x86, ACPI, mm: Revert movablemem_map support
Tim found:
WARNING: at arch/x86/kernel/smpboot.c:324 topology_sane.isra.2+0x6f/0x80()
Hardware name: S2600CP
sched: CPU #1's llc-sibling CPU #0 is not on the same node! [node: 1 != 0]. Ignoring dependency.
smpboot: Booting Node 1, Processors #1
Modules linked in:
Pid: 0, comm: swapper/1 Not tainted 3.9.0-0-generic #1
Call Trace:
set_cpu_sibling_map+0x279/0x449
start_secondary+0x11d/0x1e5
Don Morris reproduced on a HP z620 workstation, and bisected it to
commit e8d195525809 ("acpi, memory-hotplug: parse SRAT before memblock
is ready")
It turns out movable_map has some problems, and it breaks several things
1. numa_init is called several times, NOT just for srat. so those
nodes_clear(numa_nodes_parsed)
memset(&numa_meminfo, 0, sizeof(numa_meminfo))
can not be just removed. Need to consider sequence is: numaq, srat, amd, dummy.
and make fall back path working.
2. simply split acpi_numa_init to early_parse_srat.
a. that early_parse_srat is NOT called for ia64, so you break ia64.
b. for (i = 0; i < MAX_LOCAL_APIC; i++)
set_apicid_to_node(i, NUMA_NO_NODE)
still left in numa_init. So it will just clear result from early_parse_srat.
it should be moved before that....
c. it breaks ACPI_TABLE_OVERIDE...as the acpi table scan is moved
early before override from INITRD is settled.
3. that patch TITLE is total misleading, there is NO x86 in the title,
but it changes critical x86 code. It caused x86 guys did not
pay attention to find the problem early. Those patches really should
be routed via tip/x86/mm.
4. after that commit, following range can not use movable ram:
a. real_mode code.... well..funny, legacy Node0 [0,1M) could be hot-removed?
b. initrd... it will be freed after booting, so it could be on movable...
c. crashkernel for kdump...: looks like we can not put kdump kernel above 4G
anymore.
d. init_mem_mapping: can not put page table high anymore.
e. initmem_init: vmemmap can not be high local node anymore. That is
not good.
If node is hotplugable, the mem related range like page table and
vmemmap could be on the that node without problem and should be on that
node.
We have workaround patch that could fix some problems, but some can not
be fixed.
So just remove that offending commit and related ones including:
f7210e6c4ac7 ("mm/memblock.c: use CONFIG_HAVE_MEMBLOCK_NODE_MAP to
protect movablecore_map in memblock_overlaps_region().")
01a178a94e8e ("acpi, memory-hotplug: support getting hotplug info from
SRAT")
27168d38fa20 ("acpi, memory-hotplug: extend movablemem_map ranges to
the end of node")
e8d195525809 ("acpi, memory-hotplug: parse SRAT before memblock is
ready")
fb06bc8e5f42 ("page_alloc: bootmem limit with movablecore_map")
42f47e27e761 ("page_alloc: make movablemem_map have higher priority")
6981ec31146c ("page_alloc: introduce zone_movable_limit[] to keep
movable limit for nodes")
34b71f1e04fc ("page_alloc: add movable_memmap kernel parameter")
4d59a75125d5 ("x86: get pg_data_t's memory from other node")
Later we should have patches that will make sure kernel put page table
and vmemmap on local node ram instead of push them down to node0. Also
need to find way to put other kernel used ram to local node ram.
Reported-by: Tim Gardner <tim.gardner@canonical.com>
Reported-by: Don Morris <don.morris@hp.com>
Bisected-by: Don Morris <don.morris@hp.com>
Tested-by: Don Morris <don.morris@hp.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Thomas Renninger <trenn@suse.de>
Cc: Tejun Heo <tj@kernel.org>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-03-01 15:51:27 -07:00
|
|
|
|
2011-12-08 11:22:09 -07:00
|
|
|
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
2007-05-08 01:23:07 -06:00
|
|
|
static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
|
2013-07-08 16:59:52 -06:00
|
|
|
unsigned long node_start_pfn,
|
|
|
|
unsigned long node_end_pfn,
|
|
|
|
unsigned long *zones_size,
|
|
|
|
unsigned long *zholes_size)
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
{
|
2015-06-24 17:57:02 -06:00
|
|
|
unsigned long realtotalpages = 0, totalpages = 0;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
enum zone_type i;
|
|
|
|
|
2015-06-24 17:57:02 -06:00
|
|
|
for (i = 0; i < MAX_NR_ZONES; i++) {
|
|
|
|
struct zone *zone = pgdat->node_zones + i;
|
2016-03-15 15:55:18 -06:00
|
|
|
unsigned long zone_start_pfn, zone_end_pfn;
|
2015-06-24 17:57:02 -06:00
|
|
|
unsigned long size, real_size;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
2015-06-24 17:57:02 -06:00
|
|
|
size = zone_spanned_pages_in_node(pgdat->node_id, i,
|
|
|
|
node_start_pfn,
|
|
|
|
node_end_pfn,
|
2016-03-15 15:55:18 -06:00
|
|
|
&zone_start_pfn,
|
|
|
|
&zone_end_pfn,
|
2015-06-24 17:57:02 -06:00
|
|
|
zones_size);
|
|
|
|
real_size = size - zone_absent_pages_in_node(pgdat->node_id, i,
|
2013-07-08 16:59:52 -06:00
|
|
|
node_start_pfn, node_end_pfn,
|
|
|
|
zholes_size);
|
2016-03-15 15:55:18 -06:00
|
|
|
if (size)
|
|
|
|
zone->zone_start_pfn = zone_start_pfn;
|
|
|
|
else
|
|
|
|
zone->zone_start_pfn = 0;
|
2015-06-24 17:57:02 -06:00
|
|
|
zone->spanned_pages = size;
|
|
|
|
zone->present_pages = real_size;
|
|
|
|
|
|
|
|
totalpages += size;
|
|
|
|
realtotalpages += real_size;
|
|
|
|
}
|
|
|
|
|
|
|
|
pgdat->node_spanned_pages = totalpages;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
pgdat->node_present_pages = realtotalpages;
|
|
|
|
printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
|
|
|
|
realtotalpages);
|
|
|
|
}
|
|
|
|
|
Add a bitmap that is used to track flags affecting a block of pages
Here is the latest revision of the anti-fragmentation patches. Of particular
note in this version is special treatment of high-order atomic allocations.
Care is taken to group them together and avoid grouping pages of other types
near them. Artifical tests imply that it works. I'm trying to get the
hardware together that would allow setting up of a "real" test. If anyone
already has a setup and test that can trigger the atomic-allocation problem,
I'd appreciate a test of these patches and a report. The second major change
is that these patches will apply cleanly with patches that implement
anti-fragmentation through zones.
kernbench shows effectively no performance difference varying between -0.2%
and +2% on a variety of test machines. Success rates for huge page allocation
are dramatically increased. For example, on a ppc64 machine, the vanilla
kernel was only able to allocate 1% of memory as a hugepage and this was due
to a single hugepage reserved as min_free_kbytes. With these patches applied,
17% was allocatable as superpages. With reclaim-related fixes from Andy
Whitcroft, it was 40% and further reclaim-related improvements should increase
this further.
Changelog Since V28
o Group high-order atomic allocations together
o It is no longer required to set min_free_kbytes to 10% of memory. A value
of 16384 in most cases will be sufficient
o Now applied with zone-based anti-fragmentation
o Fix incorrect VM_BUG_ON within buffered_rmqueue()
o Reorder the stack so later patches do not back out work from earlier patches
o Fix bug were journal pages were being treated as movable
o Bias placement of non-movable pages to lower PFNs
o More agressive clustering of reclaimable pages in reactions to workloads
like updatedb that flood the size of inode caches
Changelog Since V27
o Renamed anti-fragmentation to Page Clustering. Anti-fragmentation was giving
the mistaken impression that it was the 100% solution for high order
allocations. Instead, it greatly increases the chances high-order
allocations will succeed and lays the foundation for defragmentation and
memory hot-remove to work properly
o Redefine page groupings based on ability to migrate or reclaim instead of
basing on reclaimability alone
o Get rid of spurious inits
o Per-cpu lists are no longer split up per-type. Instead the per-cpu list is
searched for a page of the appropriate type
o Added more explanation commentary
o Fix up bug in pageblock code where bitmap was used before being initalised
Changelog Since V26
o Fix double init of lists in setup_pageset
Changelog Since V25
o Fix loop order of for_each_rclmtype_order so that order of loop matches args
o gfpflags_to_rclmtype uses gfp_t instead of unsigned long
o Rename get_pageblock_type() to get_page_rclmtype()
o Fix alignment problem in move_freepages()
o Add mechanism for assigning flags to blocks of pages instead of page->flags
o On fallback, do not examine the preferred list of free pages a second time
The purpose of these patches is to reduce external fragmentation by grouping
pages of related types together. When pages are migrated (or reclaimed under
memory pressure), large contiguous pages will be freed.
This patch works by categorising allocations by their ability to migrate;
Movable - The pages may be moved with the page migration mechanism. These are
generally userspace pages.
Reclaimable - These are allocations for some kernel caches that are
reclaimable or allocations that are known to be very short-lived.
Unmovable - These are pages that are allocated by the kernel that
are not trivially reclaimed. For example, the memory allocated for a
loaded module would be in this category. By default, allocations are
considered to be of this type
HighAtomic - These are high-order allocations belonging to callers that
cannot sleep or perform any IO. In practice, this is restricted to
jumbo frame allocation for network receive. It is assumed that the
allocations are short-lived
Instead of having one MAX_ORDER-sized array of free lists in struct free_area,
there is one for each type of reclaimability. Once a 2^MAX_ORDER block of
pages is split for a type of allocation, it is added to the free-lists for
that type, in effect reserving it. Hence, over time, pages of the different
types can be clustered together.
When the preferred freelists are expired, the largest possible block is taken
from an alternative list. Buddies that are split from that large block are
placed on the preferred allocation-type freelists to mitigate fragmentation.
This implementation gives best-effort for low fragmentation in all zones.
Ideally, min_free_kbytes needs to be set to a value equal to 4 * (1 <<
(MAX_ORDER-1)) pages in most cases. This would be 16384 on x86 and x86_64 for
example.
Our tests show that about 60-70% of physical memory can be allocated on a
desktop after a few days uptime. In benchmarks and stress tests, we are
finding that 80% of memory is available as contiguous blocks at the end of the
test. To compare, a standard kernel was getting < 1% of memory as large pages
on a desktop and about 8-12% of memory as large pages at the end of stress
tests.
Following this email are 12 patches that implement thie page grouping feature.
The first patch introduces a mechanism for storing flags related to a whole
block of pages. Then allocations are split between movable and all other
allocations. Following that are patches to deal with per-cpu pages and make
the mechanism configurable. The next patch moves free pages between lists
when partially allocated blocks are used for pages of another migrate type.
The second last patch groups reclaimable kernel allocations such as inode
caches together. The final patch related to groupings keeps high-order atomic
allocations.
The last two patches are more concerned with control of fragmentation. The
second last patch biases placement of non-movable allocations towards the
start of memory. This is with a view of supporting memory hot-remove of DIMMs
with higher PFNs in the future. The biasing could be enforced a lot heavier
but it would cost. The last patch agressively clusters reclaimable pages like
inode caches together.
The fragmentation reduction strategy needs to track if pages within a block
can be moved or reclaimed so that pages are freed to the appropriate list.
This patch adds a bitmap for flags affecting a whole a MAX_ORDER block of
pages.
In non-SPARSEMEM configurations, the bitmap is stored in the struct zone and
allocated during initialisation. SPARSEMEM statically allocates the bitmap in
a struct mem_section so that bitmaps do not have to be resized during memory
hotadd. This wastes a small amount of memory per unused section (usually
sizeof(unsigned long)) but the complexity of dynamically allocating the memory
is quite high.
Additional credit to Andy Whitcroft who reviewed up an earlier implementation
of the mechanism an suggested how to make it a *lot* cleaner.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:47 -06:00
|
|
|
#ifndef CONFIG_SPARSEMEM
|
|
|
|
/*
|
|
|
|
* Calculate the size of the zone->blockflags rounded to an unsigned long
|
2007-10-16 02:26:01 -06:00
|
|
|
* Start by making sure zonesize is a multiple of pageblock_order by rounding
|
|
|
|
* up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
|
Add a bitmap that is used to track flags affecting a block of pages
Here is the latest revision of the anti-fragmentation patches. Of particular
note in this version is special treatment of high-order atomic allocations.
Care is taken to group them together and avoid grouping pages of other types
near them. Artifical tests imply that it works. I'm trying to get the
hardware together that would allow setting up of a "real" test. If anyone
already has a setup and test that can trigger the atomic-allocation problem,
I'd appreciate a test of these patches and a report. The second major change
is that these patches will apply cleanly with patches that implement
anti-fragmentation through zones.
kernbench shows effectively no performance difference varying between -0.2%
and +2% on a variety of test machines. Success rates for huge page allocation
are dramatically increased. For example, on a ppc64 machine, the vanilla
kernel was only able to allocate 1% of memory as a hugepage and this was due
to a single hugepage reserved as min_free_kbytes. With these patches applied,
17% was allocatable as superpages. With reclaim-related fixes from Andy
Whitcroft, it was 40% and further reclaim-related improvements should increase
this further.
Changelog Since V28
o Group high-order atomic allocations together
o It is no longer required to set min_free_kbytes to 10% of memory. A value
of 16384 in most cases will be sufficient
o Now applied with zone-based anti-fragmentation
o Fix incorrect VM_BUG_ON within buffered_rmqueue()
o Reorder the stack so later patches do not back out work from earlier patches
o Fix bug were journal pages were being treated as movable
o Bias placement of non-movable pages to lower PFNs
o More agressive clustering of reclaimable pages in reactions to workloads
like updatedb that flood the size of inode caches
Changelog Since V27
o Renamed anti-fragmentation to Page Clustering. Anti-fragmentation was giving
the mistaken impression that it was the 100% solution for high order
allocations. Instead, it greatly increases the chances high-order
allocations will succeed and lays the foundation for defragmentation and
memory hot-remove to work properly
o Redefine page groupings based on ability to migrate or reclaim instead of
basing on reclaimability alone
o Get rid of spurious inits
o Per-cpu lists are no longer split up per-type. Instead the per-cpu list is
searched for a page of the appropriate type
o Added more explanation commentary
o Fix up bug in pageblock code where bitmap was used before being initalised
Changelog Since V26
o Fix double init of lists in setup_pageset
Changelog Since V25
o Fix loop order of for_each_rclmtype_order so that order of loop matches args
o gfpflags_to_rclmtype uses gfp_t instead of unsigned long
o Rename get_pageblock_type() to get_page_rclmtype()
o Fix alignment problem in move_freepages()
o Add mechanism for assigning flags to blocks of pages instead of page->flags
o On fallback, do not examine the preferred list of free pages a second time
The purpose of these patches is to reduce external fragmentation by grouping
pages of related types together. When pages are migrated (or reclaimed under
memory pressure), large contiguous pages will be freed.
This patch works by categorising allocations by their ability to migrate;
Movable - The pages may be moved with the page migration mechanism. These are
generally userspace pages.
Reclaimable - These are allocations for some kernel caches that are
reclaimable or allocations that are known to be very short-lived.
Unmovable - These are pages that are allocated by the kernel that
are not trivially reclaimed. For example, the memory allocated for a
loaded module would be in this category. By default, allocations are
considered to be of this type
HighAtomic - These are high-order allocations belonging to callers that
cannot sleep or perform any IO. In practice, this is restricted to
jumbo frame allocation for network receive. It is assumed that the
allocations are short-lived
Instead of having one MAX_ORDER-sized array of free lists in struct free_area,
there is one for each type of reclaimability. Once a 2^MAX_ORDER block of
pages is split for a type of allocation, it is added to the free-lists for
that type, in effect reserving it. Hence, over time, pages of the different
types can be clustered together.
When the preferred freelists are expired, the largest possible block is taken
from an alternative list. Buddies that are split from that large block are
placed on the preferred allocation-type freelists to mitigate fragmentation.
This implementation gives best-effort for low fragmentation in all zones.
Ideally, min_free_kbytes needs to be set to a value equal to 4 * (1 <<
(MAX_ORDER-1)) pages in most cases. This would be 16384 on x86 and x86_64 for
example.
Our tests show that about 60-70% of physical memory can be allocated on a
desktop after a few days uptime. In benchmarks and stress tests, we are
finding that 80% of memory is available as contiguous blocks at the end of the
test. To compare, a standard kernel was getting < 1% of memory as large pages
on a desktop and about 8-12% of memory as large pages at the end of stress
tests.
Following this email are 12 patches that implement thie page grouping feature.
The first patch introduces a mechanism for storing flags related to a whole
block of pages. Then allocations are split between movable and all other
allocations. Following that are patches to deal with per-cpu pages and make
the mechanism configurable. The next patch moves free pages between lists
when partially allocated blocks are used for pages of another migrate type.
The second last patch groups reclaimable kernel allocations such as inode
caches together. The final patch related to groupings keeps high-order atomic
allocations.
The last two patches are more concerned with control of fragmentation. The
second last patch biases placement of non-movable allocations towards the
start of memory. This is with a view of supporting memory hot-remove of DIMMs
with higher PFNs in the future. The biasing could be enforced a lot heavier
but it would cost. The last patch agressively clusters reclaimable pages like
inode caches together.
The fragmentation reduction strategy needs to track if pages within a block
can be moved or reclaimed so that pages are freed to the appropriate list.
This patch adds a bitmap for flags affecting a whole a MAX_ORDER block of
pages.
In non-SPARSEMEM configurations, the bitmap is stored in the struct zone and
allocated during initialisation. SPARSEMEM statically allocates the bitmap in
a struct mem_section so that bitmaps do not have to be resized during memory
hotadd. This wastes a small amount of memory per unused section (usually
sizeof(unsigned long)) but the complexity of dynamically allocating the memory
is quite high.
Additional credit to Andy Whitcroft who reviewed up an earlier implementation
of the mechanism an suggested how to make it a *lot* cleaner.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:47 -06:00
|
|
|
* round what is now in bits to nearest long in bits, then return it in
|
|
|
|
* bytes.
|
|
|
|
*/
|
2013-02-18 10:58:02 -07:00
|
|
|
static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
|
Add a bitmap that is used to track flags affecting a block of pages
Here is the latest revision of the anti-fragmentation patches. Of particular
note in this version is special treatment of high-order atomic allocations.
Care is taken to group them together and avoid grouping pages of other types
near them. Artifical tests imply that it works. I'm trying to get the
hardware together that would allow setting up of a "real" test. If anyone
already has a setup and test that can trigger the atomic-allocation problem,
I'd appreciate a test of these patches and a report. The second major change
is that these patches will apply cleanly with patches that implement
anti-fragmentation through zones.
kernbench shows effectively no performance difference varying between -0.2%
and +2% on a variety of test machines. Success rates for huge page allocation
are dramatically increased. For example, on a ppc64 machine, the vanilla
kernel was only able to allocate 1% of memory as a hugepage and this was due
to a single hugepage reserved as min_free_kbytes. With these patches applied,
17% was allocatable as superpages. With reclaim-related fixes from Andy
Whitcroft, it was 40% and further reclaim-related improvements should increase
this further.
Changelog Since V28
o Group high-order atomic allocations together
o It is no longer required to set min_free_kbytes to 10% of memory. A value
of 16384 in most cases will be sufficient
o Now applied with zone-based anti-fragmentation
o Fix incorrect VM_BUG_ON within buffered_rmqueue()
o Reorder the stack so later patches do not back out work from earlier patches
o Fix bug were journal pages were being treated as movable
o Bias placement of non-movable pages to lower PFNs
o More agressive clustering of reclaimable pages in reactions to workloads
like updatedb that flood the size of inode caches
Changelog Since V27
o Renamed anti-fragmentation to Page Clustering. Anti-fragmentation was giving
the mistaken impression that it was the 100% solution for high order
allocations. Instead, it greatly increases the chances high-order
allocations will succeed and lays the foundation for defragmentation and
memory hot-remove to work properly
o Redefine page groupings based on ability to migrate or reclaim instead of
basing on reclaimability alone
o Get rid of spurious inits
o Per-cpu lists are no longer split up per-type. Instead the per-cpu list is
searched for a page of the appropriate type
o Added more explanation commentary
o Fix up bug in pageblock code where bitmap was used before being initalised
Changelog Since V26
o Fix double init of lists in setup_pageset
Changelog Since V25
o Fix loop order of for_each_rclmtype_order so that order of loop matches args
o gfpflags_to_rclmtype uses gfp_t instead of unsigned long
o Rename get_pageblock_type() to get_page_rclmtype()
o Fix alignment problem in move_freepages()
o Add mechanism for assigning flags to blocks of pages instead of page->flags
o On fallback, do not examine the preferred list of free pages a second time
The purpose of these patches is to reduce external fragmentation by grouping
pages of related types together. When pages are migrated (or reclaimed under
memory pressure), large contiguous pages will be freed.
This patch works by categorising allocations by their ability to migrate;
Movable - The pages may be moved with the page migration mechanism. These are
generally userspace pages.
Reclaimable - These are allocations for some kernel caches that are
reclaimable or allocations that are known to be very short-lived.
Unmovable - These are pages that are allocated by the kernel that
are not trivially reclaimed. For example, the memory allocated for a
loaded module would be in this category. By default, allocations are
considered to be of this type
HighAtomic - These are high-order allocations belonging to callers that
cannot sleep or perform any IO. In practice, this is restricted to
jumbo frame allocation for network receive. It is assumed that the
allocations are short-lived
Instead of having one MAX_ORDER-sized array of free lists in struct free_area,
there is one for each type of reclaimability. Once a 2^MAX_ORDER block of
pages is split for a type of allocation, it is added to the free-lists for
that type, in effect reserving it. Hence, over time, pages of the different
types can be clustered together.
When the preferred freelists are expired, the largest possible block is taken
from an alternative list. Buddies that are split from that large block are
placed on the preferred allocation-type freelists to mitigate fragmentation.
This implementation gives best-effort for low fragmentation in all zones.
Ideally, min_free_kbytes needs to be set to a value equal to 4 * (1 <<
(MAX_ORDER-1)) pages in most cases. This would be 16384 on x86 and x86_64 for
example.
Our tests show that about 60-70% of physical memory can be allocated on a
desktop after a few days uptime. In benchmarks and stress tests, we are
finding that 80% of memory is available as contiguous blocks at the end of the
test. To compare, a standard kernel was getting < 1% of memory as large pages
on a desktop and about 8-12% of memory as large pages at the end of stress
tests.
Following this email are 12 patches that implement thie page grouping feature.
The first patch introduces a mechanism for storing flags related to a whole
block of pages. Then allocations are split between movable and all other
allocations. Following that are patches to deal with per-cpu pages and make
the mechanism configurable. The next patch moves free pages between lists
when partially allocated blocks are used for pages of another migrate type.
The second last patch groups reclaimable kernel allocations such as inode
caches together. The final patch related to groupings keeps high-order atomic
allocations.
The last two patches are more concerned with control of fragmentation. The
second last patch biases placement of non-movable allocations towards the
start of memory. This is with a view of supporting memory hot-remove of DIMMs
with higher PFNs in the future. The biasing could be enforced a lot heavier
but it would cost. The last patch agressively clusters reclaimable pages like
inode caches together.
The fragmentation reduction strategy needs to track if pages within a block
can be moved or reclaimed so that pages are freed to the appropriate list.
This patch adds a bitmap for flags affecting a whole a MAX_ORDER block of
pages.
In non-SPARSEMEM configurations, the bitmap is stored in the struct zone and
allocated during initialisation. SPARSEMEM statically allocates the bitmap in
a struct mem_section so that bitmaps do not have to be resized during memory
hotadd. This wastes a small amount of memory per unused section (usually
sizeof(unsigned long)) but the complexity of dynamically allocating the memory
is quite high.
Additional credit to Andy Whitcroft who reviewed up an earlier implementation
of the mechanism an suggested how to make it a *lot* cleaner.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:47 -06:00
|
|
|
{
|
|
|
|
unsigned long usemapsize;
|
|
|
|
|
2013-02-18 10:58:02 -07:00
|
|
|
zonesize += zone_start_pfn & (pageblock_nr_pages-1);
|
2007-10-16 02:26:01 -06:00
|
|
|
usemapsize = roundup(zonesize, pageblock_nr_pages);
|
|
|
|
usemapsize = usemapsize >> pageblock_order;
|
Add a bitmap that is used to track flags affecting a block of pages
Here is the latest revision of the anti-fragmentation patches. Of particular
note in this version is special treatment of high-order atomic allocations.
Care is taken to group them together and avoid grouping pages of other types
near them. Artifical tests imply that it works. I'm trying to get the
hardware together that would allow setting up of a "real" test. If anyone
already has a setup and test that can trigger the atomic-allocation problem,
I'd appreciate a test of these patches and a report. The second major change
is that these patches will apply cleanly with patches that implement
anti-fragmentation through zones.
kernbench shows effectively no performance difference varying between -0.2%
and +2% on a variety of test machines. Success rates for huge page allocation
are dramatically increased. For example, on a ppc64 machine, the vanilla
kernel was only able to allocate 1% of memory as a hugepage and this was due
to a single hugepage reserved as min_free_kbytes. With these patches applied,
17% was allocatable as superpages. With reclaim-related fixes from Andy
Whitcroft, it was 40% and further reclaim-related improvements should increase
this further.
Changelog Since V28
o Group high-order atomic allocations together
o It is no longer required to set min_free_kbytes to 10% of memory. A value
of 16384 in most cases will be sufficient
o Now applied with zone-based anti-fragmentation
o Fix incorrect VM_BUG_ON within buffered_rmqueue()
o Reorder the stack so later patches do not back out work from earlier patches
o Fix bug were journal pages were being treated as movable
o Bias placement of non-movable pages to lower PFNs
o More agressive clustering of reclaimable pages in reactions to workloads
like updatedb that flood the size of inode caches
Changelog Since V27
o Renamed anti-fragmentation to Page Clustering. Anti-fragmentation was giving
the mistaken impression that it was the 100% solution for high order
allocations. Instead, it greatly increases the chances high-order
allocations will succeed and lays the foundation for defragmentation and
memory hot-remove to work properly
o Redefine page groupings based on ability to migrate or reclaim instead of
basing on reclaimability alone
o Get rid of spurious inits
o Per-cpu lists are no longer split up per-type. Instead the per-cpu list is
searched for a page of the appropriate type
o Added more explanation commentary
o Fix up bug in pageblock code where bitmap was used before being initalised
Changelog Since V26
o Fix double init of lists in setup_pageset
Changelog Since V25
o Fix loop order of for_each_rclmtype_order so that order of loop matches args
o gfpflags_to_rclmtype uses gfp_t instead of unsigned long
o Rename get_pageblock_type() to get_page_rclmtype()
o Fix alignment problem in move_freepages()
o Add mechanism for assigning flags to blocks of pages instead of page->flags
o On fallback, do not examine the preferred list of free pages a second time
The purpose of these patches is to reduce external fragmentation by grouping
pages of related types together. When pages are migrated (or reclaimed under
memory pressure), large contiguous pages will be freed.
This patch works by categorising allocations by their ability to migrate;
Movable - The pages may be moved with the page migration mechanism. These are
generally userspace pages.
Reclaimable - These are allocations for some kernel caches that are
reclaimable or allocations that are known to be very short-lived.
Unmovable - These are pages that are allocated by the kernel that
are not trivially reclaimed. For example, the memory allocated for a
loaded module would be in this category. By default, allocations are
considered to be of this type
HighAtomic - These are high-order allocations belonging to callers that
cannot sleep or perform any IO. In practice, this is restricted to
jumbo frame allocation for network receive. It is assumed that the
allocations are short-lived
Instead of having one MAX_ORDER-sized array of free lists in struct free_area,
there is one for each type of reclaimability. Once a 2^MAX_ORDER block of
pages is split for a type of allocation, it is added to the free-lists for
that type, in effect reserving it. Hence, over time, pages of the different
types can be clustered together.
When the preferred freelists are expired, the largest possible block is taken
from an alternative list. Buddies that are split from that large block are
placed on the preferred allocation-type freelists to mitigate fragmentation.
This implementation gives best-effort for low fragmentation in all zones.
Ideally, min_free_kbytes needs to be set to a value equal to 4 * (1 <<
(MAX_ORDER-1)) pages in most cases. This would be 16384 on x86 and x86_64 for
example.
Our tests show that about 60-70% of physical memory can be allocated on a
desktop after a few days uptime. In benchmarks and stress tests, we are
finding that 80% of memory is available as contiguous blocks at the end of the
test. To compare, a standard kernel was getting < 1% of memory as large pages
on a desktop and about 8-12% of memory as large pages at the end of stress
tests.
Following this email are 12 patches that implement thie page grouping feature.
The first patch introduces a mechanism for storing flags related to a whole
block of pages. Then allocations are split between movable and all other
allocations. Following that are patches to deal with per-cpu pages and make
the mechanism configurable. The next patch moves free pages between lists
when partially allocated blocks are used for pages of another migrate type.
The second last patch groups reclaimable kernel allocations such as inode
caches together. The final patch related to groupings keeps high-order atomic
allocations.
The last two patches are more concerned with control of fragmentation. The
second last patch biases placement of non-movable allocations towards the
start of memory. This is with a view of supporting memory hot-remove of DIMMs
with higher PFNs in the future. The biasing could be enforced a lot heavier
but it would cost. The last patch agressively clusters reclaimable pages like
inode caches together.
The fragmentation reduction strategy needs to track if pages within a block
can be moved or reclaimed so that pages are freed to the appropriate list.
This patch adds a bitmap for flags affecting a whole a MAX_ORDER block of
pages.
In non-SPARSEMEM configurations, the bitmap is stored in the struct zone and
allocated during initialisation. SPARSEMEM statically allocates the bitmap in
a struct mem_section so that bitmaps do not have to be resized during memory
hotadd. This wastes a small amount of memory per unused section (usually
sizeof(unsigned long)) but the complexity of dynamically allocating the memory
is quite high.
Additional credit to Andy Whitcroft who reviewed up an earlier implementation
of the mechanism an suggested how to make it a *lot* cleaner.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:47 -06:00
|
|
|
usemapsize *= NR_PAGEBLOCK_BITS;
|
|
|
|
usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
|
|
|
|
|
|
|
|
return usemapsize / 8;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __init setup_usemap(struct pglist_data *pgdat,
|
2013-02-18 10:58:02 -07:00
|
|
|
struct zone *zone,
|
|
|
|
unsigned long zone_start_pfn,
|
|
|
|
unsigned long zonesize)
|
Add a bitmap that is used to track flags affecting a block of pages
Here is the latest revision of the anti-fragmentation patches. Of particular
note in this version is special treatment of high-order atomic allocations.
Care is taken to group them together and avoid grouping pages of other types
near them. Artifical tests imply that it works. I'm trying to get the
hardware together that would allow setting up of a "real" test. If anyone
already has a setup and test that can trigger the atomic-allocation problem,
I'd appreciate a test of these patches and a report. The second major change
is that these patches will apply cleanly with patches that implement
anti-fragmentation through zones.
kernbench shows effectively no performance difference varying between -0.2%
and +2% on a variety of test machines. Success rates for huge page allocation
are dramatically increased. For example, on a ppc64 machine, the vanilla
kernel was only able to allocate 1% of memory as a hugepage and this was due
to a single hugepage reserved as min_free_kbytes. With these patches applied,
17% was allocatable as superpages. With reclaim-related fixes from Andy
Whitcroft, it was 40% and further reclaim-related improvements should increase
this further.
Changelog Since V28
o Group high-order atomic allocations together
o It is no longer required to set min_free_kbytes to 10% of memory. A value
of 16384 in most cases will be sufficient
o Now applied with zone-based anti-fragmentation
o Fix incorrect VM_BUG_ON within buffered_rmqueue()
o Reorder the stack so later patches do not back out work from earlier patches
o Fix bug were journal pages were being treated as movable
o Bias placement of non-movable pages to lower PFNs
o More agressive clustering of reclaimable pages in reactions to workloads
like updatedb that flood the size of inode caches
Changelog Since V27
o Renamed anti-fragmentation to Page Clustering. Anti-fragmentation was giving
the mistaken impression that it was the 100% solution for high order
allocations. Instead, it greatly increases the chances high-order
allocations will succeed and lays the foundation for defragmentation and
memory hot-remove to work properly
o Redefine page groupings based on ability to migrate or reclaim instead of
basing on reclaimability alone
o Get rid of spurious inits
o Per-cpu lists are no longer split up per-type. Instead the per-cpu list is
searched for a page of the appropriate type
o Added more explanation commentary
o Fix up bug in pageblock code where bitmap was used before being initalised
Changelog Since V26
o Fix double init of lists in setup_pageset
Changelog Since V25
o Fix loop order of for_each_rclmtype_order so that order of loop matches args
o gfpflags_to_rclmtype uses gfp_t instead of unsigned long
o Rename get_pageblock_type() to get_page_rclmtype()
o Fix alignment problem in move_freepages()
o Add mechanism for assigning flags to blocks of pages instead of page->flags
o On fallback, do not examine the preferred list of free pages a second time
The purpose of these patches is to reduce external fragmentation by grouping
pages of related types together. When pages are migrated (or reclaimed under
memory pressure), large contiguous pages will be freed.
This patch works by categorising allocations by their ability to migrate;
Movable - The pages may be moved with the page migration mechanism. These are
generally userspace pages.
Reclaimable - These are allocations for some kernel caches that are
reclaimable or allocations that are known to be very short-lived.
Unmovable - These are pages that are allocated by the kernel that
are not trivially reclaimed. For example, the memory allocated for a
loaded module would be in this category. By default, allocations are
considered to be of this type
HighAtomic - These are high-order allocations belonging to callers that
cannot sleep or perform any IO. In practice, this is restricted to
jumbo frame allocation for network receive. It is assumed that the
allocations are short-lived
Instead of having one MAX_ORDER-sized array of free lists in struct free_area,
there is one for each type of reclaimability. Once a 2^MAX_ORDER block of
pages is split for a type of allocation, it is added to the free-lists for
that type, in effect reserving it. Hence, over time, pages of the different
types can be clustered together.
When the preferred freelists are expired, the largest possible block is taken
from an alternative list. Buddies that are split from that large block are
placed on the preferred allocation-type freelists to mitigate fragmentation.
This implementation gives best-effort for low fragmentation in all zones.
Ideally, min_free_kbytes needs to be set to a value equal to 4 * (1 <<
(MAX_ORDER-1)) pages in most cases. This would be 16384 on x86 and x86_64 for
example.
Our tests show that about 60-70% of physical memory can be allocated on a
desktop after a few days uptime. In benchmarks and stress tests, we are
finding that 80% of memory is available as contiguous blocks at the end of the
test. To compare, a standard kernel was getting < 1% of memory as large pages
on a desktop and about 8-12% of memory as large pages at the end of stress
tests.
Following this email are 12 patches that implement thie page grouping feature.
The first patch introduces a mechanism for storing flags related to a whole
block of pages. Then allocations are split between movable and all other
allocations. Following that are patches to deal with per-cpu pages and make
the mechanism configurable. The next patch moves free pages between lists
when partially allocated blocks are used for pages of another migrate type.
The second last patch groups reclaimable kernel allocations such as inode
caches together. The final patch related to groupings keeps high-order atomic
allocations.
The last two patches are more concerned with control of fragmentation. The
second last patch biases placement of non-movable allocations towards the
start of memory. This is with a view of supporting memory hot-remove of DIMMs
with higher PFNs in the future. The biasing could be enforced a lot heavier
but it would cost. The last patch agressively clusters reclaimable pages like
inode caches together.
The fragmentation reduction strategy needs to track if pages within a block
can be moved or reclaimed so that pages are freed to the appropriate list.
This patch adds a bitmap for flags affecting a whole a MAX_ORDER block of
pages.
In non-SPARSEMEM configurations, the bitmap is stored in the struct zone and
allocated during initialisation. SPARSEMEM statically allocates the bitmap in
a struct mem_section so that bitmaps do not have to be resized during memory
hotadd. This wastes a small amount of memory per unused section (usually
sizeof(unsigned long)) but the complexity of dynamically allocating the memory
is quite high.
Additional credit to Andy Whitcroft who reviewed up an earlier implementation
of the mechanism an suggested how to make it a *lot* cleaner.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:47 -06:00
|
|
|
{
|
2013-02-18 10:58:02 -07:00
|
|
|
unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
|
Add a bitmap that is used to track flags affecting a block of pages
Here is the latest revision of the anti-fragmentation patches. Of particular
note in this version is special treatment of high-order atomic allocations.
Care is taken to group them together and avoid grouping pages of other types
near them. Artifical tests imply that it works. I'm trying to get the
hardware together that would allow setting up of a "real" test. If anyone
already has a setup and test that can trigger the atomic-allocation problem,
I'd appreciate a test of these patches and a report. The second major change
is that these patches will apply cleanly with patches that implement
anti-fragmentation through zones.
kernbench shows effectively no performance difference varying between -0.2%
and +2% on a variety of test machines. Success rates for huge page allocation
are dramatically increased. For example, on a ppc64 machine, the vanilla
kernel was only able to allocate 1% of memory as a hugepage and this was due
to a single hugepage reserved as min_free_kbytes. With these patches applied,
17% was allocatable as superpages. With reclaim-related fixes from Andy
Whitcroft, it was 40% and further reclaim-related improvements should increase
this further.
Changelog Since V28
o Group high-order atomic allocations together
o It is no longer required to set min_free_kbytes to 10% of memory. A value
of 16384 in most cases will be sufficient
o Now applied with zone-based anti-fragmentation
o Fix incorrect VM_BUG_ON within buffered_rmqueue()
o Reorder the stack so later patches do not back out work from earlier patches
o Fix bug were journal pages were being treated as movable
o Bias placement of non-movable pages to lower PFNs
o More agressive clustering of reclaimable pages in reactions to workloads
like updatedb that flood the size of inode caches
Changelog Since V27
o Renamed anti-fragmentation to Page Clustering. Anti-fragmentation was giving
the mistaken impression that it was the 100% solution for high order
allocations. Instead, it greatly increases the chances high-order
allocations will succeed and lays the foundation for defragmentation and
memory hot-remove to work properly
o Redefine page groupings based on ability to migrate or reclaim instead of
basing on reclaimability alone
o Get rid of spurious inits
o Per-cpu lists are no longer split up per-type. Instead the per-cpu list is
searched for a page of the appropriate type
o Added more explanation commentary
o Fix up bug in pageblock code where bitmap was used before being initalised
Changelog Since V26
o Fix double init of lists in setup_pageset
Changelog Since V25
o Fix loop order of for_each_rclmtype_order so that order of loop matches args
o gfpflags_to_rclmtype uses gfp_t instead of unsigned long
o Rename get_pageblock_type() to get_page_rclmtype()
o Fix alignment problem in move_freepages()
o Add mechanism for assigning flags to blocks of pages instead of page->flags
o On fallback, do not examine the preferred list of free pages a second time
The purpose of these patches is to reduce external fragmentation by grouping
pages of related types together. When pages are migrated (or reclaimed under
memory pressure), large contiguous pages will be freed.
This patch works by categorising allocations by their ability to migrate;
Movable - The pages may be moved with the page migration mechanism. These are
generally userspace pages.
Reclaimable - These are allocations for some kernel caches that are
reclaimable or allocations that are known to be very short-lived.
Unmovable - These are pages that are allocated by the kernel that
are not trivially reclaimed. For example, the memory allocated for a
loaded module would be in this category. By default, allocations are
considered to be of this type
HighAtomic - These are high-order allocations belonging to callers that
cannot sleep or perform any IO. In practice, this is restricted to
jumbo frame allocation for network receive. It is assumed that the
allocations are short-lived
Instead of having one MAX_ORDER-sized array of free lists in struct free_area,
there is one for each type of reclaimability. Once a 2^MAX_ORDER block of
pages is split for a type of allocation, it is added to the free-lists for
that type, in effect reserving it. Hence, over time, pages of the different
types can be clustered together.
When the preferred freelists are expired, the largest possible block is taken
from an alternative list. Buddies that are split from that large block are
placed on the preferred allocation-type freelists to mitigate fragmentation.
This implementation gives best-effort for low fragmentation in all zones.
Ideally, min_free_kbytes needs to be set to a value equal to 4 * (1 <<
(MAX_ORDER-1)) pages in most cases. This would be 16384 on x86 and x86_64 for
example.
Our tests show that about 60-70% of physical memory can be allocated on a
desktop after a few days uptime. In benchmarks and stress tests, we are
finding that 80% of memory is available as contiguous blocks at the end of the
test. To compare, a standard kernel was getting < 1% of memory as large pages
on a desktop and about 8-12% of memory as large pages at the end of stress
tests.
Following this email are 12 patches that implement thie page grouping feature.
The first patch introduces a mechanism for storing flags related to a whole
block of pages. Then allocations are split between movable and all other
allocations. Following that are patches to deal with per-cpu pages and make
the mechanism configurable. The next patch moves free pages between lists
when partially allocated blocks are used for pages of another migrate type.
The second last patch groups reclaimable kernel allocations such as inode
caches together. The final patch related to groupings keeps high-order atomic
allocations.
The last two patches are more concerned with control of fragmentation. The
second last patch biases placement of non-movable allocations towards the
start of memory. This is with a view of supporting memory hot-remove of DIMMs
with higher PFNs in the future. The biasing could be enforced a lot heavier
but it would cost. The last patch agressively clusters reclaimable pages like
inode caches together.
The fragmentation reduction strategy needs to track if pages within a block
can be moved or reclaimed so that pages are freed to the appropriate list.
This patch adds a bitmap for flags affecting a whole a MAX_ORDER block of
pages.
In non-SPARSEMEM configurations, the bitmap is stored in the struct zone and
allocated during initialisation. SPARSEMEM statically allocates the bitmap in
a struct mem_section so that bitmaps do not have to be resized during memory
hotadd. This wastes a small amount of memory per unused section (usually
sizeof(unsigned long)) but the complexity of dynamically allocating the memory
is quite high.
Additional credit to Andy Whitcroft who reviewed up an earlier implementation
of the mechanism an suggested how to make it a *lot* cleaner.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:47 -06:00
|
|
|
zone->pageblock_flags = NULL;
|
2009-01-06 15:39:28 -07:00
|
|
|
if (usemapsize)
|
2014-01-21 16:50:25 -07:00
|
|
|
zone->pageblock_flags =
|
|
|
|
memblock_virt_alloc_node_nopanic(usemapsize,
|
|
|
|
pgdat->node_id);
|
Add a bitmap that is used to track flags affecting a block of pages
Here is the latest revision of the anti-fragmentation patches. Of particular
note in this version is special treatment of high-order atomic allocations.
Care is taken to group them together and avoid grouping pages of other types
near them. Artifical tests imply that it works. I'm trying to get the
hardware together that would allow setting up of a "real" test. If anyone
already has a setup and test that can trigger the atomic-allocation problem,
I'd appreciate a test of these patches and a report. The second major change
is that these patches will apply cleanly with patches that implement
anti-fragmentation through zones.
kernbench shows effectively no performance difference varying between -0.2%
and +2% on a variety of test machines. Success rates for huge page allocation
are dramatically increased. For example, on a ppc64 machine, the vanilla
kernel was only able to allocate 1% of memory as a hugepage and this was due
to a single hugepage reserved as min_free_kbytes. With these patches applied,
17% was allocatable as superpages. With reclaim-related fixes from Andy
Whitcroft, it was 40% and further reclaim-related improvements should increase
this further.
Changelog Since V28
o Group high-order atomic allocations together
o It is no longer required to set min_free_kbytes to 10% of memory. A value
of 16384 in most cases will be sufficient
o Now applied with zone-based anti-fragmentation
o Fix incorrect VM_BUG_ON within buffered_rmqueue()
o Reorder the stack so later patches do not back out work from earlier patches
o Fix bug were journal pages were being treated as movable
o Bias placement of non-movable pages to lower PFNs
o More agressive clustering of reclaimable pages in reactions to workloads
like updatedb that flood the size of inode caches
Changelog Since V27
o Renamed anti-fragmentation to Page Clustering. Anti-fragmentation was giving
the mistaken impression that it was the 100% solution for high order
allocations. Instead, it greatly increases the chances high-order
allocations will succeed and lays the foundation for defragmentation and
memory hot-remove to work properly
o Redefine page groupings based on ability to migrate or reclaim instead of
basing on reclaimability alone
o Get rid of spurious inits
o Per-cpu lists are no longer split up per-type. Instead the per-cpu list is
searched for a page of the appropriate type
o Added more explanation commentary
o Fix up bug in pageblock code where bitmap was used before being initalised
Changelog Since V26
o Fix double init of lists in setup_pageset
Changelog Since V25
o Fix loop order of for_each_rclmtype_order so that order of loop matches args
o gfpflags_to_rclmtype uses gfp_t instead of unsigned long
o Rename get_pageblock_type() to get_page_rclmtype()
o Fix alignment problem in move_freepages()
o Add mechanism for assigning flags to blocks of pages instead of page->flags
o On fallback, do not examine the preferred list of free pages a second time
The purpose of these patches is to reduce external fragmentation by grouping
pages of related types together. When pages are migrated (or reclaimed under
memory pressure), large contiguous pages will be freed.
This patch works by categorising allocations by their ability to migrate;
Movable - The pages may be moved with the page migration mechanism. These are
generally userspace pages.
Reclaimable - These are allocations for some kernel caches that are
reclaimable or allocations that are known to be very short-lived.
Unmovable - These are pages that are allocated by the kernel that
are not trivially reclaimed. For example, the memory allocated for a
loaded module would be in this category. By default, allocations are
considered to be of this type
HighAtomic - These are high-order allocations belonging to callers that
cannot sleep or perform any IO. In practice, this is restricted to
jumbo frame allocation for network receive. It is assumed that the
allocations are short-lived
Instead of having one MAX_ORDER-sized array of free lists in struct free_area,
there is one for each type of reclaimability. Once a 2^MAX_ORDER block of
pages is split for a type of allocation, it is added to the free-lists for
that type, in effect reserving it. Hence, over time, pages of the different
types can be clustered together.
When the preferred freelists are expired, the largest possible block is taken
from an alternative list. Buddies that are split from that large block are
placed on the preferred allocation-type freelists to mitigate fragmentation.
This implementation gives best-effort for low fragmentation in all zones.
Ideally, min_free_kbytes needs to be set to a value equal to 4 * (1 <<
(MAX_ORDER-1)) pages in most cases. This would be 16384 on x86 and x86_64 for
example.
Our tests show that about 60-70% of physical memory can be allocated on a
desktop after a few days uptime. In benchmarks and stress tests, we are
finding that 80% of memory is available as contiguous blocks at the end of the
test. To compare, a standard kernel was getting < 1% of memory as large pages
on a desktop and about 8-12% of memory as large pages at the end of stress
tests.
Following this email are 12 patches that implement thie page grouping feature.
The first patch introduces a mechanism for storing flags related to a whole
block of pages. Then allocations are split between movable and all other
allocations. Following that are patches to deal with per-cpu pages and make
the mechanism configurable. The next patch moves free pages between lists
when partially allocated blocks are used for pages of another migrate type.
The second last patch groups reclaimable kernel allocations such as inode
caches together. The final patch related to groupings keeps high-order atomic
allocations.
The last two patches are more concerned with control of fragmentation. The
second last patch biases placement of non-movable allocations towards the
start of memory. This is with a view of supporting memory hot-remove of DIMMs
with higher PFNs in the future. The biasing could be enforced a lot heavier
but it would cost. The last patch agressively clusters reclaimable pages like
inode caches together.
The fragmentation reduction strategy needs to track if pages within a block
can be moved or reclaimed so that pages are freed to the appropriate list.
This patch adds a bitmap for flags affecting a whole a MAX_ORDER block of
pages.
In non-SPARSEMEM configurations, the bitmap is stored in the struct zone and
allocated during initialisation. SPARSEMEM statically allocates the bitmap in
a struct mem_section so that bitmaps do not have to be resized during memory
hotadd. This wastes a small amount of memory per unused section (usually
sizeof(unsigned long)) but the complexity of dynamically allocating the memory
is quite high.
Additional credit to Andy Whitcroft who reviewed up an earlier implementation
of the mechanism an suggested how to make it a *lot* cleaner.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:47 -06:00
|
|
|
}
|
|
|
|
#else
|
2013-02-18 10:58:02 -07:00
|
|
|
static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
|
|
|
|
unsigned long zone_start_pfn, unsigned long zonesize) {}
|
Add a bitmap that is used to track flags affecting a block of pages
Here is the latest revision of the anti-fragmentation patches. Of particular
note in this version is special treatment of high-order atomic allocations.
Care is taken to group them together and avoid grouping pages of other types
near them. Artifical tests imply that it works. I'm trying to get the
hardware together that would allow setting up of a "real" test. If anyone
already has a setup and test that can trigger the atomic-allocation problem,
I'd appreciate a test of these patches and a report. The second major change
is that these patches will apply cleanly with patches that implement
anti-fragmentation through zones.
kernbench shows effectively no performance difference varying between -0.2%
and +2% on a variety of test machines. Success rates for huge page allocation
are dramatically increased. For example, on a ppc64 machine, the vanilla
kernel was only able to allocate 1% of memory as a hugepage and this was due
to a single hugepage reserved as min_free_kbytes. With these patches applied,
17% was allocatable as superpages. With reclaim-related fixes from Andy
Whitcroft, it was 40% and further reclaim-related improvements should increase
this further.
Changelog Since V28
o Group high-order atomic allocations together
o It is no longer required to set min_free_kbytes to 10% of memory. A value
of 16384 in most cases will be sufficient
o Now applied with zone-based anti-fragmentation
o Fix incorrect VM_BUG_ON within buffered_rmqueue()
o Reorder the stack so later patches do not back out work from earlier patches
o Fix bug were journal pages were being treated as movable
o Bias placement of non-movable pages to lower PFNs
o More agressive clustering of reclaimable pages in reactions to workloads
like updatedb that flood the size of inode caches
Changelog Since V27
o Renamed anti-fragmentation to Page Clustering. Anti-fragmentation was giving
the mistaken impression that it was the 100% solution for high order
allocations. Instead, it greatly increases the chances high-order
allocations will succeed and lays the foundation for defragmentation and
memory hot-remove to work properly
o Redefine page groupings based on ability to migrate or reclaim instead of
basing on reclaimability alone
o Get rid of spurious inits
o Per-cpu lists are no longer split up per-type. Instead the per-cpu list is
searched for a page of the appropriate type
o Added more explanation commentary
o Fix up bug in pageblock code where bitmap was used before being initalised
Changelog Since V26
o Fix double init of lists in setup_pageset
Changelog Since V25
o Fix loop order of for_each_rclmtype_order so that order of loop matches args
o gfpflags_to_rclmtype uses gfp_t instead of unsigned long
o Rename get_pageblock_type() to get_page_rclmtype()
o Fix alignment problem in move_freepages()
o Add mechanism for assigning flags to blocks of pages instead of page->flags
o On fallback, do not examine the preferred list of free pages a second time
The purpose of these patches is to reduce external fragmentation by grouping
pages of related types together. When pages are migrated (or reclaimed under
memory pressure), large contiguous pages will be freed.
This patch works by categorising allocations by their ability to migrate;
Movable - The pages may be moved with the page migration mechanism. These are
generally userspace pages.
Reclaimable - These are allocations for some kernel caches that are
reclaimable or allocations that are known to be very short-lived.
Unmovable - These are pages that are allocated by the kernel that
are not trivially reclaimed. For example, the memory allocated for a
loaded module would be in this category. By default, allocations are
considered to be of this type
HighAtomic - These are high-order allocations belonging to callers that
cannot sleep or perform any IO. In practice, this is restricted to
jumbo frame allocation for network receive. It is assumed that the
allocations are short-lived
Instead of having one MAX_ORDER-sized array of free lists in struct free_area,
there is one for each type of reclaimability. Once a 2^MAX_ORDER block of
pages is split for a type of allocation, it is added to the free-lists for
that type, in effect reserving it. Hence, over time, pages of the different
types can be clustered together.
When the preferred freelists are expired, the largest possible block is taken
from an alternative list. Buddies that are split from that large block are
placed on the preferred allocation-type freelists to mitigate fragmentation.
This implementation gives best-effort for low fragmentation in all zones.
Ideally, min_free_kbytes needs to be set to a value equal to 4 * (1 <<
(MAX_ORDER-1)) pages in most cases. This would be 16384 on x86 and x86_64 for
example.
Our tests show that about 60-70% of physical memory can be allocated on a
desktop after a few days uptime. In benchmarks and stress tests, we are
finding that 80% of memory is available as contiguous blocks at the end of the
test. To compare, a standard kernel was getting < 1% of memory as large pages
on a desktop and about 8-12% of memory as large pages at the end of stress
tests.
Following this email are 12 patches that implement thie page grouping feature.
The first patch introduces a mechanism for storing flags related to a whole
block of pages. Then allocations are split between movable and all other
allocations. Following that are patches to deal with per-cpu pages and make
the mechanism configurable. The next patch moves free pages between lists
when partially allocated blocks are used for pages of another migrate type.
The second last patch groups reclaimable kernel allocations such as inode
caches together. The final patch related to groupings keeps high-order atomic
allocations.
The last two patches are more concerned with control of fragmentation. The
second last patch biases placement of non-movable allocations towards the
start of memory. This is with a view of supporting memory hot-remove of DIMMs
with higher PFNs in the future. The biasing could be enforced a lot heavier
but it would cost. The last patch agressively clusters reclaimable pages like
inode caches together.
The fragmentation reduction strategy needs to track if pages within a block
can be moved or reclaimed so that pages are freed to the appropriate list.
This patch adds a bitmap for flags affecting a whole a MAX_ORDER block of
pages.
In non-SPARSEMEM configurations, the bitmap is stored in the struct zone and
allocated during initialisation. SPARSEMEM statically allocates the bitmap in
a struct mem_section so that bitmaps do not have to be resized during memory
hotadd. This wastes a small amount of memory per unused section (usually
sizeof(unsigned long)) but the complexity of dynamically allocating the memory
is quite high.
Additional credit to Andy Whitcroft who reviewed up an earlier implementation
of the mechanism an suggested how to make it a *lot* cleaner.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Cc: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:25:47 -06:00
|
|
|
#endif /* CONFIG_SPARSEMEM */
|
|
|
|
|
2007-10-16 02:26:01 -06:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
|
2007-11-28 17:21:13 -07:00
|
|
|
|
2007-10-16 02:26:01 -06:00
|
|
|
/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
|
2013-09-11 15:20:27 -06:00
|
|
|
void __paginginit set_pageblock_order(void)
|
2007-10-16 02:26:01 -06:00
|
|
|
{
|
2012-05-29 16:06:31 -06:00
|
|
|
unsigned int order;
|
|
|
|
|
2007-10-16 02:26:01 -06:00
|
|
|
/* Check that pageblock_nr_pages has not already been setup */
|
|
|
|
if (pageblock_order)
|
|
|
|
return;
|
|
|
|
|
2012-05-29 16:06:31 -06:00
|
|
|
if (HPAGE_SHIFT > PAGE_SHIFT)
|
|
|
|
order = HUGETLB_PAGE_ORDER;
|
|
|
|
else
|
|
|
|
order = MAX_ORDER - 1;
|
|
|
|
|
2007-10-16 02:26:01 -06:00
|
|
|
/*
|
|
|
|
* Assume the largest contiguous order of interest is a huge page.
|
2012-05-29 16:06:31 -06:00
|
|
|
* This value may be variable depending on boot parameters on IA64 and
|
|
|
|
* powerpc.
|
2007-10-16 02:26:01 -06:00
|
|
|
*/
|
|
|
|
pageblock_order = order;
|
|
|
|
}
|
|
|
|
#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
|
|
|
|
|
2007-11-28 17:21:13 -07:00
|
|
|
/*
|
|
|
|
* When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
|
2012-05-29 16:06:31 -06:00
|
|
|
* is unused as pageblock_order is set at compile-time. See
|
|
|
|
* include/linux/pageblock-flags.h for the values of pageblock_order based on
|
|
|
|
* the kernel config
|
2007-11-28 17:21:13 -07:00
|
|
|
*/
|
2013-09-11 15:20:27 -06:00
|
|
|
void __paginginit set_pageblock_order(void)
|
2007-11-28 17:21:13 -07:00
|
|
|
{
|
|
|
|
}
|
2007-10-16 02:26:01 -06:00
|
|
|
|
|
|
|
#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
|
|
|
|
|
2012-12-12 14:52:19 -07:00
|
|
|
static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages,
|
|
|
|
unsigned long present_pages)
|
|
|
|
{
|
|
|
|
unsigned long pages = spanned_pages;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Provide a more accurate estimation if there are holes within
|
|
|
|
* the zone and SPARSEMEM is in use. If there are holes within the
|
|
|
|
* zone, each populated memory region may cost us one or two extra
|
|
|
|
* memmap pages due to alignment because memmap pages for each
|
|
|
|
* populated regions may not naturally algined on page boundary.
|
|
|
|
* So the (present_pages >> 4) heuristic is a tradeoff for that.
|
|
|
|
*/
|
|
|
|
if (spanned_pages > present_pages + (present_pages >> 4) &&
|
|
|
|
IS_ENABLED(CONFIG_SPARSEMEM))
|
|
|
|
pages = present_pages;
|
|
|
|
|
|
|
|
return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* Set up the zone data structures:
|
|
|
|
* - mark all pages reserved
|
|
|
|
* - mark all memory queues empty
|
|
|
|
* - clear the memory bitmaps
|
2012-07-31 17:46:16 -06:00
|
|
|
*
|
|
|
|
* NOTE: pgdat should get zeroed by caller.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2015-09-08 15:59:50 -06:00
|
|
|
static void __paginginit free_area_init_core(struct pglist_data *pgdat)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-09-26 00:31:13 -06:00
|
|
|
enum zone_type j;
|
2005-10-29 19:16:50 -06:00
|
|
|
int nid = pgdat->node_id;
|
2006-06-23 03:03:10 -06:00
|
|
|
int ret;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2005-10-29 19:16:52 -06:00
|
|
|
pgdat_resize_init(pgdat);
|
2012-03-23 13:56:34 -06:00
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
|
|
spin_lock_init(&pgdat->numabalancing_migrate_lock);
|
|
|
|
pgdat->numabalancing_migrate_nr_pages = 0;
|
|
|
|
pgdat->numabalancing_migrate_next_window = jiffies;
|
2016-02-02 17:57:08 -07:00
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
spin_lock_init(&pgdat->split_queue_lock);
|
|
|
|
INIT_LIST_HEAD(&pgdat->split_queue);
|
|
|
|
pgdat->split_queue_len = 0;
|
2012-03-23 13:56:34 -06:00
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
init_waitqueue_head(&pgdat->kswapd_wait);
|
2012-07-31 17:44:35 -06:00
|
|
|
init_waitqueue_head(&pgdat->pfmemalloc_wait);
|
mm, compaction: introduce kcompactd
Memory compaction can be currently performed in several contexts:
- kswapd balancing a zone after a high-order allocation failure
- direct compaction to satisfy a high-order allocation, including THP
page fault attemps
- khugepaged trying to collapse a hugepage
- manually from /proc
The purpose of compaction is two-fold. The obvious purpose is to
satisfy a (pending or future) high-order allocation, and is easy to
evaluate. The other purpose is to keep overal memory fragmentation low
and help the anti-fragmentation mechanism. The success wrt the latter
purpose is more
The current situation wrt the purposes has a few drawbacks:
- compaction is invoked only when a high-order page or hugepage is not
available (or manually). This might be too late for the purposes of
keeping memory fragmentation low.
- direct compaction increases latency of allocations. Again, it would
be better if compaction was performed asynchronously to keep
fragmentation low, before the allocation itself comes.
- (a special case of the previous) the cost of compaction during THP
page faults can easily offset the benefits of THP.
- kswapd compaction appears to be complex, fragile and not working in
some scenarios. It could also end up compacting for a high-order
allocation request when it should be reclaiming memory for a later
order-0 request.
To improve the situation, we should be able to benefit from an
equivalent of kswapd, but for compaction - i.e. a background thread
which responds to fragmentation and the need for high-order allocations
(including hugepages) somewhat proactively.
One possibility is to extend the responsibilities of kswapd, which could
however complicate its design too much. It should be better to let
kswapd handle reclaim, as order-0 allocations are often more critical
than high-order ones.
Another possibility is to extend khugepaged, but this kthread is a
single instance and tied to THP configs.
This patch goes with the option of a new set of per-node kthreads called
kcompactd, and lays the foundations, without introducing any new
tunables. The lifecycle mimics kswapd kthreads, including the memory
hotplug hooks.
For compaction, kcompactd uses the standard compaction_suitable() and
ompact_finished() criteria and the deferred compaction functionality.
Unlike direct compaction, it uses only sync compaction, as there's no
allocation latency to minimize.
This patch doesn't yet add a call to wakeup_kcompactd. The kswapd
compact/reclaim loop for high-order pages will be replaced by waking up
kcompactd in the next patch with the description of what's wrong with
the old approach.
Waking up of the kcompactd threads is also tied to kswapd activity and
follows these rules:
- we don't want to affect any fastpaths, so wake up kcompactd only from
the slowpath, as it's done for kswapd
- if kswapd is doing reclaim, it's more important than compaction, so
don't invoke kcompactd until kswapd goes to sleep
- the target order used for kswapd is passed to kcompactd
Future possible future uses for kcompactd include the ability to wake up
kcompactd on demand in special situations, such as when hugepages are
not available (currently not done due to __GFP_NO_KSWAPD) or when a
fragmentation event (i.e. __rmqueue_fallback()) occurs. It's also
possible to perform periodic compaction with kcompactd.
[arnd@arndb.de: fix build errors with kcompactd]
[paul.gortmaker@windriver.com: don't use modular references for non modular code]
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: David Rientjes <rientjes@google.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-17 15:18:08 -06:00
|
|
|
#ifdef CONFIG_COMPACTION
|
|
|
|
init_waitqueue_head(&pgdat->kcompactd_wait);
|
|
|
|
#endif
|
mm/page_ext: resurrect struct page extending code for debugging
When we debug something, we'd like to insert some information to every
page. For this purpose, we sometimes modify struct page itself. But,
this has drawbacks. First, it requires re-compile. This makes us
hesitate to use the powerful debug feature so development process is
slowed down. And, second, sometimes it is impossible to rebuild the
kernel due to third party module dependency. At third, system behaviour
would be largely different after re-compile, because it changes size of
struct page greatly and this structure is accessed by every part of
kernel. Keeping this as it is would be better to reproduce errornous
situation.
This feature is intended to overcome above mentioned problems. This
feature allocates memory for extended data per page in certain place
rather than the struct page itself. This memory can be accessed by the
accessor functions provided by this code. During the boot process, it
checks whether allocation of huge chunk of memory is needed or not. If
not, it avoids allocating memory at all. With this advantage, we can
include this feature into the kernel in default and can avoid rebuild and
solve related problems.
Until now, memcg uses this technique. But, now, memcg decides to embed
their variable to struct page itself and it's code to extend struct page
has been removed. I'd like to use this code to develop debug feature, so
this patch resurrect it.
To help these things to work well, this patch introduces two callbacks for
clients. One is the need callback which is mandatory if user wants to
avoid useless memory allocation at boot-time. The other is optional, init
callback, which is used to do proper initialization after memory is
allocated. Detailed explanation about purpose of these functions is in
code comment. Please refer it.
Others are completely same with previous extension code in memcg.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Dave Hansen <dave@sr71.net>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Jungsoo Son <jungsoo.son@lge.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:55:46 -07:00
|
|
|
pgdat_page_ext_init(pgdat);
|
2016-07-28 16:45:28 -06:00
|
|
|
spin_lock_init(&pgdat->lru_lock);
|
2016-07-28 16:46:02 -06:00
|
|
|
lruvec_init(node_lruvec(pgdat));
|
2012-01-11 07:16:11 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
for (j = 0; j < MAX_NR_ZONES; j++) {
|
|
|
|
struct zone *zone = pgdat->node_zones + j;
|
2012-12-12 14:52:12 -07:00
|
|
|
unsigned long size, realsize, freesize, memmap_pages;
|
2016-03-15 15:55:18 -06:00
|
|
|
unsigned long zone_start_pfn = zone->zone_start_pfn;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2015-06-24 17:57:02 -06:00
|
|
|
size = zone->spanned_pages;
|
|
|
|
realsize = freesize = zone->present_pages;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-09-27 02:49:56 -06:00
|
|
|
/*
|
2012-12-12 14:52:12 -07:00
|
|
|
* Adjust freesize so that it accounts for how much memory
|
2006-09-27 02:49:56 -06:00
|
|
|
* is used by this zone for memmap. This affects the watermark
|
|
|
|
* and per-cpu initialisations
|
|
|
|
*/
|
2012-12-12 14:52:19 -07:00
|
|
|
memmap_pages = calc_memmap_size(size, realsize);
|
2014-12-12 17:56:21 -07:00
|
|
|
if (!is_highmem_idx(j)) {
|
|
|
|
if (freesize >= memmap_pages) {
|
|
|
|
freesize -= memmap_pages;
|
|
|
|
if (memmap_pages)
|
|
|
|
printk(KERN_DEBUG
|
|
|
|
" %s zone: %lu pages used for memmap\n",
|
|
|
|
zone_names[j], memmap_pages);
|
|
|
|
} else
|
2016-03-17 15:19:50 -06:00
|
|
|
pr_warn(" %s zone: %lu pages exceeds freesize %lu\n",
|
2014-12-12 17:56:21 -07:00
|
|
|
zone_names[j], memmap_pages, freesize);
|
|
|
|
}
|
2006-09-27 02:49:56 -06:00
|
|
|
|
2007-02-10 02:43:07 -07:00
|
|
|
/* Account for reserved pages */
|
2012-12-12 14:52:12 -07:00
|
|
|
if (j == 0 && freesize > dma_reserve) {
|
|
|
|
freesize -= dma_reserve;
|
2008-10-18 21:27:06 -06:00
|
|
|
printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
|
2007-02-10 02:43:07 -07:00
|
|
|
zone_names[0], dma_reserve);
|
2006-09-27 02:49:56 -06:00
|
|
|
}
|
|
|
|
|
2006-09-26 00:31:12 -06:00
|
|
|
if (!is_highmem_idx(j))
|
2012-12-12 14:52:12 -07:00
|
|
|
nr_kernel_pages += freesize;
|
2012-12-12 14:52:19 -07:00
|
|
|
/* Charge for highmem memmap if there are enough kernel pages */
|
|
|
|
else if (nr_kernel_pages > memmap_pages * 2)
|
|
|
|
nr_kernel_pages -= memmap_pages;
|
2012-12-12 14:52:12 -07:00
|
|
|
nr_all_pages += freesize;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-12-12 14:52:12 -07:00
|
|
|
/*
|
|
|
|
* Set an approximate value for lowmem here, it will be adjusted
|
|
|
|
* when the bootmem allocator frees pages into the buddy system.
|
|
|
|
* And all highmem pages will be managed by the buddy system.
|
|
|
|
*/
|
|
|
|
zone->managed_pages = is_highmem_idx(j) ? realsize : freesize;
|
2006-07-03 01:24:13 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
2006-09-27 02:50:08 -06:00
|
|
|
zone->node = nid;
|
2006-07-03 01:24:13 -06:00
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
zone->name = zone_names[j];
|
2016-07-28 16:45:28 -06:00
|
|
|
zone->zone_pgdat = pgdat;
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_lock_init(&zone->lock);
|
2005-10-29 19:16:53 -06:00
|
|
|
zone_seqlock_init(zone);
|
2005-10-29 19:16:50 -06:00
|
|
|
zone_pcp_init(zone);
|
mm: page_alloc: fair zone allocator policy
Each zone that holds userspace pages of one workload must be aged at a
speed proportional to the zone size. Otherwise, the time an individual
page gets to stay in memory depends on the zone it happened to be
allocated in. Asymmetry in the zone aging creates rather unpredictable
aging behavior and results in the wrong pages being reclaimed, activated
etc.
But exactly this happens right now because of the way the page allocator
and kswapd interact. The page allocator uses per-node lists of all zones
in the system, ordered by preference, when allocating a new page. When
the first iteration does not yield any results, kswapd is woken up and the
allocator retries. Due to the way kswapd reclaims zones below the high
watermark while a zone can be allocated from when it is above the low
watermark, the allocator may keep kswapd running while kswapd reclaim
ensures that the page allocator can keep allocating from the first zone in
the zonelist for extended periods of time. Meanwhile the other zones
rarely see new allocations and thus get aged much slower in comparison.
The result is that the occasional page placed in lower zones gets
relatively more time in memory, even gets promoted to the active list
after its peers have long been evicted. Meanwhile, the bulk of the
working set may be thrashing on the preferred zone even though there may
be significant amounts of memory available in the lower zones.
Even the most basic test -- repeatedly reading a file slightly bigger than
memory -- shows how broken the zone aging is. In this scenario, no single
page should be able stay in memory long enough to get referenced twice and
activated, but activation happens in spades:
$ grep active_file /proc/zoneinfo
nr_inactive_file 0
nr_active_file 0
nr_inactive_file 0
nr_active_file 8
nr_inactive_file 1582
nr_active_file 11994
$ cat data data data data >/dev/null
$ grep active_file /proc/zoneinfo
nr_inactive_file 0
nr_active_file 70
nr_inactive_file 258753
nr_active_file 443214
nr_inactive_file 149793
nr_active_file 12021
Fix this with a very simple round robin allocator. Each zone is allowed a
batch of allocations that is proportional to the zone's size, after which
it is treated as full. The batch counters are reset when all zones have
been tried and the allocator enters the slowpath and kicks off kswapd
reclaim. Allocation and reclaim is now fairly spread out to all
available/allowable zones:
$ grep active_file /proc/zoneinfo
nr_inactive_file 0
nr_active_file 0
nr_inactive_file 174
nr_active_file 4865
nr_inactive_file 53
nr_active_file 860
$ cat data data data data >/dev/null
$ grep active_file /proc/zoneinfo
nr_inactive_file 0
nr_active_file 0
nr_inactive_file 666622
nr_active_file 4988
nr_inactive_file 190969
nr_active_file 937
When zone_reclaim_mode is enabled, allocations will now spread out to all
zones on the local node, not just the first preferred zone (which on a 4G
node might be a tiny Normal zone).
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Paul Bolle <paul.bollee@gmail.com>
Cc: Zlatko Calusic <zcalusic@bitsync.net>
Tested-by: Kevin Hilman <khilman@linaro.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-09-11 15:20:47 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!size)
|
|
|
|
continue;
|
|
|
|
|
2012-05-29 16:06:31 -06:00
|
|
|
set_pageblock_order();
|
2013-02-18 10:58:02 -07:00
|
|
|
setup_usemap(pgdat, zone, zone_start_pfn, size);
|
2015-11-05 19:47:06 -07:00
|
|
|
ret = init_currently_empty_zone(zone, zone_start_pfn, size);
|
2006-06-23 03:03:10 -06:00
|
|
|
BUG_ON(ret);
|
2008-05-14 17:05:52 -06:00
|
|
|
memmap_init(size, nid, j, zone_start_pfn);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-02 15:03:33 -06:00
|
|
|
static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2015-11-10 11:09:47 -07:00
|
|
|
unsigned long __maybe_unused start = 0;
|
2015-11-05 19:48:46 -07:00
|
|
|
unsigned long __maybe_unused offset = 0;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* Skip empty nodes */
|
|
|
|
if (!pgdat->node_spanned_pages)
|
|
|
|
return;
|
|
|
|
|
[PATCH] sparsemem memory model
Sparsemem abstracts the use of discontiguous mem_maps[]. This kind of
mem_map[] is needed by discontiguous memory machines (like in the old
CONFIG_DISCONTIGMEM case) as well as memory hotplug systems. Sparsemem
replaces DISCONTIGMEM when enabled, and it is hoped that it can eventually
become a complete replacement.
A significant advantage over DISCONTIGMEM is that it's completely separated
from CONFIG_NUMA. When producing this patch, it became apparent in that NUMA
and DISCONTIG are often confused.
Another advantage is that sparse doesn't require each NUMA node's ranges to be
contiguous. It can handle overlapping ranges between nodes with no problems,
where DISCONTIGMEM currently throws away that memory.
Sparsemem uses an array to provide different pfn_to_page() translations for
each SECTION_SIZE area of physical memory. This is what allows the mem_map[]
to be chopped up.
In order to do quick pfn_to_page() operations, the section number of the page
is encoded in page->flags. Part of the sparsemem infrastructure enables
sharing of these bits more dynamically (at compile-time) between the
page_zone() and sparsemem operations. However, on 32-bit architectures, the
number of bits is quite limited, and may require growing the size of the
page->flags type in certain conditions. Several things might force this to
occur: a decrease in the SECTION_SIZE (if you want to hotplug smaller areas of
memory), an increase in the physical address space, or an increase in the
number of used page->flags.
One thing to note is that, once sparsemem is present, the NUMA node
information no longer needs to be stored in the page->flags. It might provide
speed increases on certain platforms and will be stored there if there is
room. But, if out of room, an alternate (theoretically slower) mechanism is
used.
This patch introduces CONFIG_FLATMEM. It is used in almost all cases where
there used to be an #ifndef DISCONTIG, because SPARSEMEM and DISCONTIGMEM
often have to compile out the same areas of code.
Signed-off-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Dave Hansen <haveblue@us.ibm.com>
Signed-off-by: Martin Bligh <mbligh@aracnet.com>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-23 01:07:54 -06:00
|
|
|
#ifdef CONFIG_FLAT_NODE_MEM_MAP
|
2015-11-10 11:09:47 -07:00
|
|
|
start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
|
|
|
|
offset = pgdat->node_start_pfn - start;
|
2005-04-16 16:20:36 -06:00
|
|
|
/* ia64 gets its own node_mem_map, before this, without bootmem */
|
|
|
|
if (!pgdat->node_mem_map) {
|
2015-11-10 11:09:47 -07:00
|
|
|
unsigned long size, end;
|
[PATCH] sparsemem memory model
Sparsemem abstracts the use of discontiguous mem_maps[]. This kind of
mem_map[] is needed by discontiguous memory machines (like in the old
CONFIG_DISCONTIGMEM case) as well as memory hotplug systems. Sparsemem
replaces DISCONTIGMEM when enabled, and it is hoped that it can eventually
become a complete replacement.
A significant advantage over DISCONTIGMEM is that it's completely separated
from CONFIG_NUMA. When producing this patch, it became apparent in that NUMA
and DISCONTIG are often confused.
Another advantage is that sparse doesn't require each NUMA node's ranges to be
contiguous. It can handle overlapping ranges between nodes with no problems,
where DISCONTIGMEM currently throws away that memory.
Sparsemem uses an array to provide different pfn_to_page() translations for
each SECTION_SIZE area of physical memory. This is what allows the mem_map[]
to be chopped up.
In order to do quick pfn_to_page() operations, the section number of the page
is encoded in page->flags. Part of the sparsemem infrastructure enables
sharing of these bits more dynamically (at compile-time) between the
page_zone() and sparsemem operations. However, on 32-bit architectures, the
number of bits is quite limited, and may require growing the size of the
page->flags type in certain conditions. Several things might force this to
occur: a decrease in the SECTION_SIZE (if you want to hotplug smaller areas of
memory), an increase in the physical address space, or an increase in the
number of used page->flags.
One thing to note is that, once sparsemem is present, the NUMA node
information no longer needs to be stored in the page->flags. It might provide
speed increases on certain platforms and will be stored there if there is
room. But, if out of room, an alternate (theoretically slower) mechanism is
used.
This patch introduces CONFIG_FLATMEM. It is used in almost all cases where
there used to be an #ifndef DISCONTIG, because SPARSEMEM and DISCONTIGMEM
often have to compile out the same areas of code.
Signed-off-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Dave Hansen <haveblue@us.ibm.com>
Signed-off-by: Martin Bligh <mbligh@aracnet.com>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-23 01:07:54 -06:00
|
|
|
struct page *map;
|
|
|
|
|
2006-05-20 16:00:31 -06:00
|
|
|
/*
|
|
|
|
* The zone's endpoints aren't required to be MAX_ORDER
|
|
|
|
* aligned but the node_mem_map endpoints must be in order
|
|
|
|
* for the buddy allocator to function correctly.
|
|
|
|
*/
|
2013-02-22 17:35:23 -07:00
|
|
|
end = pgdat_end_pfn(pgdat);
|
2006-05-20 16:00:31 -06:00
|
|
|
end = ALIGN(end, MAX_ORDER_NR_PAGES);
|
|
|
|
size = (end - start) * sizeof(struct page);
|
2005-06-23 01:07:39 -06:00
|
|
|
map = alloc_remap(pgdat->node_id, size);
|
|
|
|
if (!map)
|
2014-01-21 16:50:25 -07:00
|
|
|
map = memblock_virt_alloc_node_nopanic(size,
|
|
|
|
pgdat->node_id);
|
2015-11-05 19:48:46 -07:00
|
|
|
pgdat->node_mem_map = map + offset;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2007-05-31 01:40:54 -06:00
|
|
|
#ifndef CONFIG_NEED_MULTIPLE_NODES
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* With no DISCONTIG, the global mem_map is just set as node 0's
|
|
|
|
*/
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
if (pgdat == NODE_DATA(0)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
mem_map = NODE_DATA(0)->node_mem_map;
|
2015-11-05 19:48:46 -07:00
|
|
|
#if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM)
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
|
2015-11-05 19:48:46 -07:00
|
|
|
mem_map -= offset;
|
2011-12-08 11:22:09 -07:00
|
|
|
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
[PATCH] sparsemem memory model
Sparsemem abstracts the use of discontiguous mem_maps[]. This kind of
mem_map[] is needed by discontiguous memory machines (like in the old
CONFIG_DISCONTIGMEM case) as well as memory hotplug systems. Sparsemem
replaces DISCONTIGMEM when enabled, and it is hoped that it can eventually
become a complete replacement.
A significant advantage over DISCONTIGMEM is that it's completely separated
from CONFIG_NUMA. When producing this patch, it became apparent in that NUMA
and DISCONTIG are often confused.
Another advantage is that sparse doesn't require each NUMA node's ranges to be
contiguous. It can handle overlapping ranges between nodes with no problems,
where DISCONTIGMEM currently throws away that memory.
Sparsemem uses an array to provide different pfn_to_page() translations for
each SECTION_SIZE area of physical memory. This is what allows the mem_map[]
to be chopped up.
In order to do quick pfn_to_page() operations, the section number of the page
is encoded in page->flags. Part of the sparsemem infrastructure enables
sharing of these bits more dynamically (at compile-time) between the
page_zone() and sparsemem operations. However, on 32-bit architectures, the
number of bits is quite limited, and may require growing the size of the
page->flags type in certain conditions. Several things might force this to
occur: a decrease in the SECTION_SIZE (if you want to hotplug smaller areas of
memory), an increase in the physical address space, or an increase in the
number of used page->flags.
One thing to note is that, once sparsemem is present, the NUMA node
information no longer needs to be stored in the page->flags. It might provide
speed increases on certain platforms and will be stored there if there is
room. But, if out of room, an alternate (theoretically slower) mechanism is
used.
This patch introduces CONFIG_FLATMEM. It is used in almost all cases where
there used to be an #ifndef DISCONTIG, because SPARSEMEM and DISCONTIGMEM
often have to compile out the same areas of code.
Signed-off-by: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Dave Hansen <haveblue@us.ibm.com>
Signed-off-by: Martin Bligh <mbligh@aracnet.com>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-23 01:07:54 -06:00
|
|
|
#endif /* CONFIG_FLAT_NODE_MEM_MAP */
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-07-23 22:27:20 -06:00
|
|
|
void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
|
|
|
|
unsigned long node_start_pfn, unsigned long *zholes_size)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-07-23 22:27:20 -06:00
|
|
|
pg_data_t *pgdat = NODE_DATA(nid);
|
2013-07-08 16:59:52 -06:00
|
|
|
unsigned long start_pfn = 0;
|
|
|
|
unsigned long end_pfn = 0;
|
2008-07-23 22:27:20 -06:00
|
|
|
|
2012-07-31 17:46:14 -06:00
|
|
|
/* pg_data_t should be reset to zero when it's allocated */
|
2016-07-28 16:45:49 -06:00
|
|
|
WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx);
|
2012-07-31 17:46:14 -06:00
|
|
|
|
2015-06-30 15:57:02 -06:00
|
|
|
reset_deferred_meminit(pgdat);
|
2005-04-16 16:20:36 -06:00
|
|
|
pgdat->node_id = nid;
|
|
|
|
pgdat->node_start_pfn = node_start_pfn;
|
mm, vmstat: add infrastructure for per-node vmstats
Patchset: "Move LRU page reclaim from zones to nodes v9"
This series moves LRUs from the zones to the node. While this is a
current rebase, the test results were based on mmotm as of June 23rd.
Conceptually, this series is simple but there are a lot of details.
Some of the broad motivations for this are;
1. The residency of a page partially depends on what zone the page was
allocated from. This is partially combatted by the fair zone allocation
policy but that is a partial solution that introduces overhead in the
page allocator paths.
2. Currently, reclaim on node 0 behaves slightly different to node 1. For
example, direct reclaim scans in zonelist order and reclaims even if
the zone is over the high watermark regardless of the age of pages
in that LRU. Kswapd on the other hand starts reclaim on the highest
unbalanced zone. A difference in distribution of file/anon pages due
to when they were allocated results can result in a difference in
again. While the fair zone allocation policy mitigates some of the
problems here, the page reclaim results on a multi-zone node will
always be different to a single-zone node.
it was scheduled on as a result.
3. kswapd and the page allocator scan zones in the opposite order to
avoid interfering with each other but it's sensitive to timing. This
mitigates the page allocator using pages that were allocated very recently
in the ideal case but it's sensitive to timing. When kswapd is allocating
from lower zones then it's great but during the rebalancing of the highest
zone, the page allocator and kswapd interfere with each other. It's worse
if the highest zone is small and difficult to balance.
4. slab shrinkers are node-based which makes it harder to identify the exact
relationship between slab reclaim and LRU reclaim.
The reason we have zone-based reclaim is that we used to have
large highmem zones in common configurations and it was necessary
to quickly find ZONE_NORMAL pages for reclaim. Today, this is much
less of a concern as machines with lots of memory will (or should) use
64-bit kernels. Combinations of 32-bit hardware and 64-bit hardware are
rare. Machines that do use highmem should have relatively low highmem:lowmem
ratios than we worried about in the past.
Conceptually, moving to node LRUs should be easier to understand. The
page allocator plays fewer tricks to game reclaim and reclaim behaves
similarly on all nodes.
The series has been tested on a 16 core UMA machine and a 2-socket 48
core NUMA machine. The UMA results are presented in most cases as the NUMA
machine behaved similarly.
pagealloc
---------
This is a microbenchmark that shows the benefit of removing the fair zone
allocation policy. It was tested uip to order-4 but only orders 0 and 1 are
shown as the other orders were comparable.
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 nodelru-v9
Min total-odr0-1 490.00 ( 0.00%) 457.00 ( 6.73%)
Min total-odr0-2 347.00 ( 0.00%) 329.00 ( 5.19%)
Min total-odr0-4 288.00 ( 0.00%) 273.00 ( 5.21%)
Min total-odr0-8 251.00 ( 0.00%) 239.00 ( 4.78%)
Min total-odr0-16 234.00 ( 0.00%) 222.00 ( 5.13%)
Min total-odr0-32 223.00 ( 0.00%) 211.00 ( 5.38%)
Min total-odr0-64 217.00 ( 0.00%) 208.00 ( 4.15%)
Min total-odr0-128 214.00 ( 0.00%) 204.00 ( 4.67%)
Min total-odr0-256 250.00 ( 0.00%) 230.00 ( 8.00%)
Min total-odr0-512 271.00 ( 0.00%) 269.00 ( 0.74%)
Min total-odr0-1024 291.00 ( 0.00%) 282.00 ( 3.09%)
Min total-odr0-2048 303.00 ( 0.00%) 296.00 ( 2.31%)
Min total-odr0-4096 311.00 ( 0.00%) 309.00 ( 0.64%)
Min total-odr0-8192 316.00 ( 0.00%) 314.00 ( 0.63%)
Min total-odr0-16384 317.00 ( 0.00%) 315.00 ( 0.63%)
Min total-odr1-1 742.00 ( 0.00%) 712.00 ( 4.04%)
Min total-odr1-2 562.00 ( 0.00%) 530.00 ( 5.69%)
Min total-odr1-4 457.00 ( 0.00%) 433.00 ( 5.25%)
Min total-odr1-8 411.00 ( 0.00%) 381.00 ( 7.30%)
Min total-odr1-16 381.00 ( 0.00%) 356.00 ( 6.56%)
Min total-odr1-32 372.00 ( 0.00%) 346.00 ( 6.99%)
Min total-odr1-64 372.00 ( 0.00%) 343.00 ( 7.80%)
Min total-odr1-128 375.00 ( 0.00%) 351.00 ( 6.40%)
Min total-odr1-256 379.00 ( 0.00%) 351.00 ( 7.39%)
Min total-odr1-512 385.00 ( 0.00%) 355.00 ( 7.79%)
Min total-odr1-1024 386.00 ( 0.00%) 358.00 ( 7.25%)
Min total-odr1-2048 390.00 ( 0.00%) 362.00 ( 7.18%)
Min total-odr1-4096 390.00 ( 0.00%) 362.00 ( 7.18%)
Min total-odr1-8192 388.00 ( 0.00%) 363.00 ( 6.44%)
This shows a steady improvement throughout. The primary benefit is from
reduced system CPU usage which is obvious from the overall times;
4.7.0-rc4 4.7.0-rc4
mmotm-20160623nodelru-v8
User 189.19 191.80
System 2604.45 2533.56
Elapsed 2855.30 2786.39
The vmstats also showed that the fair zone allocation policy was definitely
removed as can be seen here;
4.7.0-rc3 4.7.0-rc3
mmotm-20160623 nodelru-v8
DMA32 allocs 28794729769 0
Normal allocs 48432501431 77227309877
Movable allocs 0 0
tiobench on ext4
----------------
tiobench is a benchmark that artifically benefits if old pages remain resident
while new pages get reclaimed. The fair zone allocation policy mitigates this
problem so pages age fairly. While the benchmark has problems, it is important
that tiobench performance remains constant as it implies that page aging
problems that the fair zone allocation policy fixes are not re-introduced.
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 nodelru-v9
Min PotentialReadSpeed 89.65 ( 0.00%) 90.21 ( 0.62%)
Min SeqRead-MB/sec-1 82.68 ( 0.00%) 82.01 ( -0.81%)
Min SeqRead-MB/sec-2 72.76 ( 0.00%) 72.07 ( -0.95%)
Min SeqRead-MB/sec-4 75.13 ( 0.00%) 74.92 ( -0.28%)
Min SeqRead-MB/sec-8 64.91 ( 0.00%) 65.19 ( 0.43%)
Min SeqRead-MB/sec-16 62.24 ( 0.00%) 62.22 ( -0.03%)
Min RandRead-MB/sec-1 0.88 ( 0.00%) 0.88 ( 0.00%)
Min RandRead-MB/sec-2 0.95 ( 0.00%) 0.92 ( -3.16%)
Min RandRead-MB/sec-4 1.43 ( 0.00%) 1.34 ( -6.29%)
Min RandRead-MB/sec-8 1.61 ( 0.00%) 1.60 ( -0.62%)
Min RandRead-MB/sec-16 1.80 ( 0.00%) 1.90 ( 5.56%)
Min SeqWrite-MB/sec-1 76.41 ( 0.00%) 76.85 ( 0.58%)
Min SeqWrite-MB/sec-2 74.11 ( 0.00%) 73.54 ( -0.77%)
Min SeqWrite-MB/sec-4 80.05 ( 0.00%) 80.13 ( 0.10%)
Min SeqWrite-MB/sec-8 72.88 ( 0.00%) 73.20 ( 0.44%)
Min SeqWrite-MB/sec-16 75.91 ( 0.00%) 76.44 ( 0.70%)
Min RandWrite-MB/sec-1 1.18 ( 0.00%) 1.14 ( -3.39%)
Min RandWrite-MB/sec-2 1.02 ( 0.00%) 1.03 ( 0.98%)
Min RandWrite-MB/sec-4 1.05 ( 0.00%) 0.98 ( -6.67%)
Min RandWrite-MB/sec-8 0.89 ( 0.00%) 0.92 ( 3.37%)
Min RandWrite-MB/sec-16 0.92 ( 0.00%) 0.93 ( 1.09%)
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 approx-v9
User 645.72 525.90
System 403.85 331.75
Elapsed 6795.36 6783.67
This shows that the series has little or not impact on tiobench which is
desirable and a reduction in system CPU usage. It indicates that the fair
zone allocation policy was removed in a manner that didn't reintroduce
one class of page aging bug. There were only minor differences in overall
reclaim activity
4.7.0-rc4 4.7.0-rc4
mmotm-20160623nodelru-v8
Minor Faults 645838 647465
Major Faults 573 640
Swap Ins 0 0
Swap Outs 0 0
DMA allocs 0 0
DMA32 allocs 46041453 44190646
Normal allocs 78053072 79887245
Movable allocs 0 0
Allocation stalls 24 67
Stall zone DMA 0 0
Stall zone DMA32 0 0
Stall zone Normal 0 2
Stall zone HighMem 0 0
Stall zone Movable 0 65
Direct pages scanned 10969 30609
Kswapd pages scanned 93375144 93492094
Kswapd pages reclaimed 93372243 93489370
Direct pages reclaimed 10969 30609
Kswapd efficiency 99% 99%
Kswapd velocity 13741.015 13781.934
Direct efficiency 100% 100%
Direct velocity 1.614 4.512
Percentage direct scans 0% 0%
kswapd activity was roughly comparable. There were differences in direct
reclaim activity but negligible in the context of the overall workload
(velocity of 4 pages per second with the patches applied, 1.6 pages per
second in the baseline kernel).
pgbench read-only large configuration on ext4
---------------------------------------------
pgbench is a database benchmark that can be sensitive to page reclaim
decisions. This also checks if removing the fair zone allocation policy
is safe
pgbench Transactions
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 nodelru-v8
Hmean 1 188.26 ( 0.00%) 189.78 ( 0.81%)
Hmean 5 330.66 ( 0.00%) 328.69 ( -0.59%)
Hmean 12 370.32 ( 0.00%) 380.72 ( 2.81%)
Hmean 21 368.89 ( 0.00%) 369.00 ( 0.03%)
Hmean 30 382.14 ( 0.00%) 360.89 ( -5.56%)
Hmean 32 428.87 ( 0.00%) 432.96 ( 0.95%)
Negligible differences again. As with tiobench, overall reclaim activity
was comparable.
bonnie++ on ext4
----------------
No interesting performance difference, negligible differences on reclaim
stats.
paralleldd on ext4
------------------
This workload uses varying numbers of dd instances to read large amounts of
data from disk.
4.7.0-rc3 4.7.0-rc3
mmotm-20160623 nodelru-v9
Amean Elapsd-1 186.04 ( 0.00%) 189.41 ( -1.82%)
Amean Elapsd-3 192.27 ( 0.00%) 191.38 ( 0.46%)
Amean Elapsd-5 185.21 ( 0.00%) 182.75 ( 1.33%)
Amean Elapsd-7 183.71 ( 0.00%) 182.11 ( 0.87%)
Amean Elapsd-12 180.96 ( 0.00%) 181.58 ( -0.35%)
Amean Elapsd-16 181.36 ( 0.00%) 183.72 ( -1.30%)
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 nodelru-v9
User 1548.01 1552.44
System 8609.71 8515.08
Elapsed 3587.10 3594.54
There is little or no change in performance but some drop in system CPU usage.
4.7.0-rc3 4.7.0-rc3
mmotm-20160623 nodelru-v9
Minor Faults 362662 367360
Major Faults 1204 1143
Swap Ins 22 0
Swap Outs 2855 1029
DMA allocs 0 0
DMA32 allocs 31409797 28837521
Normal allocs 46611853 49231282
Movable allocs 0 0
Direct pages scanned 0 0
Kswapd pages scanned 40845270 40869088
Kswapd pages reclaimed 40830976 40855294
Direct pages reclaimed 0 0
Kswapd efficiency 99% 99%
Kswapd velocity 11386.711 11369.769
Direct efficiency 100% 100%
Direct velocity 0.000 0.000
Percentage direct scans 0% 0%
Page writes by reclaim 2855 1029
Page writes file 0 0
Page writes anon 2855 1029
Page reclaim immediate 771 1628
Sector Reads 293312636 293536360
Sector Writes 18213568 18186480
Page rescued immediate 0 0
Slabs scanned 128257 132747
Direct inode steals 181 56
Kswapd inode steals 59 1131
It basically shows that kswapd was active at roughly the same rate in
both kernels. There was also comparable slab scanning activity and direct
reclaim was avoided in both cases. There appears to be a large difference
in numbers of inodes reclaimed but the workload has few active inodes and
is likely a timing artifact.
stutter
-------
stutter simulates a simple workload. One part uses a lot of anonymous
memory, a second measures mmap latency and a third copies a large file.
The primary metric is checking for mmap latency.
stutter
4.7.0-rc4 4.7.0-rc4
mmotm-20160623 nodelru-v8
Min mmap 16.6283 ( 0.00%) 13.4258 ( 19.26%)
1st-qrtle mmap 54.7570 ( 0.00%) 34.9121 ( 36.24%)
2nd-qrtle mmap 57.3163 ( 0.00%) 46.1147 ( 19.54%)
3rd-qrtle mmap 58.9976 ( 0.00%) 47.1882 ( 20.02%)
Max-90% mmap 59.7433 ( 0.00%) 47.4453 ( 20.58%)
Max-93% mmap 60.1298 ( 0.00%) 47.6037 ( 20.83%)
Max-95% mmap 73.4112 ( 0.00%) 82.8719 (-12.89%)
Max-99% mmap 92.8542 ( 0.00%) 88.8870 ( 4.27%)
Max mmap 1440.6569 ( 0.00%) 121.4201 ( 91.57%)
Mean mmap 59.3493 ( 0.00%) 42.2991 ( 28.73%)
Best99%Mean mmap 57.2121 ( 0.00%) 41.8207 ( 26.90%)
Best95%Mean mmap 55.9113 ( 0.00%) 39.9620 ( 28.53%)
Best90%Mean mmap 55.6199 ( 0.00%) 39.3124 ( 29.32%)
Best50%Mean mmap 53.2183 ( 0.00%) 33.1307 ( 37.75%)
Best10%Mean mmap 45.9842 ( 0.00%) 20.4040 ( 55.63%)
Best5%Mean mmap 43.2256 ( 0.00%) 17.9654 ( 58.44%)
Best1%Mean mmap 32.9388 ( 0.00%) 16.6875 ( 49.34%)
This shows a number of improvements with the worst-case outlier greatly
improved.
Some of the vmstats are interesting
4.7.0-rc4 4.7.0-rc4
mmotm-20160623nodelru-v8
Swap Ins 163 502
Swap Outs 0 0
DMA allocs 0 0
DMA32 allocs 618719206 1381662383
Normal allocs 891235743 564138421
Movable allocs 0 0
Allocation stalls 2603 1
Direct pages scanned 216787 2
Kswapd pages scanned 50719775 41778378
Kswapd pages reclaimed 41541765 41777639
Direct pages reclaimed 209159 0
Kswapd efficiency 81% 99%
Kswapd velocity 16859.554 14329.059
Direct efficiency 96% 0%
Direct velocity 72.061 0.001
Percentage direct scans 0% 0%
Page writes by reclaim 6215049 0
Page writes file 6215049 0
Page writes anon 0 0
Page reclaim immediate 70673 90
Sector Reads 81940800 81680456
Sector Writes 100158984 98816036
Page rescued immediate 0 0
Slabs scanned 1366954 22683
While this is not guaranteed in all cases, this particular test showed
a large reduction in direct reclaim activity. It's also worth noting
that no page writes were issued from reclaim context.
This series is not without its hazards. There are at least three areas
that I'm concerned with even though I could not reproduce any problems in
that area.
1. Reclaim/compaction is going to be affected because the amount of reclaim is
no longer targetted at a specific zone. Compaction works on a per-zone basis
so there is no guarantee that reclaiming a few THP's worth page pages will
have a positive impact on compaction success rates.
2. The Slab/LRU reclaim ratio is affected because the frequency the shrinkers
are called is now different. This may or may not be a problem but if it
is, it'll be because shrinkers are not called enough and some balancing
is required.
3. The anon/file reclaim ratio may be affected. Pages about to be dirtied are
distributed between zones and the fair zone allocation policy used to do
something very similar for anon. The distribution is now different but not
necessarily in any way that matters but it's still worth bearing in mind.
VM statistic counters for reclaim decisions are zone-based. If the kernel
is to reclaim on a per-node basis then we need to track per-node
statistics but there is no infrastructure for that. The most notable
change is that the old node_page_state is renamed to
sum_zone_node_page_state. The new node_page_state takes a pglist_data and
uses per-node stats but none exist yet. There is some renaming such as
vm_stat to vm_zone_stat and the addition of vm_node_stat and the renaming
of mod_state to mod_zone_state. Otherwise, this is mostly a mechanical
patch with no functional change. There is a lot of similarity between the
node and zone helpers which is unfortunate but there was no obvious way of
reusing the code and maintaining type safety.
Link: http://lkml.kernel.org/r/1467970510-21195-2-git-send-email-mgorman@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Rik van Riel <riel@surriel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-28 16:45:24 -06:00
|
|
|
pgdat->per_cpu_nodestats = NULL;
|
2013-07-08 16:59:52 -06:00
|
|
|
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
|
|
|
|
get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
|
2015-02-11 16:26:01 -07:00
|
|
|
pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
|
2015-09-08 16:04:19 -06:00
|
|
|
(u64)start_pfn << PAGE_SHIFT,
|
|
|
|
end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
|
2016-03-15 15:55:18 -06:00
|
|
|
#else
|
|
|
|
start_pfn = node_start_pfn;
|
2013-07-08 16:59:52 -06:00
|
|
|
#endif
|
|
|
|
calculate_node_totalpages(pgdat, start_pfn, end_pfn,
|
|
|
|
zones_size, zholes_size);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
alloc_node_mem_map(pgdat);
|
2008-06-01 14:15:22 -06:00
|
|
|
#ifdef CONFIG_FLAT_NODE_MEM_MAP
|
|
|
|
printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
|
|
|
|
nid, (unsigned long)pgdat,
|
|
|
|
(unsigned long)pgdat->node_mem_map);
|
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2015-09-08 15:59:50 -06:00
|
|
|
free_area_init_core(pgdat);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2011-12-08 11:22:09 -07:00
|
|
|
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
|
2007-05-23 14:57:55 -06:00
|
|
|
|
|
|
|
#if MAX_NUMNODES > 1
|
|
|
|
/*
|
|
|
|
* Figure out the number of possible node ids.
|
|
|
|
*/
|
2013-04-29 16:08:01 -06:00
|
|
|
void __init setup_nr_node_ids(void)
|
2007-05-23 14:57:55 -06:00
|
|
|
{
|
2015-09-08 15:59:48 -06:00
|
|
|
unsigned int highest;
|
2007-05-23 14:57:55 -06:00
|
|
|
|
2015-09-08 15:59:48 -06:00
|
|
|
highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
|
2007-05-23 14:57:55 -06:00
|
|
|
nr_node_ids = highest + 1;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
x86, numa: Implement pfn -> nid mapping granularity check
SPARSEMEM w/o VMEMMAP and DISCONTIGMEM, both used only on 32bit, use
sections array to map pfn to nid which is limited in granularity. If
NUMA nodes are laid out such that the mapping cannot be accurate, boot
will fail triggering BUG_ON() in mminit_verify_page_links().
On 32bit, it's 512MiB w/ PAE and SPARSEMEM. This seems to have been
granular enough until commit 2706a0bf7b (x86, NUMA: Enable
CONFIG_AMD_NUMA on 32bit too). Apparently, there is a machine which
aligns NUMA nodes to 128MiB and has only AMD NUMA but not SRAT. This
led to the following BUG_ON().
On node 0 totalpages: 2096615
DMA zone: 32 pages used for memmap
DMA zone: 0 pages reserved
DMA zone: 3927 pages, LIFO batch:0
Normal zone: 1740 pages used for memmap
Normal zone: 220978 pages, LIFO batch:31
HighMem zone: 16405 pages used for memmap
HighMem zone: 1853533 pages, LIFO batch:31
BUG: Int 6: CR2 (null)
EDI (null) ESI 00000002 EBP 00000002 ESP c1543ecc
EBX f2400000 EDX 00000006 ECX (null) EAX 00000001
err (null) EIP c16209aa CS 00000060 flg 00010002
Stack: f2400000 00220000 f7200800 c1620613 00220000 01000000 04400000 00238000
(null) f7200000 00000002 f7200b58 f7200800 c1620929 000375fe (null)
f7200b80 c16395f0 00200a02 f7200a80 (null) 000375fe 00000002 (null)
Pid: 0, comm: swapper Not tainted 2.6.39-rc5-00181-g2706a0b #17
Call Trace:
[<c136b1e5>] ? early_fault+0x2e/0x2e
[<c16209aa>] ? mminit_verify_page_links+0x12/0x42
[<c1620613>] ? memmap_init_zone+0xaf/0x10c
[<c1620929>] ? free_area_init_node+0x2b9/0x2e3
[<c1607e99>] ? free_area_init_nodes+0x3f2/0x451
[<c1601d80>] ? paging_init+0x112/0x118
[<c15f578d>] ? setup_arch+0x791/0x82f
[<c15f43d9>] ? start_kernel+0x6a/0x257
This patch implements node_map_pfn_alignment() which determines
maximum internode alignment and update numa_register_memblks() to
reject NUMA configuration if alignment exceeds the pfn -> nid mapping
granularity of the memory model as determined by PAGES_PER_SECTION.
This makes the problematic machine boot w/ flatmem by rejecting the
NUMA config and provides protection against crazy NUMA configurations.
Signed-off-by: Tejun Heo <tj@kernel.org>
Link: http://lkml.kernel.org/r/20110712074534.GB2872@htj.dyndns.org
LKML-Reference: <20110628174613.GP478@escobedo.osrc.amd.com>
Reported-and-Tested-by: Hans Rosenfeld <hans.rosenfeld@amd.com>
Cc: Conny Seidel <conny.seidel@amd.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2011-07-12 01:45:34 -06:00
|
|
|
/**
|
|
|
|
* node_map_pfn_alignment - determine the maximum internode alignment
|
|
|
|
*
|
|
|
|
* This function should be called after node map is populated and sorted.
|
|
|
|
* It calculates the maximum power of two alignment which can distinguish
|
|
|
|
* all the nodes.
|
|
|
|
*
|
|
|
|
* For example, if all nodes are 1GiB and aligned to 1GiB, the return value
|
|
|
|
* would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
|
|
|
|
* nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
|
|
|
|
* shifted, 1GiB is enough and this function will indicate so.
|
|
|
|
*
|
|
|
|
* This is used to test whether pfn -> nid mapping of the chosen memory
|
|
|
|
* model has fine enough granularity to avoid incorrect mapping for the
|
|
|
|
* populated node map.
|
|
|
|
*
|
|
|
|
* Returns the determined alignment in pfn's. 0 if there is no alignment
|
|
|
|
* requirement (single node).
|
|
|
|
*/
|
|
|
|
unsigned long __init node_map_pfn_alignment(void)
|
|
|
|
{
|
|
|
|
unsigned long accl_mask = 0, last_end = 0;
|
2011-07-12 02:46:30 -06:00
|
|
|
unsigned long start, end, mask;
|
x86, numa: Implement pfn -> nid mapping granularity check
SPARSEMEM w/o VMEMMAP and DISCONTIGMEM, both used only on 32bit, use
sections array to map pfn to nid which is limited in granularity. If
NUMA nodes are laid out such that the mapping cannot be accurate, boot
will fail triggering BUG_ON() in mminit_verify_page_links().
On 32bit, it's 512MiB w/ PAE and SPARSEMEM. This seems to have been
granular enough until commit 2706a0bf7b (x86, NUMA: Enable
CONFIG_AMD_NUMA on 32bit too). Apparently, there is a machine which
aligns NUMA nodes to 128MiB and has only AMD NUMA but not SRAT. This
led to the following BUG_ON().
On node 0 totalpages: 2096615
DMA zone: 32 pages used for memmap
DMA zone: 0 pages reserved
DMA zone: 3927 pages, LIFO batch:0
Normal zone: 1740 pages used for memmap
Normal zone: 220978 pages, LIFO batch:31
HighMem zone: 16405 pages used for memmap
HighMem zone: 1853533 pages, LIFO batch:31
BUG: Int 6: CR2 (null)
EDI (null) ESI 00000002 EBP 00000002 ESP c1543ecc
EBX f2400000 EDX 00000006 ECX (null) EAX 00000001
err (null) EIP c16209aa CS 00000060 flg 00010002
Stack: f2400000 00220000 f7200800 c1620613 00220000 01000000 04400000 00238000
(null) f7200000 00000002 f7200b58 f7200800 c1620929 000375fe (null)
f7200b80 c16395f0 00200a02 f7200a80 (null) 000375fe 00000002 (null)
Pid: 0, comm: swapper Not tainted 2.6.39-rc5-00181-g2706a0b #17
Call Trace:
[<c136b1e5>] ? early_fault+0x2e/0x2e
[<c16209aa>] ? mminit_verify_page_links+0x12/0x42
[<c1620613>] ? memmap_init_zone+0xaf/0x10c
[<c1620929>] ? free_area_init_node+0x2b9/0x2e3
[<c1607e99>] ? free_area_init_nodes+0x3f2/0x451
[<c1601d80>] ? paging_init+0x112/0x118
[<c15f578d>] ? setup_arch+0x791/0x82f
[<c15f43d9>] ? start_kernel+0x6a/0x257
This patch implements node_map_pfn_alignment() which determines
maximum internode alignment and update numa_register_memblks() to
reject NUMA configuration if alignment exceeds the pfn -> nid mapping
granularity of the memory model as determined by PAGES_PER_SECTION.
This makes the problematic machine boot w/ flatmem by rejecting the
NUMA config and provides protection against crazy NUMA configurations.
Signed-off-by: Tejun Heo <tj@kernel.org>
Link: http://lkml.kernel.org/r/20110712074534.GB2872@htj.dyndns.org
LKML-Reference: <20110628174613.GP478@escobedo.osrc.amd.com>
Reported-and-Tested-by: Hans Rosenfeld <hans.rosenfeld@amd.com>
Cc: Conny Seidel <conny.seidel@amd.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2011-07-12 01:45:34 -06:00
|
|
|
int last_nid = -1;
|
2011-07-12 02:46:30 -06:00
|
|
|
int i, nid;
|
x86, numa: Implement pfn -> nid mapping granularity check
SPARSEMEM w/o VMEMMAP and DISCONTIGMEM, both used only on 32bit, use
sections array to map pfn to nid which is limited in granularity. If
NUMA nodes are laid out such that the mapping cannot be accurate, boot
will fail triggering BUG_ON() in mminit_verify_page_links().
On 32bit, it's 512MiB w/ PAE and SPARSEMEM. This seems to have been
granular enough until commit 2706a0bf7b (x86, NUMA: Enable
CONFIG_AMD_NUMA on 32bit too). Apparently, there is a machine which
aligns NUMA nodes to 128MiB and has only AMD NUMA but not SRAT. This
led to the following BUG_ON().
On node 0 totalpages: 2096615
DMA zone: 32 pages used for memmap
DMA zone: 0 pages reserved
DMA zone: 3927 pages, LIFO batch:0
Normal zone: 1740 pages used for memmap
Normal zone: 220978 pages, LIFO batch:31
HighMem zone: 16405 pages used for memmap
HighMem zone: 1853533 pages, LIFO batch:31
BUG: Int 6: CR2 (null)
EDI (null) ESI 00000002 EBP 00000002 ESP c1543ecc
EBX f2400000 EDX 00000006 ECX (null) EAX 00000001
err (null) EIP c16209aa CS 00000060 flg 00010002
Stack: f2400000 00220000 f7200800 c1620613 00220000 01000000 04400000 00238000
(null) f7200000 00000002 f7200b58 f7200800 c1620929 000375fe (null)
f7200b80 c16395f0 00200a02 f7200a80 (null) 000375fe 00000002 (null)
Pid: 0, comm: swapper Not tainted 2.6.39-rc5-00181-g2706a0b #17
Call Trace:
[<c136b1e5>] ? early_fault+0x2e/0x2e
[<c16209aa>] ? mminit_verify_page_links+0x12/0x42
[<c1620613>] ? memmap_init_zone+0xaf/0x10c
[<c1620929>] ? free_area_init_node+0x2b9/0x2e3
[<c1607e99>] ? free_area_init_nodes+0x3f2/0x451
[<c1601d80>] ? paging_init+0x112/0x118
[<c15f578d>] ? setup_arch+0x791/0x82f
[<c15f43d9>] ? start_kernel+0x6a/0x257
This patch implements node_map_pfn_alignment() which determines
maximum internode alignment and update numa_register_memblks() to
reject NUMA configuration if alignment exceeds the pfn -> nid mapping
granularity of the memory model as determined by PAGES_PER_SECTION.
This makes the problematic machine boot w/ flatmem by rejecting the
NUMA config and provides protection against crazy NUMA configurations.
Signed-off-by: Tejun Heo <tj@kernel.org>
Link: http://lkml.kernel.org/r/20110712074534.GB2872@htj.dyndns.org
LKML-Reference: <20110628174613.GP478@escobedo.osrc.amd.com>
Reported-and-Tested-by: Hans Rosenfeld <hans.rosenfeld@amd.com>
Cc: Conny Seidel <conny.seidel@amd.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2011-07-12 01:45:34 -06:00
|
|
|
|
2011-07-12 02:46:30 -06:00
|
|
|
for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
|
x86, numa: Implement pfn -> nid mapping granularity check
SPARSEMEM w/o VMEMMAP and DISCONTIGMEM, both used only on 32bit, use
sections array to map pfn to nid which is limited in granularity. If
NUMA nodes are laid out such that the mapping cannot be accurate, boot
will fail triggering BUG_ON() in mminit_verify_page_links().
On 32bit, it's 512MiB w/ PAE and SPARSEMEM. This seems to have been
granular enough until commit 2706a0bf7b (x86, NUMA: Enable
CONFIG_AMD_NUMA on 32bit too). Apparently, there is a machine which
aligns NUMA nodes to 128MiB and has only AMD NUMA but not SRAT. This
led to the following BUG_ON().
On node 0 totalpages: 2096615
DMA zone: 32 pages used for memmap
DMA zone: 0 pages reserved
DMA zone: 3927 pages, LIFO batch:0
Normal zone: 1740 pages used for memmap
Normal zone: 220978 pages, LIFO batch:31
HighMem zone: 16405 pages used for memmap
HighMem zone: 1853533 pages, LIFO batch:31
BUG: Int 6: CR2 (null)
EDI (null) ESI 00000002 EBP 00000002 ESP c1543ecc
EBX f2400000 EDX 00000006 ECX (null) EAX 00000001
err (null) EIP c16209aa CS 00000060 flg 00010002
Stack: f2400000 00220000 f7200800 c1620613 00220000 01000000 04400000 00238000
(null) f7200000 00000002 f7200b58 f7200800 c1620929 000375fe (null)
f7200b80 c16395f0 00200a02 f7200a80 (null) 000375fe 00000002 (null)
Pid: 0, comm: swapper Not tainted 2.6.39-rc5-00181-g2706a0b #17
Call Trace:
[<c136b1e5>] ? early_fault+0x2e/0x2e
[<c16209aa>] ? mminit_verify_page_links+0x12/0x42
[<c1620613>] ? memmap_init_zone+0xaf/0x10c
[<c1620929>] ? free_area_init_node+0x2b9/0x2e3
[<c1607e99>] ? free_area_init_nodes+0x3f2/0x451
[<c1601d80>] ? paging_init+0x112/0x118
[<c15f578d>] ? setup_arch+0x791/0x82f
[<c15f43d9>] ? start_kernel+0x6a/0x257
This patch implements node_map_pfn_alignment() which determines
maximum internode alignment and update numa_register_memblks() to
reject NUMA configuration if alignment exceeds the pfn -> nid mapping
granularity of the memory model as determined by PAGES_PER_SECTION.
This makes the problematic machine boot w/ flatmem by rejecting the
NUMA config and provides protection against crazy NUMA configurations.
Signed-off-by: Tejun Heo <tj@kernel.org>
Link: http://lkml.kernel.org/r/20110712074534.GB2872@htj.dyndns.org
LKML-Reference: <20110628174613.GP478@escobedo.osrc.amd.com>
Reported-and-Tested-by: Hans Rosenfeld <hans.rosenfeld@amd.com>
Cc: Conny Seidel <conny.seidel@amd.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2011-07-12 01:45:34 -06:00
|
|
|
if (!start || last_nid < 0 || last_nid == nid) {
|
|
|
|
last_nid = nid;
|
|
|
|
last_end = end;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Start with a mask granular enough to pin-point to the
|
|
|
|
* start pfn and tick off bits one-by-one until it becomes
|
|
|
|
* too coarse to separate the current node from the last.
|
|
|
|
*/
|
|
|
|
mask = ~((1 << __ffs(start)) - 1);
|
|
|
|
while (mask && last_end <= (start & (mask << 1)))
|
|
|
|
mask <<= 1;
|
|
|
|
|
|
|
|
/* accumulate all internode masks */
|
|
|
|
accl_mask |= mask;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* convert mask to number of pages */
|
|
|
|
return ~accl_mask + 1;
|
|
|
|
}
|
|
|
|
|
2007-02-10 02:42:57 -07:00
|
|
|
/* Find the lowest pfn for a node */
|
2008-07-23 22:28:12 -06:00
|
|
|
static unsigned long __init find_min_pfn_for_node(int nid)
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
{
|
2007-02-10 02:42:57 -07:00
|
|
|
unsigned long min_pfn = ULONG_MAX;
|
2011-07-12 02:46:30 -06:00
|
|
|
unsigned long start_pfn;
|
|
|
|
int i;
|
2006-11-23 05:01:41 -07:00
|
|
|
|
2011-07-12 02:46:30 -06:00
|
|
|
for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
|
|
|
|
min_pfn = min(min_pfn, start_pfn);
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
2007-02-10 02:42:57 -07:00
|
|
|
if (min_pfn == ULONG_MAX) {
|
2016-03-17 15:19:50 -06:00
|
|
|
pr_warn("Could not find start_pfn for node %d\n", nid);
|
2007-02-10 02:42:57 -07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return min_pfn;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* find_min_pfn_with_active_regions - Find the minimum PFN registered
|
|
|
|
*
|
|
|
|
* It returns the minimum PFN based on information provided via
|
2014-06-04 17:10:53 -06:00
|
|
|
* memblock_set_node().
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*/
|
|
|
|
unsigned long __init find_min_pfn_with_active_regions(void)
|
|
|
|
{
|
|
|
|
return find_min_pfn_for_node(MAX_NUMNODES);
|
|
|
|
}
|
|
|
|
|
2007-10-16 02:25:39 -06:00
|
|
|
/*
|
|
|
|
* early_calculate_totalpages()
|
|
|
|
* Sum pages in active regions for movable zone.
|
2012-12-12 14:51:46 -07:00
|
|
|
* Populate N_MEMORY for calculating usable_nodes.
|
2007-10-16 02:25:39 -06:00
|
|
|
*/
|
2007-10-16 02:26:03 -06:00
|
|
|
static unsigned long __init early_calculate_totalpages(void)
|
2007-07-17 05:03:15 -06:00
|
|
|
{
|
|
|
|
unsigned long totalpages = 0;
|
2011-07-12 02:46:30 -06:00
|
|
|
unsigned long start_pfn, end_pfn;
|
|
|
|
int i, nid;
|
|
|
|
|
|
|
|
for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
|
|
|
|
unsigned long pages = end_pfn - start_pfn;
|
2007-07-17 05:03:15 -06:00
|
|
|
|
2007-10-16 02:25:39 -06:00
|
|
|
totalpages += pages;
|
|
|
|
if (pages)
|
2012-12-12 14:51:46 -07:00
|
|
|
node_set_state(nid, N_MEMORY);
|
2007-10-16 02:25:39 -06:00
|
|
|
}
|
2013-09-11 15:20:34 -06:00
|
|
|
return totalpages;
|
2007-07-17 05:03:15 -06:00
|
|
|
}
|
|
|
|
|
2007-07-17 05:03:12 -06:00
|
|
|
/*
|
|
|
|
* Find the PFN the Movable zone begins in each node. Kernel memory
|
|
|
|
* is spread evenly between nodes as long as the nodes have enough
|
|
|
|
* memory. When they don't, some nodes will have more kernelcore than
|
|
|
|
* others
|
|
|
|
*/
|
2012-03-21 17:34:15 -06:00
|
|
|
static void __init find_zone_movable_pfns_for_nodes(void)
|
2007-07-17 05:03:12 -06:00
|
|
|
{
|
|
|
|
int i, nid;
|
|
|
|
unsigned long usable_startpfn;
|
|
|
|
unsigned long kernelcore_node, kernelcore_remaining;
|
2009-06-30 12:41:37 -06:00
|
|
|
/* save the state before borrow the nodemask */
|
2012-12-12 14:51:46 -07:00
|
|
|
nodemask_t saved_node_state = node_states[N_MEMORY];
|
2007-10-16 02:25:39 -06:00
|
|
|
unsigned long totalpages = early_calculate_totalpages();
|
2012-12-12 14:51:46 -07:00
|
|
|
int usable_nodes = nodes_weight(node_states[N_MEMORY]);
|
2014-04-07 16:37:52 -06:00
|
|
|
struct memblock_region *r;
|
2014-01-21 16:49:38 -07:00
|
|
|
|
|
|
|
/* Need to find movable_zone earlier when movable_node is specified. */
|
|
|
|
find_usable_zone_for_movable();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If movable_node is specified, ignore kernelcore and movablecore
|
|
|
|
* options.
|
|
|
|
*/
|
|
|
|
if (movable_node_is_enabled()) {
|
2014-04-07 16:37:52 -06:00
|
|
|
for_each_memblock(memory, r) {
|
|
|
|
if (!memblock_is_hotpluggable(r))
|
2014-01-21 16:49:38 -07:00
|
|
|
continue;
|
|
|
|
|
2014-04-07 16:37:52 -06:00
|
|
|
nid = r->nid;
|
2014-01-21 16:49:38 -07:00
|
|
|
|
2014-04-07 16:37:52 -06:00
|
|
|
usable_startpfn = PFN_DOWN(r->base);
|
2014-01-21 16:49:38 -07:00
|
|
|
zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
|
|
|
|
min(usable_startpfn, zone_movable_pfn[nid]) :
|
|
|
|
usable_startpfn;
|
|
|
|
}
|
|
|
|
|
|
|
|
goto out2;
|
|
|
|
}
|
2007-07-17 05:03:12 -06:00
|
|
|
|
2016-03-15 15:55:22 -06:00
|
|
|
/*
|
|
|
|
* If kernelcore=mirror is specified, ignore movablecore option
|
|
|
|
*/
|
|
|
|
if (mirrored_kernelcore) {
|
|
|
|
bool mem_below_4gb_not_mirrored = false;
|
|
|
|
|
|
|
|
for_each_memblock(memory, r) {
|
|
|
|
if (memblock_is_mirror(r))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
nid = r->nid;
|
|
|
|
|
|
|
|
usable_startpfn = memblock_region_memory_base_pfn(r);
|
|
|
|
|
|
|
|
if (usable_startpfn < 0x100000) {
|
|
|
|
mem_below_4gb_not_mirrored = true;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
|
|
|
|
min(usable_startpfn, zone_movable_pfn[nid]) :
|
|
|
|
usable_startpfn;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (mem_below_4gb_not_mirrored)
|
|
|
|
pr_warn("This configuration results in unmirrored kernel memory.");
|
|
|
|
|
|
|
|
goto out2;
|
|
|
|
}
|
|
|
|
|
2007-07-17 05:03:15 -06:00
|
|
|
/*
|
2014-01-21 16:49:38 -07:00
|
|
|
* If movablecore=nn[KMG] was specified, calculate what size of
|
2007-07-17 05:03:15 -06:00
|
|
|
* kernelcore that corresponds so that memory usable for
|
|
|
|
* any allocation type is evenly spread. If both kernelcore
|
|
|
|
* and movablecore are specified, then the value of kernelcore
|
|
|
|
* will be used for required_kernelcore if it's greater than
|
|
|
|
* what movablecore would have allowed.
|
|
|
|
*/
|
|
|
|
if (required_movablecore) {
|
|
|
|
unsigned long corepages;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Round-up so that ZONE_MOVABLE is at least as large as what
|
|
|
|
* was requested by the user
|
|
|
|
*/
|
|
|
|
required_movablecore =
|
|
|
|
roundup(required_movablecore, MAX_ORDER_NR_PAGES);
|
2015-11-05 19:48:11 -07:00
|
|
|
required_movablecore = min(totalpages, required_movablecore);
|
2007-07-17 05:03:15 -06:00
|
|
|
corepages = totalpages - required_movablecore;
|
|
|
|
|
|
|
|
required_kernelcore = max(required_kernelcore, corepages);
|
|
|
|
}
|
|
|
|
|
2015-11-05 19:48:56 -07:00
|
|
|
/*
|
|
|
|
* If kernelcore was not specified or kernelcore size is larger
|
|
|
|
* than totalpages, there is no ZONE_MOVABLE.
|
|
|
|
*/
|
|
|
|
if (!required_kernelcore || required_kernelcore >= totalpages)
|
2009-06-30 12:41:37 -06:00
|
|
|
goto out;
|
2007-07-17 05:03:12 -06:00
|
|
|
|
|
|
|
/* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
|
|
|
|
usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
|
|
|
|
|
|
|
|
restart:
|
|
|
|
/* Spread kernelcore memory as evenly as possible throughout nodes */
|
|
|
|
kernelcore_node = required_kernelcore / usable_nodes;
|
2012-12-12 14:51:46 -07:00
|
|
|
for_each_node_state(nid, N_MEMORY) {
|
2011-07-12 02:46:30 -06:00
|
|
|
unsigned long start_pfn, end_pfn;
|
|
|
|
|
2007-07-17 05:03:12 -06:00
|
|
|
/*
|
|
|
|
* Recalculate kernelcore_node if the division per node
|
|
|
|
* now exceeds what is necessary to satisfy the requested
|
|
|
|
* amount of memory for the kernel
|
|
|
|
*/
|
|
|
|
if (required_kernelcore < kernelcore_node)
|
|
|
|
kernelcore_node = required_kernelcore / usable_nodes;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* As the map is walked, we track how much memory is usable
|
|
|
|
* by the kernel using kernelcore_remaining. When it is
|
|
|
|
* 0, the rest of the node is usable by ZONE_MOVABLE
|
|
|
|
*/
|
|
|
|
kernelcore_remaining = kernelcore_node;
|
|
|
|
|
|
|
|
/* Go through each range of PFNs within this node */
|
2011-07-12 02:46:30 -06:00
|
|
|
for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
|
2007-07-17 05:03:12 -06:00
|
|
|
unsigned long size_pages;
|
|
|
|
|
2011-07-12 02:46:30 -06:00
|
|
|
start_pfn = max(start_pfn, zone_movable_pfn[nid]);
|
2007-07-17 05:03:12 -06:00
|
|
|
if (start_pfn >= end_pfn)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Account for what is only usable for kernelcore */
|
|
|
|
if (start_pfn < usable_startpfn) {
|
|
|
|
unsigned long kernel_pages;
|
|
|
|
kernel_pages = min(end_pfn, usable_startpfn)
|
|
|
|
- start_pfn;
|
|
|
|
|
|
|
|
kernelcore_remaining -= min(kernel_pages,
|
|
|
|
kernelcore_remaining);
|
|
|
|
required_kernelcore -= min(kernel_pages,
|
|
|
|
required_kernelcore);
|
|
|
|
|
|
|
|
/* Continue if range is now fully accounted */
|
|
|
|
if (end_pfn <= usable_startpfn) {
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Push zone_movable_pfn to the end so
|
|
|
|
* that if we have to rebalance
|
|
|
|
* kernelcore across nodes, we will
|
|
|
|
* not double account here
|
|
|
|
*/
|
|
|
|
zone_movable_pfn[nid] = end_pfn;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
start_pfn = usable_startpfn;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The usable PFN range for ZONE_MOVABLE is from
|
|
|
|
* start_pfn->end_pfn. Calculate size_pages as the
|
|
|
|
* number of pages used as kernelcore
|
|
|
|
*/
|
|
|
|
size_pages = end_pfn - start_pfn;
|
|
|
|
if (size_pages > kernelcore_remaining)
|
|
|
|
size_pages = kernelcore_remaining;
|
|
|
|
zone_movable_pfn[nid] = start_pfn + size_pages;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Some kernelcore has been met, update counts and
|
|
|
|
* break if the kernelcore for this node has been
|
2013-09-11 15:20:34 -06:00
|
|
|
* satisfied
|
2007-07-17 05:03:12 -06:00
|
|
|
*/
|
|
|
|
required_kernelcore -= min(required_kernelcore,
|
|
|
|
size_pages);
|
|
|
|
kernelcore_remaining -= size_pages;
|
|
|
|
if (!kernelcore_remaining)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there is still required_kernelcore, we do another pass with one
|
|
|
|
* less node in the count. This will push zone_movable_pfn[nid] further
|
|
|
|
* along on the nodes that still have memory until kernelcore is
|
2013-09-11 15:20:34 -06:00
|
|
|
* satisfied
|
2007-07-17 05:03:12 -06:00
|
|
|
*/
|
|
|
|
usable_nodes--;
|
|
|
|
if (usable_nodes && required_kernelcore > usable_nodes)
|
|
|
|
goto restart;
|
|
|
|
|
2014-01-21 16:49:38 -07:00
|
|
|
out2:
|
2007-07-17 05:03:12 -06:00
|
|
|
/* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
|
|
|
|
for (nid = 0; nid < MAX_NUMNODES; nid++)
|
|
|
|
zone_movable_pfn[nid] =
|
|
|
|
roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
|
2009-06-30 12:41:37 -06:00
|
|
|
|
x86, ACPI, mm: Revert movablemem_map support
Tim found:
WARNING: at arch/x86/kernel/smpboot.c:324 topology_sane.isra.2+0x6f/0x80()
Hardware name: S2600CP
sched: CPU #1's llc-sibling CPU #0 is not on the same node! [node: 1 != 0]. Ignoring dependency.
smpboot: Booting Node 1, Processors #1
Modules linked in:
Pid: 0, comm: swapper/1 Not tainted 3.9.0-0-generic #1
Call Trace:
set_cpu_sibling_map+0x279/0x449
start_secondary+0x11d/0x1e5
Don Morris reproduced on a HP z620 workstation, and bisected it to
commit e8d195525809 ("acpi, memory-hotplug: parse SRAT before memblock
is ready")
It turns out movable_map has some problems, and it breaks several things
1. numa_init is called several times, NOT just for srat. so those
nodes_clear(numa_nodes_parsed)
memset(&numa_meminfo, 0, sizeof(numa_meminfo))
can not be just removed. Need to consider sequence is: numaq, srat, amd, dummy.
and make fall back path working.
2. simply split acpi_numa_init to early_parse_srat.
a. that early_parse_srat is NOT called for ia64, so you break ia64.
b. for (i = 0; i < MAX_LOCAL_APIC; i++)
set_apicid_to_node(i, NUMA_NO_NODE)
still left in numa_init. So it will just clear result from early_parse_srat.
it should be moved before that....
c. it breaks ACPI_TABLE_OVERIDE...as the acpi table scan is moved
early before override from INITRD is settled.
3. that patch TITLE is total misleading, there is NO x86 in the title,
but it changes critical x86 code. It caused x86 guys did not
pay attention to find the problem early. Those patches really should
be routed via tip/x86/mm.
4. after that commit, following range can not use movable ram:
a. real_mode code.... well..funny, legacy Node0 [0,1M) could be hot-removed?
b. initrd... it will be freed after booting, so it could be on movable...
c. crashkernel for kdump...: looks like we can not put kdump kernel above 4G
anymore.
d. init_mem_mapping: can not put page table high anymore.
e. initmem_init: vmemmap can not be high local node anymore. That is
not good.
If node is hotplugable, the mem related range like page table and
vmemmap could be on the that node without problem and should be on that
node.
We have workaround patch that could fix some problems, but some can not
be fixed.
So just remove that offending commit and related ones including:
f7210e6c4ac7 ("mm/memblock.c: use CONFIG_HAVE_MEMBLOCK_NODE_MAP to
protect movablecore_map in memblock_overlaps_region().")
01a178a94e8e ("acpi, memory-hotplug: support getting hotplug info from
SRAT")
27168d38fa20 ("acpi, memory-hotplug: extend movablemem_map ranges to
the end of node")
e8d195525809 ("acpi, memory-hotplug: parse SRAT before memblock is
ready")
fb06bc8e5f42 ("page_alloc: bootmem limit with movablecore_map")
42f47e27e761 ("page_alloc: make movablemem_map have higher priority")
6981ec31146c ("page_alloc: introduce zone_movable_limit[] to keep
movable limit for nodes")
34b71f1e04fc ("page_alloc: add movable_memmap kernel parameter")
4d59a75125d5 ("x86: get pg_data_t's memory from other node")
Later we should have patches that will make sure kernel put page table
and vmemmap on local node ram instead of push them down to node0. Also
need to find way to put other kernel used ram to local node ram.
Reported-by: Tim Gardner <tim.gardner@canonical.com>
Reported-by: Don Morris <don.morris@hp.com>
Bisected-by: Don Morris <don.morris@hp.com>
Tested-by: Don Morris <don.morris@hp.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Thomas Renninger <trenn@suse.de>
Cc: Tejun Heo <tj@kernel.org>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-03-01 15:51:27 -07:00
|
|
|
out:
|
2009-06-30 12:41:37 -06:00
|
|
|
/* restore the node_state */
|
2012-12-12 14:51:46 -07:00
|
|
|
node_states[N_MEMORY] = saved_node_state;
|
2007-07-17 05:03:12 -06:00
|
|
|
}
|
|
|
|
|
2012-12-12 14:51:46 -07:00
|
|
|
/* Any regular or high memory on that node ? */
|
|
|
|
static void check_for_memory(pg_data_t *pgdat, int nid)
|
2007-10-16 02:25:39 -06:00
|
|
|
{
|
|
|
|
enum zone_type zone_type;
|
|
|
|
|
2012-12-12 14:51:46 -07:00
|
|
|
if (N_MEMORY == N_NORMAL_MEMORY)
|
|
|
|
return;
|
|
|
|
|
|
|
|
for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
|
2007-10-16 02:25:39 -06:00
|
|
|
struct zone *zone = &pgdat->node_zones[zone_type];
|
2013-11-12 16:07:20 -07:00
|
|
|
if (populated_zone(zone)) {
|
2012-12-12 14:51:46 -07:00
|
|
|
node_set_state(nid, N_HIGH_MEMORY);
|
|
|
|
if (N_NORMAL_MEMORY != N_HIGH_MEMORY &&
|
|
|
|
zone_type <= ZONE_NORMAL)
|
|
|
|
node_set_state(nid, N_NORMAL_MEMORY);
|
2012-01-12 18:19:07 -07:00
|
|
|
break;
|
|
|
|
}
|
2007-10-16 02:25:39 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
/**
|
|
|
|
* free_area_init_nodes - Initialise all pg_data_t and zone data
|
2006-10-04 03:15:25 -06:00
|
|
|
* @max_zone_pfn: an array of max PFNs for each zone
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
*
|
|
|
|
* This will call free_area_init_node() for each active node in the system.
|
2014-06-04 17:10:53 -06:00
|
|
|
* Using the page ranges provided by memblock_set_node(), the size of each
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
* zone in each node and their holes is calculated. If the maximum PFN
|
|
|
|
* between two adjacent zones match, it is assumed that the zone is empty.
|
|
|
|
* For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
|
|
|
|
* that arch_max_dma32_pfn has no pages. It is also assumed that a zone
|
|
|
|
* starts where the previous one ended. For example, ZONE_DMA32 starts
|
|
|
|
* at arch_max_dma_pfn.
|
|
|
|
*/
|
|
|
|
void __init free_area_init_nodes(unsigned long *max_zone_pfn)
|
|
|
|
{
|
2011-07-12 02:46:30 -06:00
|
|
|
unsigned long start_pfn, end_pfn;
|
|
|
|
int i, nid;
|
2007-02-10 02:42:57 -07:00
|
|
|
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
/* Record where the zone boundaries are */
|
|
|
|
memset(arch_zone_lowest_possible_pfn, 0,
|
|
|
|
sizeof(arch_zone_lowest_possible_pfn));
|
|
|
|
memset(arch_zone_highest_possible_pfn, 0,
|
|
|
|
sizeof(arch_zone_highest_possible_pfn));
|
2016-07-26 16:22:17 -06:00
|
|
|
|
|
|
|
start_pfn = find_min_pfn_with_active_regions();
|
|
|
|
|
|
|
|
for (i = 0; i < MAX_NR_ZONES; i++) {
|
2007-07-17 05:03:12 -06:00
|
|
|
if (i == ZONE_MOVABLE)
|
|
|
|
continue;
|
2016-07-26 16:22:17 -06:00
|
|
|
|
|
|
|
end_pfn = max(max_zone_pfn[i], start_pfn);
|
|
|
|
arch_zone_lowest_possible_pfn[i] = start_pfn;
|
|
|
|
arch_zone_highest_possible_pfn[i] = end_pfn;
|
|
|
|
|
|
|
|
start_pfn = end_pfn;
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
2007-07-17 05:03:12 -06:00
|
|
|
arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
|
|
|
|
arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
|
|
|
|
|
|
|
|
/* Find the PFNs that ZONE_MOVABLE begins at in each node */
|
|
|
|
memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
|
2012-03-21 17:34:15 -06:00
|
|
|
find_zone_movable_pfns_for_nodes();
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
|
|
|
/* Print out the zone ranges */
|
2014-12-10 16:42:53 -07:00
|
|
|
pr_info("Zone ranges:\n");
|
2007-07-17 05:03:12 -06:00
|
|
|
for (i = 0; i < MAX_NR_ZONES; i++) {
|
|
|
|
if (i == ZONE_MOVABLE)
|
|
|
|
continue;
|
2014-12-10 16:42:53 -07:00
|
|
|
pr_info(" %-8s ", zone_names[i]);
|
2010-03-05 14:42:14 -07:00
|
|
|
if (arch_zone_lowest_possible_pfn[i] ==
|
|
|
|
arch_zone_highest_possible_pfn[i])
|
2014-12-10 16:42:53 -07:00
|
|
|
pr_cont("empty\n");
|
2010-03-05 14:42:14 -07:00
|
|
|
else
|
2015-02-11 16:26:01 -07:00
|
|
|
pr_cont("[mem %#018Lx-%#018Lx]\n",
|
|
|
|
(u64)arch_zone_lowest_possible_pfn[i]
|
|
|
|
<< PAGE_SHIFT,
|
|
|
|
((u64)arch_zone_highest_possible_pfn[i]
|
2012-05-29 16:06:30 -06:00
|
|
|
<< PAGE_SHIFT) - 1);
|
2007-07-17 05:03:12 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Print out the PFNs ZONE_MOVABLE begins at in each node */
|
2014-12-10 16:42:53 -07:00
|
|
|
pr_info("Movable zone start for each node\n");
|
2007-07-17 05:03:12 -06:00
|
|
|
for (i = 0; i < MAX_NUMNODES; i++) {
|
|
|
|
if (zone_movable_pfn[i])
|
2015-02-11 16:26:01 -07:00
|
|
|
pr_info(" Node %d: %#018Lx\n", i,
|
|
|
|
(u64)zone_movable_pfn[i] << PAGE_SHIFT);
|
2007-07-17 05:03:12 -06:00
|
|
|
}
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
2012-10-08 17:32:24 -06:00
|
|
|
/* Print out the early node map */
|
2014-12-10 16:42:53 -07:00
|
|
|
pr_info("Early memory node ranges\n");
|
2011-07-12 02:46:30 -06:00
|
|
|
for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
|
2015-02-11 16:26:01 -07:00
|
|
|
pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
|
|
|
|
(u64)start_pfn << PAGE_SHIFT,
|
|
|
|
((u64)end_pfn << PAGE_SHIFT) - 1);
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
|
|
|
/* Initialise every node */
|
2008-07-23 22:26:51 -06:00
|
|
|
mminit_verify_pageflags_layout();
|
2007-02-20 14:57:52 -07:00
|
|
|
setup_nr_node_ids();
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
for_each_online_node(nid) {
|
|
|
|
pg_data_t *pgdat = NODE_DATA(nid);
|
2008-07-23 22:27:20 -06:00
|
|
|
free_area_init_node(nid, NULL,
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
find_min_pfn_for_node(nid), NULL);
|
2007-10-16 02:25:39 -06:00
|
|
|
|
|
|
|
/* Any memory on that node */
|
|
|
|
if (pgdat->node_present_pages)
|
2012-12-12 14:51:46 -07:00
|
|
|
node_set_state(nid, N_MEMORY);
|
|
|
|
check_for_memory(pgdat, nid);
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
}
|
|
|
|
}
|
2007-07-17 05:03:12 -06:00
|
|
|
|
2007-07-17 05:03:15 -06:00
|
|
|
static int __init cmdline_parse_core(char *p, unsigned long *core)
|
2007-07-17 05:03:12 -06:00
|
|
|
{
|
|
|
|
unsigned long long coremem;
|
|
|
|
if (!p)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
coremem = memparse(p, &p);
|
2007-07-17 05:03:15 -06:00
|
|
|
*core = coremem >> PAGE_SHIFT;
|
2007-07-17 05:03:12 -06:00
|
|
|
|
2007-07-17 05:03:15 -06:00
|
|
|
/* Paranoid check that UL is enough for the coremem value */
|
2007-07-17 05:03:12 -06:00
|
|
|
WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2007-07-17 05:03:14 -06:00
|
|
|
|
2007-07-17 05:03:15 -06:00
|
|
|
/*
|
|
|
|
* kernelcore=size sets the amount of memory for use for allocations that
|
|
|
|
* cannot be reclaimed or migrated.
|
|
|
|
*/
|
|
|
|
static int __init cmdline_parse_kernelcore(char *p)
|
|
|
|
{
|
2016-03-15 15:55:22 -06:00
|
|
|
/* parse kernelcore=mirror */
|
|
|
|
if (parse_option_str(p, "mirror")) {
|
|
|
|
mirrored_kernelcore = true;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-07-17 05:03:15 -06:00
|
|
|
return cmdline_parse_core(p, &required_kernelcore);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* movablecore=size sets the amount of memory for use for allocations that
|
|
|
|
* can be reclaimed or migrated.
|
|
|
|
*/
|
|
|
|
static int __init cmdline_parse_movablecore(char *p)
|
|
|
|
{
|
|
|
|
return cmdline_parse_core(p, &required_movablecore);
|
|
|
|
}
|
|
|
|
|
2007-07-17 05:03:14 -06:00
|
|
|
early_param("kernelcore", cmdline_parse_kernelcore);
|
2007-07-17 05:03:15 -06:00
|
|
|
early_param("movablecore", cmdline_parse_movablecore);
|
2007-07-17 05:03:14 -06:00
|
|
|
|
2011-12-08 11:22:09 -07:00
|
|
|
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
|
[PATCH] Introduce mechanism for registering active regions of memory
At a basic level, architectures define structures to record where active
ranges of page frames are located. Once located, the code to calculate zone
sizes and holes in each architecture is very similar. Some of this zone and
hole sizing code is difficult to read for no good reason. This set of patches
eliminates the similar-looking architecture-specific code.
The patches introduce a mechanism where architectures register where the
active ranges of page frames are with add_active_range(). When all areas have
been discovered, free_area_init_nodes() is called to initialise the pgdat and
zones. The zone sizes and holes are then calculated in an architecture
independent manner.
Patch 1 introduces the mechanism for registering and initialising PFN ranges
Patch 2 changes ppc to use the mechanism - 139 arch-specific LOC removed
Patch 3 changes x86 to use the mechanism - 136 arch-specific LOC removed
Patch 4 changes x86_64 to use the mechanism - 74 arch-specific LOC removed
Patch 5 changes ia64 to use the mechanism - 52 arch-specific LOC removed
Patch 6 accounts for mem_map as a memory hole as the pages are not reclaimable.
It adjusts the watermarks slightly
Tony Luck has successfully tested for ia64 on Itanium with tiger_defconfig,
gensparse_defconfig and defconfig. Bob Picco has also tested and debugged on
IA64. Jack Steiner successfully boot tested on a mammoth SGI IA64-based
machine. These were on patches against 2.6.17-rc1 and release 3 of these
patches but there have been no ia64-changes since release 3.
There are differences in the zone sizes for x86_64 as the arch-specific code
for x86_64 accounts the kernel image and the starting mem_maps as memory holes
but the architecture-independent code accounts the memory as present.
The big benefit of this set of patches is a sizable reduction of
architecture-specific code, some of which is very hairy. There should be a
greater reduction when other architectures use the same mechanisms for zone
and hole sizing but I lack the hardware to test on.
Additional credit;
Dave Hansen for the initial suggestion and comments on early patches
Andy Whitcroft for reviewing early versions and catching numerous
errors
Tony Luck for testing and debugging on IA64
Bob Picco for fixing bugs related to pfn registration, reviewing a
number of patch revisions, providing a number of suggestions
on future direction and testing heavily
Jack Steiner and Robin Holt for testing on IA64 and clarifying
issues related to memory holes
Yasunori for testing on IA64
Andi Kleen for reviewing and feeding back about x86_64
Christian Kujau for providing valuable information related to ACPI
problems on x86_64 and testing potential fixes
This patch:
Define the structure to represent an active range of page frames within a node
in an architecture independent manner. Architectures are expected to register
active ranges of PFNs using add_active_range(nid, start_pfn, end_pfn) and call
free_area_init_nodes() passing the PFNs of the end of each zone.
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Bob Picco <bob.picco@hp.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Andi Kleen <ak@muc.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "Keith Mannthey" <kmannth@gmail.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-09-27 02:49:43 -06:00
|
|
|
|
2013-07-03 16:03:14 -06:00
|
|
|
void adjust_managed_page_count(struct page *page, long count)
|
|
|
|
{
|
|
|
|
spin_lock(&managed_page_count_lock);
|
|
|
|
page_zone(page)->managed_pages += count;
|
|
|
|
totalram_pages += count;
|
2013-07-03 16:03:21 -06:00
|
|
|
#ifdef CONFIG_HIGHMEM
|
|
|
|
if (PageHighMem(page))
|
|
|
|
totalhigh_pages += count;
|
|
|
|
#endif
|
2013-07-03 16:03:14 -06:00
|
|
|
spin_unlock(&managed_page_count_lock);
|
|
|
|
}
|
2013-07-03 16:03:21 -06:00
|
|
|
EXPORT_SYMBOL(adjust_managed_page_count);
|
2013-07-03 16:03:14 -06:00
|
|
|
|
mm: change signature of free_reserved_area() to fix building warnings
Change signature of free_reserved_area() according to Russell King's
suggestion to fix following build warnings:
arch/arm/mm/init.c: In function 'mem_init':
arch/arm/mm/init.c:603:2: warning: passing argument 1 of 'free_reserved_area' makes integer from pointer without a cast [enabled by default]
free_reserved_area(__va(PHYS_PFN_OFFSET), swapper_pg_dir, 0, NULL);
^
In file included from include/linux/mman.h:4:0,
from arch/arm/mm/init.c:15:
include/linux/mm.h:1301:22: note: expected 'long unsigned int' but argument is of type 'void *'
extern unsigned long free_reserved_area(unsigned long start, unsigned long end,
mm/page_alloc.c: In function 'free_reserved_area':
>> mm/page_alloc.c:5134:3: warning: passing argument 1 of 'virt_to_phys' makes pointer from integer without a cast [enabled by default]
In file included from arch/mips/include/asm/page.h:49:0,
from include/linux/mmzone.h:20,
from include/linux/gfp.h:4,
from include/linux/mm.h:8,
from mm/page_alloc.c:18:
arch/mips/include/asm/io.h:119:29: note: expected 'const volatile void *' but argument is of type 'long unsigned int'
mm/page_alloc.c: In function 'free_area_init_nodes':
mm/page_alloc.c:5030:34: warning: array subscript is below array bounds [-Warray-bounds]
Also address some minor code review comments.
Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
Reported-by: Arnd Bergmann <arnd@arndb.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: <sworddragon2@aol.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Jianguo Wu <wujianguo@huawei.com>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Kamezawa Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michel Lespinasse <walken@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Russell King <rmk@arm.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-07-03 16:02:48 -06:00
|
|
|
unsigned long free_reserved_area(void *start, void *end, int poison, char *s)
|
2013-04-29 16:06:21 -06:00
|
|
|
{
|
mm: change signature of free_reserved_area() to fix building warnings
Change signature of free_reserved_area() according to Russell King's
suggestion to fix following build warnings:
arch/arm/mm/init.c: In function 'mem_init':
arch/arm/mm/init.c:603:2: warning: passing argument 1 of 'free_reserved_area' makes integer from pointer without a cast [enabled by default]
free_reserved_area(__va(PHYS_PFN_OFFSET), swapper_pg_dir, 0, NULL);
^
In file included from include/linux/mman.h:4:0,
from arch/arm/mm/init.c:15:
include/linux/mm.h:1301:22: note: expected 'long unsigned int' but argument is of type 'void *'
extern unsigned long free_reserved_area(unsigned long start, unsigned long end,
mm/page_alloc.c: In function 'free_reserved_area':
>> mm/page_alloc.c:5134:3: warning: passing argument 1 of 'virt_to_phys' makes pointer from integer without a cast [enabled by default]
In file included from arch/mips/include/asm/page.h:49:0,
from include/linux/mmzone.h:20,
from include/linux/gfp.h:4,
from include/linux/mm.h:8,
from mm/page_alloc.c:18:
arch/mips/include/asm/io.h:119:29: note: expected 'const volatile void *' but argument is of type 'long unsigned int'
mm/page_alloc.c: In function 'free_area_init_nodes':
mm/page_alloc.c:5030:34: warning: array subscript is below array bounds [-Warray-bounds]
Also address some minor code review comments.
Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
Reported-by: Arnd Bergmann <arnd@arndb.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: <sworddragon2@aol.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Jianguo Wu <wujianguo@huawei.com>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Kamezawa Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michel Lespinasse <walken@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Russell King <rmk@arm.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-07-03 16:02:48 -06:00
|
|
|
void *pos;
|
|
|
|
unsigned long pages = 0;
|
2013-04-29 16:06:21 -06:00
|
|
|
|
mm: change signature of free_reserved_area() to fix building warnings
Change signature of free_reserved_area() according to Russell King's
suggestion to fix following build warnings:
arch/arm/mm/init.c: In function 'mem_init':
arch/arm/mm/init.c:603:2: warning: passing argument 1 of 'free_reserved_area' makes integer from pointer without a cast [enabled by default]
free_reserved_area(__va(PHYS_PFN_OFFSET), swapper_pg_dir, 0, NULL);
^
In file included from include/linux/mman.h:4:0,
from arch/arm/mm/init.c:15:
include/linux/mm.h:1301:22: note: expected 'long unsigned int' but argument is of type 'void *'
extern unsigned long free_reserved_area(unsigned long start, unsigned long end,
mm/page_alloc.c: In function 'free_reserved_area':
>> mm/page_alloc.c:5134:3: warning: passing argument 1 of 'virt_to_phys' makes pointer from integer without a cast [enabled by default]
In file included from arch/mips/include/asm/page.h:49:0,
from include/linux/mmzone.h:20,
from include/linux/gfp.h:4,
from include/linux/mm.h:8,
from mm/page_alloc.c:18:
arch/mips/include/asm/io.h:119:29: note: expected 'const volatile void *' but argument is of type 'long unsigned int'
mm/page_alloc.c: In function 'free_area_init_nodes':
mm/page_alloc.c:5030:34: warning: array subscript is below array bounds [-Warray-bounds]
Also address some minor code review comments.
Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
Reported-by: Arnd Bergmann <arnd@arndb.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: <sworddragon2@aol.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Jianguo Wu <wujianguo@huawei.com>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Kamezawa Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michel Lespinasse <walken@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Russell King <rmk@arm.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-07-03 16:02:48 -06:00
|
|
|
start = (void *)PAGE_ALIGN((unsigned long)start);
|
|
|
|
end = (void *)((unsigned long)end & PAGE_MASK);
|
|
|
|
for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
|
2013-07-03 16:02:51 -06:00
|
|
|
if ((unsigned int)poison <= 0xFF)
|
mm: change signature of free_reserved_area() to fix building warnings
Change signature of free_reserved_area() according to Russell King's
suggestion to fix following build warnings:
arch/arm/mm/init.c: In function 'mem_init':
arch/arm/mm/init.c:603:2: warning: passing argument 1 of 'free_reserved_area' makes integer from pointer without a cast [enabled by default]
free_reserved_area(__va(PHYS_PFN_OFFSET), swapper_pg_dir, 0, NULL);
^
In file included from include/linux/mman.h:4:0,
from arch/arm/mm/init.c:15:
include/linux/mm.h:1301:22: note: expected 'long unsigned int' but argument is of type 'void *'
extern unsigned long free_reserved_area(unsigned long start, unsigned long end,
mm/page_alloc.c: In function 'free_reserved_area':
>> mm/page_alloc.c:5134:3: warning: passing argument 1 of 'virt_to_phys' makes pointer from integer without a cast [enabled by default]
In file included from arch/mips/include/asm/page.h:49:0,
from include/linux/mmzone.h:20,
from include/linux/gfp.h:4,
from include/linux/mm.h:8,
from mm/page_alloc.c:18:
arch/mips/include/asm/io.h:119:29: note: expected 'const volatile void *' but argument is of type 'long unsigned int'
mm/page_alloc.c: In function 'free_area_init_nodes':
mm/page_alloc.c:5030:34: warning: array subscript is below array bounds [-Warray-bounds]
Also address some minor code review comments.
Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
Reported-by: Arnd Bergmann <arnd@arndb.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: <sworddragon2@aol.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Jianguo Wu <wujianguo@huawei.com>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Kamezawa Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michel Lespinasse <walken@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Russell King <rmk@arm.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-07-03 16:02:48 -06:00
|
|
|
memset(pos, poison, PAGE_SIZE);
|
|
|
|
free_reserved_page(virt_to_page(pos));
|
2013-04-29 16:06:21 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
if (pages && s)
|
mm: change signature of free_reserved_area() to fix building warnings
Change signature of free_reserved_area() according to Russell King's
suggestion to fix following build warnings:
arch/arm/mm/init.c: In function 'mem_init':
arch/arm/mm/init.c:603:2: warning: passing argument 1 of 'free_reserved_area' makes integer from pointer without a cast [enabled by default]
free_reserved_area(__va(PHYS_PFN_OFFSET), swapper_pg_dir, 0, NULL);
^
In file included from include/linux/mman.h:4:0,
from arch/arm/mm/init.c:15:
include/linux/mm.h:1301:22: note: expected 'long unsigned int' but argument is of type 'void *'
extern unsigned long free_reserved_area(unsigned long start, unsigned long end,
mm/page_alloc.c: In function 'free_reserved_area':
>> mm/page_alloc.c:5134:3: warning: passing argument 1 of 'virt_to_phys' makes pointer from integer without a cast [enabled by default]
In file included from arch/mips/include/asm/page.h:49:0,
from include/linux/mmzone.h:20,
from include/linux/gfp.h:4,
from include/linux/mm.h:8,
from mm/page_alloc.c:18:
arch/mips/include/asm/io.h:119:29: note: expected 'const volatile void *' but argument is of type 'long unsigned int'
mm/page_alloc.c: In function 'free_area_init_nodes':
mm/page_alloc.c:5030:34: warning: array subscript is below array bounds [-Warray-bounds]
Also address some minor code review comments.
Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
Reported-by: Arnd Bergmann <arnd@arndb.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: <sworddragon2@aol.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Jianguo Wu <wujianguo@huawei.com>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Kamezawa Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michel Lespinasse <walken@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Russell King <rmk@arm.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-07-03 16:02:48 -06:00
|
|
|
pr_info("Freeing %s memory: %ldK (%p - %p)\n",
|
2013-04-29 16:06:21 -06:00
|
|
|
s, pages << (PAGE_SHIFT - 10), start, end);
|
|
|
|
|
|
|
|
return pages;
|
|
|
|
}
|
mm: change signature of free_reserved_area() to fix building warnings
Change signature of free_reserved_area() according to Russell King's
suggestion to fix following build warnings:
arch/arm/mm/init.c: In function 'mem_init':
arch/arm/mm/init.c:603:2: warning: passing argument 1 of 'free_reserved_area' makes integer from pointer without a cast [enabled by default]
free_reserved_area(__va(PHYS_PFN_OFFSET), swapper_pg_dir, 0, NULL);
^
In file included from include/linux/mman.h:4:0,
from arch/arm/mm/init.c:15:
include/linux/mm.h:1301:22: note: expected 'long unsigned int' but argument is of type 'void *'
extern unsigned long free_reserved_area(unsigned long start, unsigned long end,
mm/page_alloc.c: In function 'free_reserved_area':
>> mm/page_alloc.c:5134:3: warning: passing argument 1 of 'virt_to_phys' makes pointer from integer without a cast [enabled by default]
In file included from arch/mips/include/asm/page.h:49:0,
from include/linux/mmzone.h:20,
from include/linux/gfp.h:4,
from include/linux/mm.h:8,
from mm/page_alloc.c:18:
arch/mips/include/asm/io.h:119:29: note: expected 'const volatile void *' but argument is of type 'long unsigned int'
mm/page_alloc.c: In function 'free_area_init_nodes':
mm/page_alloc.c:5030:34: warning: array subscript is below array bounds [-Warray-bounds]
Also address some minor code review comments.
Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
Reported-by: Arnd Bergmann <arnd@arndb.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: <sworddragon2@aol.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Jianguo Wu <wujianguo@huawei.com>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Kamezawa Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michel Lespinasse <walken@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Russell King <rmk@arm.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-07-03 16:02:48 -06:00
|
|
|
EXPORT_SYMBOL(free_reserved_area);
|
2013-04-29 16:06:21 -06:00
|
|
|
|
2013-04-29 16:07:00 -06:00
|
|
|
#ifdef CONFIG_HIGHMEM
|
|
|
|
void free_highmem_page(struct page *page)
|
|
|
|
{
|
|
|
|
__free_reserved_page(page);
|
|
|
|
totalram_pages++;
|
2013-07-03 16:03:11 -06:00
|
|
|
page_zone(page)->managed_pages++;
|
2013-04-29 16:07:00 -06:00
|
|
|
totalhigh_pages++;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2013-07-03 16:03:41 -06:00
|
|
|
|
|
|
|
void __init mem_init_print_info(const char *str)
|
|
|
|
{
|
|
|
|
unsigned long physpages, codesize, datasize, rosize, bss_size;
|
|
|
|
unsigned long init_code_size, init_data_size;
|
|
|
|
|
|
|
|
physpages = get_num_physpages();
|
|
|
|
codesize = _etext - _stext;
|
|
|
|
datasize = _edata - _sdata;
|
|
|
|
rosize = __end_rodata - __start_rodata;
|
|
|
|
bss_size = __bss_stop - __bss_start;
|
|
|
|
init_data_size = __init_end - __init_begin;
|
|
|
|
init_code_size = _einittext - _sinittext;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Detect special cases and adjust section sizes accordingly:
|
|
|
|
* 1) .init.* may be embedded into .data sections
|
|
|
|
* 2) .init.text.* may be out of [__init_begin, __init_end],
|
|
|
|
* please refer to arch/tile/kernel/vmlinux.lds.S.
|
|
|
|
* 3) .rodata.* may be embedded into .text or .data sections.
|
|
|
|
*/
|
|
|
|
#define adj_init_size(start, end, size, pos, adj) \
|
2013-09-11 15:20:34 -06:00
|
|
|
do { \
|
|
|
|
if (start <= pos && pos < end && size > adj) \
|
|
|
|
size -= adj; \
|
|
|
|
} while (0)
|
2013-07-03 16:03:41 -06:00
|
|
|
|
|
|
|
adj_init_size(__init_begin, __init_end, init_data_size,
|
|
|
|
_sinittext, init_code_size);
|
|
|
|
adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
|
|
|
|
adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
|
|
|
|
adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
|
|
|
|
adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
|
|
|
|
|
|
|
|
#undef adj_init_size
|
|
|
|
|
2016-03-17 15:19:47 -06:00
|
|
|
pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
|
2013-07-03 16:03:41 -06:00
|
|
|
#ifdef CONFIG_HIGHMEM
|
2016-03-17 15:19:47 -06:00
|
|
|
", %luK highmem"
|
2013-07-03 16:03:41 -06:00
|
|
|
#endif
|
2016-03-17 15:19:47 -06:00
|
|
|
"%s%s)\n",
|
|
|
|
nr_free_pages() << (PAGE_SHIFT - 10),
|
|
|
|
physpages << (PAGE_SHIFT - 10),
|
|
|
|
codesize >> 10, datasize >> 10, rosize >> 10,
|
|
|
|
(init_data_size + init_code_size) >> 10, bss_size >> 10,
|
|
|
|
(physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10),
|
|
|
|
totalcma_pages << (PAGE_SHIFT - 10),
|
2013-07-03 16:03:41 -06:00
|
|
|
#ifdef CONFIG_HIGHMEM
|
2016-03-17 15:19:47 -06:00
|
|
|
totalhigh_pages << (PAGE_SHIFT - 10),
|
2013-07-03 16:03:41 -06:00
|
|
|
#endif
|
2016-03-17 15:19:47 -06:00
|
|
|
str ? ", " : "", str ? str : "");
|
2013-07-03 16:03:41 -06:00
|
|
|
}
|
|
|
|
|
2006-09-27 02:49:56 -06:00
|
|
|
/**
|
2006-10-04 03:15:25 -06:00
|
|
|
* set_dma_reserve - set the specified number of pages reserved in the first zone
|
|
|
|
* @new_dma_reserve: The number of pages to mark reserved
|
2006-09-27 02:49:56 -06:00
|
|
|
*
|
2015-09-08 16:04:10 -06:00
|
|
|
* The per-cpu batchsize and zone watermarks are determined by managed_pages.
|
2006-09-27 02:49:56 -06:00
|
|
|
* In the DMA zone, a significant percentage may be consumed by kernel image
|
|
|
|
* and other unfreeable allocations which can skew the watermarks badly. This
|
2006-10-04 03:15:25 -06:00
|
|
|
* function may optionally be used to account for unfreeable pages in the
|
|
|
|
* first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
|
|
|
|
* smaller per-cpu batchsize.
|
2006-09-27 02:49:56 -06:00
|
|
|
*/
|
|
|
|
void __init set_dma_reserve(unsigned long new_dma_reserve)
|
|
|
|
{
|
|
|
|
dma_reserve = new_dma_reserve;
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
void __init free_area_init(unsigned long *zones_size)
|
|
|
|
{
|
2008-07-23 22:27:20 -06:00
|
|
|
free_area_init_node(0, zones_size,
|
2005-04-16 16:20:36 -06:00
|
|
|
__pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int page_alloc_cpu_notify(struct notifier_block *self,
|
|
|
|
unsigned long action, void *hcpu)
|
|
|
|
{
|
|
|
|
int cpu = (unsigned long)hcpu;
|
|
|
|
|
2007-05-09 03:35:10 -06:00
|
|
|
if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
|
2012-03-21 17:34:06 -06:00
|
|
|
lru_add_drain_cpu(cpu);
|
2008-02-04 23:29:11 -07:00
|
|
|
drain_pages(cpu);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Spill the event counters of the dead processor
|
|
|
|
* into the current processors event counters.
|
|
|
|
* This artificially elevates the count of the current
|
|
|
|
* processor.
|
|
|
|
*/
|
2006-06-30 02:55:45 -06:00
|
|
|
vm_events_fold_cpu(cpu);
|
2008-02-04 23:29:11 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Zero the differential counters of the dead processor
|
|
|
|
* so that the vm statistics are consistent.
|
|
|
|
*
|
|
|
|
* This is only okay since the processor is dead and cannot
|
|
|
|
* race with what we are doing.
|
|
|
|
*/
|
2013-09-11 15:21:30 -06:00
|
|
|
cpu_vm_stats_fold(cpu);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
return NOTIFY_OK;
|
|
|
|
}
|
|
|
|
|
|
|
|
void __init page_alloc_init(void)
|
|
|
|
{
|
|
|
|
hotcpu_notifier(page_alloc_cpu_notify, 0);
|
|
|
|
}
|
|
|
|
|
2006-04-10 23:52:59 -06:00
|
|
|
/*
|
2015-09-08 16:04:13 -06:00
|
|
|
* calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
|
2006-04-10 23:52:59 -06:00
|
|
|
* or min_free_kbytes changes.
|
|
|
|
*/
|
|
|
|
static void calculate_totalreserve_pages(void)
|
|
|
|
{
|
|
|
|
struct pglist_data *pgdat;
|
|
|
|
unsigned long reserve_pages = 0;
|
2006-09-26 00:31:18 -06:00
|
|
|
enum zone_type i, j;
|
2006-04-10 23:52:59 -06:00
|
|
|
|
|
|
|
for_each_online_pgdat(pgdat) {
|
2016-07-28 16:46:11 -06:00
|
|
|
|
|
|
|
pgdat->totalreserve_pages = 0;
|
|
|
|
|
2006-04-10 23:52:59 -06:00
|
|
|
for (i = 0; i < MAX_NR_ZONES; i++) {
|
|
|
|
struct zone *zone = pgdat->node_zones + i;
|
2014-08-06 17:07:14 -06:00
|
|
|
long max = 0;
|
2006-04-10 23:52:59 -06:00
|
|
|
|
|
|
|
/* Find valid and maximum lowmem_reserve in the zone */
|
|
|
|
for (j = i; j < MAX_NR_ZONES; j++) {
|
|
|
|
if (zone->lowmem_reserve[j] > max)
|
|
|
|
max = zone->lowmem_reserve[j];
|
|
|
|
}
|
|
|
|
|
2009-06-16 16:32:12 -06:00
|
|
|
/* we treat the high watermark as reserved pages. */
|
|
|
|
max += high_wmark_pages(zone);
|
2006-04-10 23:52:59 -06:00
|
|
|
|
2013-02-22 17:33:52 -07:00
|
|
|
if (max > zone->managed_pages)
|
|
|
|
max = zone->managed_pages;
|
2016-01-14 16:20:15 -07:00
|
|
|
|
2016-07-28 16:46:11 -06:00
|
|
|
pgdat->totalreserve_pages += max;
|
2016-01-14 16:20:15 -07:00
|
|
|
|
2006-04-10 23:52:59 -06:00
|
|
|
reserve_pages += max;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
totalreserve_pages = reserve_pages;
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* setup_per_zone_lowmem_reserve - called whenever
|
2015-09-08 16:04:13 -06:00
|
|
|
* sysctl_lowmem_reserve_ratio changes. Ensures that each zone
|
2005-04-16 16:20:36 -06:00
|
|
|
* has a correct pages reserved value, so an adequate number of
|
|
|
|
* pages are left in the zone after a successful __alloc_pages().
|
|
|
|
*/
|
|
|
|
static void setup_per_zone_lowmem_reserve(void)
|
|
|
|
{
|
|
|
|
struct pglist_data *pgdat;
|
2006-09-26 00:31:18 -06:00
|
|
|
enum zone_type j, idx;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-03-27 02:15:59 -07:00
|
|
|
for_each_online_pgdat(pgdat) {
|
2005-04-16 16:20:36 -06:00
|
|
|
for (j = 0; j < MAX_NR_ZONES; j++) {
|
|
|
|
struct zone *zone = pgdat->node_zones + j;
|
2013-02-22 17:33:52 -07:00
|
|
|
unsigned long managed_pages = zone->managed_pages;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
zone->lowmem_reserve[j] = 0;
|
|
|
|
|
2006-09-26 00:31:18 -06:00
|
|
|
idx = j;
|
|
|
|
while (idx) {
|
2005-04-16 16:20:36 -06:00
|
|
|
struct zone *lower_zone;
|
|
|
|
|
2006-09-26 00:31:18 -06:00
|
|
|
idx--;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (sysctl_lowmem_reserve_ratio[idx] < 1)
|
|
|
|
sysctl_lowmem_reserve_ratio[idx] = 1;
|
|
|
|
|
|
|
|
lower_zone = pgdat->node_zones + idx;
|
2013-02-22 17:33:52 -07:00
|
|
|
lower_zone->lowmem_reserve[j] = managed_pages /
|
2005-04-16 16:20:36 -06:00
|
|
|
sysctl_lowmem_reserve_ratio[idx];
|
2013-02-22 17:33:52 -07:00
|
|
|
managed_pages += lower_zone->managed_pages;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2006-04-10 23:52:59 -06:00
|
|
|
|
|
|
|
/* update totalreserve_pages */
|
|
|
|
calculate_totalreserve_pages();
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2011-04-25 15:36:42 -06:00
|
|
|
static void __setup_per_zone_wmarks(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
|
|
|
|
unsigned long lowmem_pages = 0;
|
|
|
|
struct zone *zone;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
/* Calculate total number of !ZONE_HIGHMEM pages */
|
|
|
|
for_each_zone(zone) {
|
|
|
|
if (!is_highmem(zone))
|
2013-02-22 17:33:52 -07:00
|
|
|
lowmem_pages += zone->managed_pages;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
for_each_zone(zone) {
|
2006-05-15 10:43:59 -06:00
|
|
|
u64 tmp;
|
|
|
|
|
2008-10-18 21:27:11 -06:00
|
|
|
spin_lock_irqsave(&zone->lock, flags);
|
2013-02-22 17:33:52 -07:00
|
|
|
tmp = (u64)pages_min * zone->managed_pages;
|
2006-05-15 10:43:59 -06:00
|
|
|
do_div(tmp, lowmem_pages);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (is_highmem(zone)) {
|
|
|
|
/*
|
2005-11-13 17:06:45 -07:00
|
|
|
* __GFP_HIGH and PF_MEMALLOC allocations usually don't
|
|
|
|
* need highmem pages, so cap pages_min to a small
|
|
|
|
* value here.
|
|
|
|
*
|
2009-06-16 16:32:12 -06:00
|
|
|
* The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
|
2015-04-14 16:47:14 -06:00
|
|
|
* deltas control asynch page reclaim, and so should
|
2005-11-13 17:06:45 -07:00
|
|
|
* not be capped for highmem.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2013-02-22 17:32:22 -07:00
|
|
|
unsigned long min_pages;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-02-22 17:33:52 -07:00
|
|
|
min_pages = zone->managed_pages / 1024;
|
2013-02-22 17:32:22 -07:00
|
|
|
min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
|
2009-06-16 16:32:12 -06:00
|
|
|
zone->watermark[WMARK_MIN] = min_pages;
|
2005-04-16 16:20:36 -06:00
|
|
|
} else {
|
2005-11-13 17:06:45 -07:00
|
|
|
/*
|
|
|
|
* If it's a lowmem zone, reserve a number of pages
|
2005-04-16 16:20:36 -06:00
|
|
|
* proportionate to the zone's size.
|
|
|
|
*/
|
2009-06-16 16:32:12 -06:00
|
|
|
zone->watermark[WMARK_MIN] = tmp;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2016-03-17 15:19:14 -06:00
|
|
|
/*
|
|
|
|
* Set the kswapd watermarks distance according to the
|
|
|
|
* scale factor in proportion to available memory, but
|
|
|
|
* ensure a minimum size on small systems.
|
|
|
|
*/
|
|
|
|
tmp = max_t(u64, tmp >> 2,
|
|
|
|
mult_frac(zone->managed_pages,
|
|
|
|
watermark_scale_factor, 10000));
|
|
|
|
|
|
|
|
zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp;
|
|
|
|
zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
|
2012-01-25 04:49:24 -07:00
|
|
|
|
2008-10-18 21:27:11 -06:00
|
|
|
spin_unlock_irqrestore(&zone->lock, flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2006-04-10 23:52:59 -06:00
|
|
|
|
|
|
|
/* update totalreserve_pages */
|
|
|
|
calculate_totalreserve_pages();
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2011-04-25 15:36:42 -06:00
|
|
|
/**
|
|
|
|
* setup_per_zone_wmarks - called when min_free_kbytes changes
|
|
|
|
* or when memory is hot-{added|removed}
|
|
|
|
*
|
|
|
|
* Ensures that the watermark[min,low,high] values for each zone are set
|
|
|
|
* correctly with respect to min_free_kbytes.
|
|
|
|
*/
|
|
|
|
void setup_per_zone_wmarks(void)
|
|
|
|
{
|
|
|
|
mutex_lock(&zonelists_mutex);
|
|
|
|
__setup_per_zone_wmarks();
|
|
|
|
mutex_unlock(&zonelists_mutex);
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* Initialise min_free_kbytes.
|
|
|
|
*
|
|
|
|
* For small machines we want it small (128k min). For large machines
|
|
|
|
* we want it large (64MB max). But it is not linear, because network
|
|
|
|
* bandwidth does not increase linearly with machine size. We use
|
|
|
|
*
|
2013-09-11 15:20:34 -06:00
|
|
|
* min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
|
2005-04-16 16:20:36 -06:00
|
|
|
* min_free_kbytes = sqrt(lowmem_kbytes * 16)
|
|
|
|
*
|
|
|
|
* which yields
|
|
|
|
*
|
|
|
|
* 16MB: 512k
|
|
|
|
* 32MB: 724k
|
|
|
|
* 64MB: 1024k
|
|
|
|
* 128MB: 1448k
|
|
|
|
* 256MB: 2048k
|
|
|
|
* 512MB: 2896k
|
|
|
|
* 1024MB: 4096k
|
|
|
|
* 2048MB: 5792k
|
|
|
|
* 4096MB: 8192k
|
|
|
|
* 8192MB: 11584k
|
|
|
|
* 16384MB: 16384k
|
|
|
|
*/
|
2011-05-24 18:11:32 -06:00
|
|
|
int __meminit init_per_zone_wmark_min(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
unsigned long lowmem_kbytes;
|
2013-07-08 17:00:40 -06:00
|
|
|
int new_min_free_kbytes;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
|
2013-07-08 17:00:40 -06:00
|
|
|
new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
|
|
|
|
|
|
|
|
if (new_min_free_kbytes > user_min_free_kbytes) {
|
|
|
|
min_free_kbytes = new_min_free_kbytes;
|
|
|
|
if (min_free_kbytes < 128)
|
|
|
|
min_free_kbytes = 128;
|
|
|
|
if (min_free_kbytes > 65536)
|
|
|
|
min_free_kbytes = 65536;
|
|
|
|
} else {
|
|
|
|
pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
|
|
|
|
new_min_free_kbytes, user_min_free_kbytes);
|
|
|
|
}
|
2009-06-16 16:32:48 -06:00
|
|
|
setup_per_zone_wmarks();
|
2011-05-24 18:11:33 -06:00
|
|
|
refresh_zone_stat_thresholds();
|
2005-04-16 16:20:36 -06:00
|
|
|
setup_per_zone_lowmem_reserve();
|
2016-08-10 17:27:49 -06:00
|
|
|
|
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
setup_min_unmapped_ratio();
|
|
|
|
setup_min_slab_ratio();
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
|
|
|
}
|
2016-05-05 17:22:12 -06:00
|
|
|
core_initcall(init_per_zone_wmark_min)
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
2013-09-11 15:20:34 -06:00
|
|
|
* min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
|
2005-04-16 16:20:36 -06:00
|
|
|
* that we can call two helper functions whenever min_free_kbytes
|
|
|
|
* changes.
|
|
|
|
*/
|
2014-06-06 15:38:09 -06:00
|
|
|
int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
|
2009-09-23 16:57:19 -06:00
|
|
|
void __user *buffer, size_t *length, loff_t *ppos)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2014-01-23 16:53:17 -07:00
|
|
|
int rc;
|
|
|
|
|
|
|
|
rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
2013-07-08 17:00:40 -06:00
|
|
|
if (write) {
|
|
|
|
user_min_free_kbytes = min_free_kbytes;
|
2009-06-16 16:32:48 -06:00
|
|
|
setup_per_zone_wmarks();
|
2013-07-08 17:00:40 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-03-17 15:19:14 -06:00
|
|
|
int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
|
|
|
|
void __user *buffer, size_t *length, loff_t *ppos)
|
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
if (write)
|
|
|
|
setup_per_zone_wmarks();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-07-03 01:24:13 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
2016-08-10 17:27:49 -06:00
|
|
|
static void setup_min_unmapped_ratio(void)
|
2006-07-03 01:24:13 -06:00
|
|
|
{
|
2016-08-10 17:27:49 -06:00
|
|
|
pg_data_t *pgdat;
|
2006-07-03 01:24:13 -06:00
|
|
|
struct zone *zone;
|
|
|
|
|
2016-07-28 16:46:32 -06:00
|
|
|
for_each_online_pgdat(pgdat)
|
2016-08-10 17:27:46 -06:00
|
|
|
pgdat->min_unmapped_pages = 0;
|
2016-07-28 16:46:32 -06:00
|
|
|
|
2006-07-03 01:24:13 -06:00
|
|
|
for_each_zone(zone)
|
2016-07-28 16:46:32 -06:00
|
|
|
zone->zone_pgdat->min_unmapped_pages += (zone->managed_pages *
|
2006-07-03 01:24:13 -06:00
|
|
|
sysctl_min_unmapped_ratio) / 100;
|
|
|
|
}
|
2006-09-26 00:31:52 -06:00
|
|
|
|
2016-08-10 17:27:49 -06:00
|
|
|
|
|
|
|
int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
|
2009-09-23 16:57:19 -06:00
|
|
|
void __user *buffer, size_t *length, loff_t *ppos)
|
2006-09-26 00:31:52 -06:00
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
2009-09-23 16:57:19 -06:00
|
|
|
rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
|
2006-09-26 00:31:52 -06:00
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
2016-08-10 17:27:49 -06:00
|
|
|
setup_min_unmapped_ratio();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void setup_min_slab_ratio(void)
|
|
|
|
{
|
|
|
|
pg_data_t *pgdat;
|
|
|
|
struct zone *zone;
|
|
|
|
|
2016-07-28 16:46:32 -06:00
|
|
|
for_each_online_pgdat(pgdat)
|
|
|
|
pgdat->min_slab_pages = 0;
|
|
|
|
|
2006-09-26 00:31:52 -06:00
|
|
|
for_each_zone(zone)
|
2016-07-28 16:46:32 -06:00
|
|
|
zone->zone_pgdat->min_slab_pages += (zone->managed_pages *
|
2006-09-26 00:31:52 -06:00
|
|
|
sysctl_min_slab_ratio) / 100;
|
2016-08-10 17:27:49 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
|
|
|
|
void __user *buffer, size_t *length, loff_t *ppos)
|
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
setup_min_slab_ratio();
|
|
|
|
|
2006-09-26 00:31:52 -06:00
|
|
|
return 0;
|
|
|
|
}
|
2006-07-03 01:24:13 -06:00
|
|
|
#endif
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* lowmem_reserve_ratio_sysctl_handler - just a wrapper around
|
|
|
|
* proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
|
|
|
|
* whenever sysctl_lowmem_reserve_ratio changes.
|
|
|
|
*
|
|
|
|
* The reserve ratio obviously has absolutely no relation with the
|
2009-06-16 16:32:12 -06:00
|
|
|
* minimum watermarks. The lowmem reserve ratio can only make sense
|
2005-04-16 16:20:36 -06:00
|
|
|
* if in function of the boot time zone sizes.
|
|
|
|
*/
|
2014-06-06 15:38:09 -06:00
|
|
|
int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
|
2009-09-23 16:57:19 -06:00
|
|
|
void __user *buffer, size_t *length, loff_t *ppos)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2009-09-23 16:57:19 -06:00
|
|
|
proc_dointvec_minmax(table, write, buffer, length, ppos);
|
2005-04-16 16:20:36 -06:00
|
|
|
setup_per_zone_lowmem_reserve();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-01-08 02:00:40 -07:00
|
|
|
/*
|
|
|
|
* percpu_pagelist_fraction - changes the pcp->high for each zone on each
|
2013-09-11 15:20:34 -06:00
|
|
|
* cpu. It is the fraction of total pages in each zone that a hot per cpu
|
|
|
|
* pagelist can have before it gets flushed back to buddy allocator.
|
2006-01-08 02:00:40 -07:00
|
|
|
*/
|
2014-06-06 15:38:09 -06:00
|
|
|
int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
|
2009-09-23 16:57:19 -06:00
|
|
|
void __user *buffer, size_t *length, loff_t *ppos)
|
2006-01-08 02:00:40 -07:00
|
|
|
{
|
|
|
|
struct zone *zone;
|
2014-06-23 14:22:04 -06:00
|
|
|
int old_percpu_pagelist_fraction;
|
2006-01-08 02:00:40 -07:00
|
|
|
int ret;
|
|
|
|
|
2014-06-23 14:22:04 -06:00
|
|
|
mutex_lock(&pcp_batch_high_lock);
|
|
|
|
old_percpu_pagelist_fraction = percpu_pagelist_fraction;
|
|
|
|
|
2009-09-23 16:57:19 -06:00
|
|
|
ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
|
2014-06-23 14:22:04 -06:00
|
|
|
if (!write || ret < 0)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
/* Sanity checking to avoid pcp imbalance */
|
|
|
|
if (percpu_pagelist_fraction &&
|
|
|
|
percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
|
|
|
|
percpu_pagelist_fraction = old_percpu_pagelist_fraction;
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* No change? */
|
|
|
|
if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
|
|
|
|
goto out;
|
2013-07-03 16:01:29 -06:00
|
|
|
|
2009-06-23 13:37:04 -06:00
|
|
|
for_each_populated_zone(zone) {
|
2014-06-23 14:22:04 -06:00
|
|
|
unsigned int cpu;
|
|
|
|
|
2013-07-03 16:01:34 -06:00
|
|
|
for_each_possible_cpu(cpu)
|
2014-06-23 14:22:04 -06:00
|
|
|
pageset_set_high_and_batch(zone,
|
|
|
|
per_cpu_ptr(zone->pageset, cpu));
|
2006-01-08 02:00:40 -07:00
|
|
|
}
|
2014-06-23 14:22:04 -06:00
|
|
|
out:
|
2013-07-03 16:01:29 -06:00
|
|
|
mutex_unlock(&pcp_batch_high_lock);
|
2014-06-23 14:22:04 -06:00
|
|
|
return ret;
|
2006-01-08 02:00:40 -07:00
|
|
|
}
|
|
|
|
|
2015-06-24 17:56:28 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
2006-08-24 04:08:07 -06:00
|
|
|
int hashdist = HASHDIST_DEFAULT;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
static int __init set_hashdist(char *str)
|
|
|
|
{
|
|
|
|
if (!str)
|
|
|
|
return 0;
|
|
|
|
hashdist = simple_strtoul(str, &str, 0);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
__setup("hashdist=", set_hashdist);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* allocate a large system hash table from bootmem
|
|
|
|
* - it is assumed that the hash table must contain an exact power-of-2
|
|
|
|
* quantity of entries
|
|
|
|
* - limit is the number of hash buckets, not the total allocation size
|
|
|
|
*/
|
|
|
|
void *__init alloc_large_system_hash(const char *tablename,
|
|
|
|
unsigned long bucketsize,
|
|
|
|
unsigned long numentries,
|
|
|
|
int scale,
|
|
|
|
int flags,
|
|
|
|
unsigned int *_hash_shift,
|
|
|
|
unsigned int *_hash_mask,
|
2012-05-23 07:33:35 -06:00
|
|
|
unsigned long low_limit,
|
|
|
|
unsigned long high_limit)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2012-05-23 07:33:35 -06:00
|
|
|
unsigned long long max = high_limit;
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned long log2qty, size;
|
|
|
|
void *table = NULL;
|
|
|
|
|
|
|
|
/* allow the kernel cmdline to have a say */
|
|
|
|
if (!numentries) {
|
|
|
|
/* round applicable memory size up to nearest megabyte */
|
2006-12-06 21:37:33 -07:00
|
|
|
numentries = nr_kernel_pages;
|
2013-09-11 15:20:26 -06:00
|
|
|
|
|
|
|
/* It isn't necessary when PAGE_SIZE >= 1MB */
|
|
|
|
if (PAGE_SHIFT < 20)
|
|
|
|
numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* limit to 1 bucket per 2^scale bytes of low memory */
|
|
|
|
if (scale > PAGE_SHIFT)
|
|
|
|
numentries >>= (scale - PAGE_SHIFT);
|
|
|
|
else
|
|
|
|
numentries <<= (PAGE_SHIFT - scale);
|
2007-01-05 17:36:30 -07:00
|
|
|
|
|
|
|
/* Make sure we've got at least a 0-order allocation.. */
|
2009-09-21 18:03:07 -06:00
|
|
|
if (unlikely(flags & HASH_SMALL)) {
|
|
|
|
/* Makes no sense without HASH_EARLY */
|
|
|
|
WARN_ON(!(flags & HASH_EARLY));
|
|
|
|
if (!(numentries >> *_hash_shift)) {
|
|
|
|
numentries = 1UL << *_hash_shift;
|
|
|
|
BUG_ON(!numentries);
|
|
|
|
}
|
|
|
|
} else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
|
2007-01-05 17:36:30 -07:00
|
|
|
numentries = PAGE_SIZE / bucketsize;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2006-03-25 04:08:02 -07:00
|
|
|
numentries = roundup_pow_of_two(numentries);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* limit allocation size to 1/16 total memory by default */
|
|
|
|
if (max == 0) {
|
|
|
|
max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
|
|
|
|
do_div(max, bucketsize);
|
|
|
|
}
|
2012-02-08 13:39:07 -07:00
|
|
|
max = min(max, 0x80000000ULL);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-05-23 07:33:35 -06:00
|
|
|
if (numentries < low_limit)
|
|
|
|
numentries = low_limit;
|
2005-04-16 16:20:36 -06:00
|
|
|
if (numentries > max)
|
|
|
|
numentries = max;
|
|
|
|
|
2006-12-08 03:37:49 -07:00
|
|
|
log2qty = ilog2(numentries);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
do {
|
|
|
|
size = bucketsize << log2qty;
|
|
|
|
if (flags & HASH_EARLY)
|
2014-01-21 16:50:25 -07:00
|
|
|
table = memblock_virt_alloc_nopanic(size, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
else if (hashdist)
|
|
|
|
table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
|
|
|
|
else {
|
2007-07-16 00:38:05 -06:00
|
|
|
/*
|
|
|
|
* If bucketsize is not a power-of-two, we may free
|
2009-06-16 16:32:19 -06:00
|
|
|
* some pages at the end of hash table which
|
|
|
|
* alloc_pages_exact() automatically does
|
2007-07-16 00:38:05 -06:00
|
|
|
*/
|
2009-07-07 03:33:01 -06:00
|
|
|
if (get_order(size) < MAX_ORDER) {
|
2009-06-16 16:32:19 -06:00
|
|
|
table = alloc_pages_exact(size, GFP_ATOMIC);
|
2009-07-07 03:33:01 -06:00
|
|
|
kmemleak_alloc(table, size, 1, GFP_ATOMIC);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
} while (!table && size > PAGE_SIZE && --log2qty);
|
|
|
|
|
|
|
|
if (!table)
|
|
|
|
panic("Failed to allocate %s hash table\n", tablename);
|
|
|
|
|
2016-03-17 15:19:50 -06:00
|
|
|
pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n",
|
|
|
|
tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (_hash_shift)
|
|
|
|
*_hash_shift = log2qty;
|
|
|
|
if (_hash_mask)
|
|
|
|
*_hash_mask = (1 << log2qty) - 1;
|
|
|
|
|
|
|
|
return table;
|
|
|
|
}
|
2006-03-27 02:15:25 -07:00
|
|
|
|
2007-10-16 02:26:11 -06:00
|
|
|
/*
|
2012-07-31 17:43:01 -06:00
|
|
|
* This function checks whether pageblock includes unmovable pages or not.
|
|
|
|
* If @count is not zero, it is okay to include less @count unmovable pages
|
|
|
|
*
|
2013-09-11 15:20:34 -06:00
|
|
|
* PageLRU check without isolation or lru_lock could race so that
|
2012-07-31 17:43:01 -06:00
|
|
|
* MIGRATE_MOVABLE block might include unmovable pages. It means you can't
|
|
|
|
* expect this function should be exact.
|
2007-10-16 02:26:11 -06:00
|
|
|
*/
|
2012-12-11 17:00:45 -07:00
|
|
|
bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
|
|
|
|
bool skip_hwpoisoned_pages)
|
2010-10-26 15:21:30 -06:00
|
|
|
{
|
|
|
|
unsigned long pfn, iter, found;
|
2011-12-29 05:09:50 -07:00
|
|
|
int mt;
|
|
|
|
|
2010-10-26 15:21:30 -06:00
|
|
|
/*
|
|
|
|
* For avoiding noise data, lru_add_drain_all() should be called
|
2012-07-31 17:43:01 -06:00
|
|
|
* If ZONE_MOVABLE, the zone never contains unmovable pages
|
2010-10-26 15:21:30 -06:00
|
|
|
*/
|
|
|
|
if (zone_idx(zone) == ZONE_MOVABLE)
|
2012-07-31 17:43:01 -06:00
|
|
|
return false;
|
2011-12-29 05:09:50 -07:00
|
|
|
mt = get_pageblock_migratetype(page);
|
|
|
|
if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt))
|
2012-07-31 17:43:01 -06:00
|
|
|
return false;
|
2010-10-26 15:21:30 -06:00
|
|
|
|
|
|
|
pfn = page_to_pfn(page);
|
|
|
|
for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
|
|
|
|
unsigned long check = pfn + iter;
|
|
|
|
|
2011-02-25 15:44:25 -07:00
|
|
|
if (!pfn_valid_within(check))
|
2010-10-26 15:21:30 -06:00
|
|
|
continue;
|
2011-02-25 15:44:25 -07:00
|
|
|
|
2010-10-26 15:21:30 -06:00
|
|
|
page = pfn_to_page(check);
|
2013-09-11 15:22:09 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Hugepages are not in LRU lists, but they're movable.
|
|
|
|
* We need not scan over tail pages bacause we don't
|
|
|
|
* handle each tail page individually in migration.
|
|
|
|
*/
|
|
|
|
if (PageHuge(page)) {
|
|
|
|
iter = round_up(iter + 1, 1<<compound_order(page)) - 1;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2012-07-31 17:42:59 -06:00
|
|
|
/*
|
|
|
|
* We can't use page_count without pin a page
|
|
|
|
* because another CPU can free compound page.
|
|
|
|
* This check already skips compound tails of THP
|
2016-05-19 18:10:49 -06:00
|
|
|
* because their page->_refcount is zero at all time.
|
2012-07-31 17:42:59 -06:00
|
|
|
*/
|
2016-03-17 15:19:26 -06:00
|
|
|
if (!page_ref_count(page)) {
|
2010-10-26 15:21:30 -06:00
|
|
|
if (PageBuddy(page))
|
|
|
|
iter += (1 << page_order(page)) - 1;
|
|
|
|
continue;
|
|
|
|
}
|
2012-07-31 17:42:59 -06:00
|
|
|
|
2012-12-11 17:00:45 -07:00
|
|
|
/*
|
|
|
|
* The HWPoisoned page may be not in buddy system, and
|
|
|
|
* page_count() is not 0.
|
|
|
|
*/
|
|
|
|
if (skip_hwpoisoned_pages && PageHWPoison(page))
|
|
|
|
continue;
|
|
|
|
|
2010-10-26 15:21:30 -06:00
|
|
|
if (!PageLRU(page))
|
|
|
|
found++;
|
|
|
|
/*
|
mm: vmscan: invoke slab shrinkers from shrink_zone()
The slab shrinkers are currently invoked from the zonelist walkers in
kswapd, direct reclaim, and zone reclaim, all of which roughly gauge the
eligible LRU pages and assemble a nodemask to pass to NUMA-aware
shrinkers, which then again have to walk over the nodemask. This is
redundant code, extra runtime work, and fairly inaccurate when it comes to
the estimation of actually scannable LRU pages. The code duplication will
only get worse when making the shrinkers cgroup-aware and requiring them
to have out-of-band cgroup hierarchy walks as well.
Instead, invoke the shrinkers from shrink_zone(), which is where all
reclaimers end up, to avoid this duplication.
Take the count for eligible LRU pages out of get_scan_count(), which
considers many more factors than just the availability of swap space, like
zone_reclaimable_pages() currently does. Accumulate the number over all
visited lruvecs to get the per-zone value.
Some nodes have multiple zones due to memory addressing restrictions. To
avoid putting too much pressure on the shrinkers, only invoke them once
for each such node, using the class zone of the allocation as the pivot
zone.
For now, this integrates the slab shrinking better into the reclaim logic
and gets rid of duplicative invocations from kswapd, direct reclaim, and
zone reclaim. It also prepares for cgroup-awareness, allowing
memcg-capable shrinkers to be added at the lruvec level without much
duplication of both code and runtime work.
This changes kswapd behavior, which used to invoke the shrinkers for each
zone, but with scan ratios gathered from the entire node, resulting in
meaningless pressure quantities on multi-zone nodes.
Zone reclaim behavior also changes. It used to shrink slabs until the
same amount of pages were shrunk as were reclaimed from the LRUs. Now it
merely invokes the shrinkers once with the zone's scan ratio, which makes
the shrinkers go easier on caches that implement aging and would prefer
feeding back pressure from recently used slab objects to unused LRU pages.
[vdavydov@parallels.com: assure class zone is populated]
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Dave Chinner <david@fromorbit.com>
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-12 17:56:13 -07:00
|
|
|
* If there are RECLAIMABLE pages, we need to check
|
|
|
|
* it. But now, memory offline itself doesn't call
|
|
|
|
* shrink_node_slabs() and it still to be fixed.
|
2010-10-26 15:21:30 -06:00
|
|
|
*/
|
|
|
|
/*
|
|
|
|
* If the page is not RAM, page_count()should be 0.
|
|
|
|
* we don't need more check. This is an _used_ not-movable page.
|
|
|
|
*
|
|
|
|
* The problematic thing here is PG_reserved pages. PG_reserved
|
|
|
|
* is set to both of a memory hole page and a _used_ kernel
|
|
|
|
* page at boot.
|
|
|
|
*/
|
|
|
|
if (found > count)
|
2012-07-31 17:43:01 -06:00
|
|
|
return true;
|
2010-10-26 15:21:30 -06:00
|
|
|
}
|
2012-07-31 17:43:01 -06:00
|
|
|
return false;
|
2010-10-26 15:21:30 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
bool is_pageblock_removable_nolock(struct page *page)
|
|
|
|
{
|
2012-01-20 15:33:58 -07:00
|
|
|
struct zone *zone;
|
|
|
|
unsigned long pfn;
|
2012-01-20 15:33:55 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We have to be careful here because we are iterating over memory
|
|
|
|
* sections which are not zone aware so we might end up outside of
|
|
|
|
* the zone but still within the section.
|
2012-01-20 15:33:58 -07:00
|
|
|
* We have to take care about the node as well. If the node is offline
|
|
|
|
* its NODE_DATA will be NULL - see page_zone.
|
2012-01-20 15:33:55 -07:00
|
|
|
*/
|
2012-01-20 15:33:58 -07:00
|
|
|
if (!node_online(page_to_nid(page)))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
zone = page_zone(page);
|
|
|
|
pfn = page_to_pfn(page);
|
2013-02-22 17:35:23 -07:00
|
|
|
if (!zone_spans_pfn(zone, pfn))
|
2012-01-20 15:33:55 -07:00
|
|
|
return false;
|
|
|
|
|
2012-12-11 17:00:45 -07:00
|
|
|
return !has_unmovable_pages(zone, page, 0, true);
|
2007-10-16 02:26:11 -06:00
|
|
|
}
|
2007-10-16 02:26:12 -06:00
|
|
|
|
2016-02-05 16:36:41 -07:00
|
|
|
#if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)
|
2011-12-29 05:09:50 -07:00
|
|
|
|
|
|
|
static unsigned long pfn_max_align_down(unsigned long pfn)
|
|
|
|
{
|
|
|
|
return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
|
|
|
|
pageblock_nr_pages) - 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long pfn_max_align_up(unsigned long pfn)
|
|
|
|
{
|
|
|
|
return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
|
|
|
|
pageblock_nr_pages));
|
|
|
|
}
|
|
|
|
|
|
|
|
/* [start, end) must belong to a single zone. */
|
2012-10-08 17:32:41 -06:00
|
|
|
static int __alloc_contig_migrate_range(struct compact_control *cc,
|
|
|
|
unsigned long start, unsigned long end)
|
2011-12-29 05:09:50 -07:00
|
|
|
{
|
|
|
|
/* This function is based on compact_zone() from compaction.c. */
|
2012-10-08 17:33:51 -06:00
|
|
|
unsigned long nr_reclaimed;
|
2011-12-29 05:09:50 -07:00
|
|
|
unsigned long pfn = start;
|
|
|
|
unsigned int tries = 0;
|
|
|
|
int ret = 0;
|
|
|
|
|
2012-12-12 14:51:19 -07:00
|
|
|
migrate_prep();
|
2011-12-29 05:09:50 -07:00
|
|
|
|
2012-10-08 17:32:41 -06:00
|
|
|
while (pfn < end || !list_empty(&cc->migratepages)) {
|
2011-12-29 05:09:50 -07:00
|
|
|
if (fatal_signal_pending(current)) {
|
|
|
|
ret = -EINTR;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2012-10-08 17:32:41 -06:00
|
|
|
if (list_empty(&cc->migratepages)) {
|
|
|
|
cc->nr_migratepages = 0;
|
mm, compaction: move pageblock checks up from isolate_migratepages_range()
isolate_migratepages_range() is the main function of the compaction
scanner, called either on a single pageblock by isolate_migratepages()
during regular compaction, or on an arbitrary range by CMA's
__alloc_contig_migrate_range(). It currently perfoms two pageblock-wide
compaction suitability checks, and because of the CMA callpath, it tracks
if it crossed a pageblock boundary in order to repeat those checks.
However, closer inspection shows that those checks are always true for CMA:
- isolation_suitable() is true because CMA sets cc->ignore_skip_hint to true
- migrate_async_suitable() check is skipped because CMA uses sync compaction
We can therefore move the compaction-specific checks to
isolate_migratepages() and simplify isolate_migratepages_range().
Furthermore, we can mimic the freepage scanner family of functions, which
has isolate_freepages_block() function called both by compaction from
isolate_freepages() and by CMA from isolate_freepages_range(), where each
use-case adds own specific glue code. This allows further code
simplification.
Thus, we rename isolate_migratepages_range() to
isolate_migratepages_block() and limit its functionality to a single
pageblock (or its subset). For CMA, a new different
isolate_migratepages_range() is created as a CMA-specific wrapper for the
_block() function. The checks specific to compaction are moved to
isolate_migratepages(). As part of the unification of these two families
of functions, we remove the redundant zone parameter where applicable,
since zone pointer is already passed in cc->zone.
Furthermore, going back to compact_zone() and compact_finished() when
pageblock is found unsuitable (now by isolate_migratepages()) is wasteful
- the checks are meant to skip pageblocks quickly. The patch therefore
also introduces a simple loop into isolate_migratepages() so that it does
not return immediately on failed pageblock checks, but keeps going until
isolate_migratepages_range() gets called once. Similarily to
isolate_freepages(), the function periodically checks if it needs to
reschedule or abort async compaction.
[iamjoonsoo.kim@lge.com: fix isolated page counting bug in compaction]
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Minchan Kim <minchan@kernel.org>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-09 16:27:09 -06:00
|
|
|
pfn = isolate_migratepages_range(cc, pfn, end);
|
2011-12-29 05:09:50 -07:00
|
|
|
if (!pfn) {
|
|
|
|
ret = -EINTR;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
tries = 0;
|
|
|
|
} else if (++tries == 5) {
|
|
|
|
ret = ret < 0 ? ret : -EBUSY;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2012-10-08 17:33:51 -06:00
|
|
|
nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
|
|
|
|
&cc->migratepages);
|
|
|
|
cc->nr_migratepages -= nr_reclaimed;
|
2012-10-08 17:31:55 -06:00
|
|
|
|
2013-02-22 17:35:14 -07:00
|
|
|
ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
|
2014-06-04 17:08:28 -06:00
|
|
|
NULL, 0, cc->mode, MR_CMA);
|
2011-12-29 05:09:50 -07:00
|
|
|
}
|
2013-02-22 17:32:09 -07:00
|
|
|
if (ret < 0) {
|
|
|
|
putback_movable_pages(&cc->migratepages);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
return 0;
|
2011-12-29 05:09:50 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* alloc_contig_range() -- tries to allocate given range of pages
|
|
|
|
* @start: start PFN to allocate
|
|
|
|
* @end: one-past-the-last PFN to allocate
|
2012-04-03 07:06:15 -06:00
|
|
|
* @migratetype: migratetype of the underlaying pageblocks (either
|
|
|
|
* #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
|
|
|
|
* in range must have the same migratetype and it must
|
|
|
|
* be either of the two.
|
2011-12-29 05:09:50 -07:00
|
|
|
*
|
|
|
|
* The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
|
|
|
|
* aligned, however it's the caller's responsibility to guarantee that
|
|
|
|
* we are the only thread that changes migrate type of pageblocks the
|
|
|
|
* pages fall in.
|
|
|
|
*
|
|
|
|
* The PFN range must belong to a single zone.
|
|
|
|
*
|
|
|
|
* Returns zero on success or negative error code. On success all
|
|
|
|
* pages which PFN is in [start, end) are allocated for the caller and
|
|
|
|
* need to be freed with free_contig_range().
|
|
|
|
*/
|
2012-04-03 07:06:15 -06:00
|
|
|
int alloc_contig_range(unsigned long start, unsigned long end,
|
|
|
|
unsigned migratetype)
|
2011-12-29 05:09:50 -07:00
|
|
|
{
|
|
|
|
unsigned long outer_start, outer_end;
|
2015-11-06 17:29:57 -07:00
|
|
|
unsigned int order;
|
|
|
|
int ret = 0;
|
2011-12-29 05:09:50 -07:00
|
|
|
|
2012-10-08 17:32:41 -06:00
|
|
|
struct compact_control cc = {
|
|
|
|
.nr_migratepages = 0,
|
|
|
|
.order = -1,
|
|
|
|
.zone = page_zone(pfn_to_page(start)),
|
2014-06-04 17:08:28 -06:00
|
|
|
.mode = MIGRATE_SYNC,
|
2012-10-08 17:32:41 -06:00
|
|
|
.ignore_skip_hint = true,
|
|
|
|
};
|
|
|
|
INIT_LIST_HEAD(&cc.migratepages);
|
|
|
|
|
2011-12-29 05:09:50 -07:00
|
|
|
/*
|
|
|
|
* What we do here is we mark all pageblocks in range as
|
|
|
|
* MIGRATE_ISOLATE. Because pageblock and max order pages may
|
|
|
|
* have different sizes, and due to the way page allocator
|
|
|
|
* work, we align the range to biggest of the two pages so
|
|
|
|
* that page allocator won't try to merge buddies from
|
|
|
|
* different pageblocks and change MIGRATE_ISOLATE to some
|
|
|
|
* other migration type.
|
|
|
|
*
|
|
|
|
* Once the pageblocks are marked as MIGRATE_ISOLATE, we
|
|
|
|
* migrate the pages from an unaligned range (ie. pages that
|
|
|
|
* we are interested in). This will put all the pages in
|
|
|
|
* range back to page allocator as MIGRATE_ISOLATE.
|
|
|
|
*
|
|
|
|
* When this is done, we take the pages in range from page
|
|
|
|
* allocator removing them from the buddy system. This way
|
|
|
|
* page allocator will never consider using them.
|
|
|
|
*
|
|
|
|
* This lets us mark the pageblocks back as
|
|
|
|
* MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
|
|
|
|
* aligned range but not in the unaligned, original range are
|
|
|
|
* put back to page allocator so that buddy can use them.
|
|
|
|
*/
|
|
|
|
|
|
|
|
ret = start_isolate_page_range(pfn_max_align_down(start),
|
2012-12-11 17:00:45 -07:00
|
|
|
pfn_max_align_up(end), migratetype,
|
|
|
|
false);
|
2011-12-29 05:09:50 -07:00
|
|
|
if (ret)
|
2012-10-25 14:37:56 -06:00
|
|
|
return ret;
|
2011-12-29 05:09:50 -07:00
|
|
|
|
mm/cma: always check which page caused allocation failure
Now, we have tracepoint in test_pages_isolated() to notify pfn which
cannot be isolated. But, in alloc_contig_range(), some error path
doesn't call test_pages_isolated() so it's still hard to know exact pfn
that causes allocation failure.
This patch change this situation by calling test_pages_isolated() in
almost error path. In allocation failure case, some overhead is added
by this change, but, allocation failure is really rare event so it would
not matter.
In fatal signal pending case, we don't call test_pages_isolated()
because this failure is intentional one.
There was a bogus outer_start problem due to unchecked buddy order and
this patch also fix it. Before this patch, it didn't matter, because
end result is same thing. But, after this patch, tracepoint will report
failed pfn so it should be accurate.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Nazarewicz <mina86@mina86.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-14 16:18:45 -07:00
|
|
|
/*
|
|
|
|
* In case of -EBUSY, we'd like to know which page causes problem.
|
|
|
|
* So, just fall through. We will check it in test_pages_isolated().
|
|
|
|
*/
|
2012-10-08 17:32:41 -06:00
|
|
|
ret = __alloc_contig_migrate_range(&cc, start, end);
|
mm/cma: always check which page caused allocation failure
Now, we have tracepoint in test_pages_isolated() to notify pfn which
cannot be isolated. But, in alloc_contig_range(), some error path
doesn't call test_pages_isolated() so it's still hard to know exact pfn
that causes allocation failure.
This patch change this situation by calling test_pages_isolated() in
almost error path. In allocation failure case, some overhead is added
by this change, but, allocation failure is really rare event so it would
not matter.
In fatal signal pending case, we don't call test_pages_isolated()
because this failure is intentional one.
There was a bogus outer_start problem due to unchecked buddy order and
this patch also fix it. Before this patch, it didn't matter, because
end result is same thing. But, after this patch, tracepoint will report
failed pfn so it should be accurate.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Nazarewicz <mina86@mina86.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-14 16:18:45 -07:00
|
|
|
if (ret && ret != -EBUSY)
|
2011-12-29 05:09:50 -07:00
|
|
|
goto done;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Pages from [start, end) are within a MAX_ORDER_NR_PAGES
|
|
|
|
* aligned blocks that are marked as MIGRATE_ISOLATE. What's
|
|
|
|
* more, all pages in [start, end) are free in page allocator.
|
|
|
|
* What we are going to do is to allocate all pages from
|
|
|
|
* [start, end) (that is remove them from page allocator).
|
|
|
|
*
|
|
|
|
* The only problem is that pages at the beginning and at the
|
|
|
|
* end of interesting range may be not aligned with pages that
|
|
|
|
* page allocator holds, ie. they can be part of higher order
|
|
|
|
* pages. Because of this, we reserve the bigger range and
|
|
|
|
* once this is done free the pages we are not interested in.
|
|
|
|
*
|
|
|
|
* We don't have to hold zone->lock here because the pages are
|
|
|
|
* isolated thus they won't get removed from buddy.
|
|
|
|
*/
|
|
|
|
|
|
|
|
lru_add_drain_all();
|
2014-12-10 16:43:07 -07:00
|
|
|
drain_all_pages(cc.zone);
|
2011-12-29 05:09:50 -07:00
|
|
|
|
|
|
|
order = 0;
|
|
|
|
outer_start = start;
|
|
|
|
while (!PageBuddy(pfn_to_page(outer_start))) {
|
|
|
|
if (++order >= MAX_ORDER) {
|
mm/cma: always check which page caused allocation failure
Now, we have tracepoint in test_pages_isolated() to notify pfn which
cannot be isolated. But, in alloc_contig_range(), some error path
doesn't call test_pages_isolated() so it's still hard to know exact pfn
that causes allocation failure.
This patch change this situation by calling test_pages_isolated() in
almost error path. In allocation failure case, some overhead is added
by this change, but, allocation failure is really rare event so it would
not matter.
In fatal signal pending case, we don't call test_pages_isolated()
because this failure is intentional one.
There was a bogus outer_start problem due to unchecked buddy order and
this patch also fix it. Before this patch, it didn't matter, because
end result is same thing. But, after this patch, tracepoint will report
failed pfn so it should be accurate.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Nazarewicz <mina86@mina86.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-14 16:18:45 -07:00
|
|
|
outer_start = start;
|
|
|
|
break;
|
2011-12-29 05:09:50 -07:00
|
|
|
}
|
|
|
|
outer_start &= ~0UL << order;
|
|
|
|
}
|
|
|
|
|
mm/cma: always check which page caused allocation failure
Now, we have tracepoint in test_pages_isolated() to notify pfn which
cannot be isolated. But, in alloc_contig_range(), some error path
doesn't call test_pages_isolated() so it's still hard to know exact pfn
that causes allocation failure.
This patch change this situation by calling test_pages_isolated() in
almost error path. In allocation failure case, some overhead is added
by this change, but, allocation failure is really rare event so it would
not matter.
In fatal signal pending case, we don't call test_pages_isolated()
because this failure is intentional one.
There was a bogus outer_start problem due to unchecked buddy order and
this patch also fix it. Before this patch, it didn't matter, because
end result is same thing. But, after this patch, tracepoint will report
failed pfn so it should be accurate.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Nazarewicz <mina86@mina86.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-14 16:18:45 -07:00
|
|
|
if (outer_start != start) {
|
|
|
|
order = page_order(pfn_to_page(outer_start));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* outer_start page could be small order buddy page and
|
|
|
|
* it doesn't include start page. Adjust outer_start
|
|
|
|
* in this case to report failed page properly
|
|
|
|
* on tracepoint in test_pages_isolated()
|
|
|
|
*/
|
|
|
|
if (outer_start + (1UL << order) <= start)
|
|
|
|
outer_start = start;
|
|
|
|
}
|
|
|
|
|
2011-12-29 05:09:50 -07:00
|
|
|
/* Make sure the range is really isolated. */
|
2012-12-11 17:00:45 -07:00
|
|
|
if (test_pages_isolated(outer_start, end, false)) {
|
2014-11-13 16:19:27 -07:00
|
|
|
pr_info("%s: [%lx, %lx) PFNs busy\n",
|
|
|
|
__func__, outer_start, end);
|
2011-12-29 05:09:50 -07:00
|
|
|
ret = -EBUSY;
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
2012-01-25 04:49:24 -07:00
|
|
|
/* Grab isolated pages from freelists. */
|
2012-10-08 17:32:41 -06:00
|
|
|
outer_end = isolate_freepages_range(&cc, outer_start, end);
|
2011-12-29 05:09:50 -07:00
|
|
|
if (!outer_end) {
|
|
|
|
ret = -EBUSY;
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Free head and tail (if any) */
|
|
|
|
if (start != outer_start)
|
|
|
|
free_contig_range(outer_start, start - outer_start);
|
|
|
|
if (end != outer_end)
|
|
|
|
free_contig_range(end, outer_end - end);
|
|
|
|
|
|
|
|
done:
|
|
|
|
undo_isolate_page_range(pfn_max_align_down(start),
|
2012-04-03 07:06:15 -06:00
|
|
|
pfn_max_align_up(end), migratetype);
|
2011-12-29 05:09:50 -07:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
void free_contig_range(unsigned long pfn, unsigned nr_pages)
|
|
|
|
{
|
2012-12-20 16:05:18 -07:00
|
|
|
unsigned int count = 0;
|
|
|
|
|
|
|
|
for (; nr_pages--; pfn++) {
|
|
|
|
struct page *page = pfn_to_page(pfn);
|
|
|
|
|
|
|
|
count += page_count(page) != 1;
|
|
|
|
__free_page(page);
|
|
|
|
}
|
|
|
|
WARN(count != 0, "%d pages are still in use!\n", count);
|
2011-12-29 05:09:50 -07:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2012-07-31 17:43:35 -06:00
|
|
|
#ifdef CONFIG_MEMORY_HOTPLUG
|
2013-07-03 16:01:33 -06:00
|
|
|
/*
|
|
|
|
* The zone indicated has a new number of managed_pages; batch sizes and percpu
|
|
|
|
* page high values need to be recalulated.
|
|
|
|
*/
|
2012-07-31 17:43:35 -06:00
|
|
|
void __meminit zone_pcp_update(struct zone *zone)
|
|
|
|
{
|
2013-07-03 16:01:33 -06:00
|
|
|
unsigned cpu;
|
2013-07-03 16:01:29 -06:00
|
|
|
mutex_lock(&pcp_batch_high_lock);
|
2013-07-03 16:01:33 -06:00
|
|
|
for_each_possible_cpu(cpu)
|
2013-07-03 16:01:41 -06:00
|
|
|
pageset_set_high_and_batch(zone,
|
|
|
|
per_cpu_ptr(zone->pageset, cpu));
|
2013-07-03 16:01:29 -06:00
|
|
|
mutex_unlock(&pcp_batch_high_lock);
|
2012-07-31 17:43:35 -06:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2012-07-31 17:43:32 -06:00
|
|
|
void zone_pcp_reset(struct zone *zone)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
2012-10-08 17:33:39 -06:00
|
|
|
int cpu;
|
|
|
|
struct per_cpu_pageset *pset;
|
2012-07-31 17:43:32 -06:00
|
|
|
|
|
|
|
/* avoid races with drain_pages() */
|
|
|
|
local_irq_save(flags);
|
|
|
|
if (zone->pageset != &boot_pageset) {
|
2012-10-08 17:33:39 -06:00
|
|
|
for_each_online_cpu(cpu) {
|
|
|
|
pset = per_cpu_ptr(zone->pageset, cpu);
|
|
|
|
drain_zonestat(zone, pset);
|
|
|
|
}
|
2012-07-31 17:43:32 -06:00
|
|
|
free_percpu(zone->pageset);
|
|
|
|
zone->pageset = &boot_pageset;
|
|
|
|
}
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
2012-12-11 17:01:01 -07:00
|
|
|
#ifdef CONFIG_MEMORY_HOTREMOVE
|
2007-10-16 02:26:12 -06:00
|
|
|
/*
|
2016-05-19 18:12:06 -06:00
|
|
|
* All pages in the range must be in a single zone and isolated
|
|
|
|
* before calling this.
|
2007-10-16 02:26:12 -06:00
|
|
|
*/
|
|
|
|
void
|
|
|
|
__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
|
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
struct zone *zone;
|
2014-06-04 17:10:21 -06:00
|
|
|
unsigned int order, i;
|
2007-10-16 02:26:12 -06:00
|
|
|
unsigned long pfn;
|
|
|
|
unsigned long flags;
|
|
|
|
/* find the first valid pfn */
|
|
|
|
for (pfn = start_pfn; pfn < end_pfn; pfn++)
|
|
|
|
if (pfn_valid(pfn))
|
|
|
|
break;
|
|
|
|
if (pfn == end_pfn)
|
|
|
|
return;
|
|
|
|
zone = page_zone(pfn_to_page(pfn));
|
|
|
|
spin_lock_irqsave(&zone->lock, flags);
|
|
|
|
pfn = start_pfn;
|
|
|
|
while (pfn < end_pfn) {
|
|
|
|
if (!pfn_valid(pfn)) {
|
|
|
|
pfn++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
page = pfn_to_page(pfn);
|
2012-12-11 17:00:45 -07:00
|
|
|
/*
|
|
|
|
* The HWPoisoned page may be not in buddy system, and
|
|
|
|
* page_count() is not 0.
|
|
|
|
*/
|
|
|
|
if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
|
|
|
|
pfn++;
|
|
|
|
SetPageReserved(page);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2007-10-16 02:26:12 -06:00
|
|
|
BUG_ON(page_count(page));
|
|
|
|
BUG_ON(!PageBuddy(page));
|
|
|
|
order = page_order(page);
|
|
|
|
#ifdef CONFIG_DEBUG_VM
|
2016-03-17 15:19:50 -06:00
|
|
|
pr_info("remove from free list %lx %d %lx\n",
|
|
|
|
pfn, 1 << order, end_pfn);
|
2007-10-16 02:26:12 -06:00
|
|
|
#endif
|
|
|
|
list_del(&page->lru);
|
|
|
|
rmv_page_order(page);
|
|
|
|
zone->free_area[order].nr_free--;
|
|
|
|
for (i = 0; i < (1 << order); i++)
|
|
|
|
SetPageReserved((page+i));
|
|
|
|
pfn += (1 << order);
|
|
|
|
}
|
|
|
|
spin_unlock_irqrestore(&zone->lock, flags);
|
|
|
|
}
|
|
|
|
#endif
|
2009-12-16 04:19:58 -07:00
|
|
|
|
|
|
|
bool is_free_buddy_page(struct page *page)
|
|
|
|
{
|
|
|
|
struct zone *zone = page_zone(page);
|
|
|
|
unsigned long pfn = page_to_pfn(page);
|
|
|
|
unsigned long flags;
|
2014-06-04 17:10:21 -06:00
|
|
|
unsigned int order;
|
2009-12-16 04:19:58 -07:00
|
|
|
|
|
|
|
spin_lock_irqsave(&zone->lock, flags);
|
|
|
|
for (order = 0; order < MAX_ORDER; order++) {
|
|
|
|
struct page *page_head = page - (pfn & ((1 << order) - 1));
|
|
|
|
|
|
|
|
if (PageBuddy(page_head) && page_order(page_head) >= order)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
spin_unlock_irqrestore(&zone->lock, flags);
|
|
|
|
|
|
|
|
return order < MAX_ORDER;
|
|
|
|
}
|