2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* linux/mm/slab.c
|
|
|
|
* Written by Mark Hemment, 1996/97.
|
|
|
|
* (markhe@nextd.demon.co.uk)
|
|
|
|
*
|
|
|
|
* kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
|
|
|
|
*
|
|
|
|
* Major cleanup, different bufctl logic, per-cpu arrays
|
|
|
|
* (c) 2000 Manfred Spraul
|
|
|
|
*
|
|
|
|
* Cleanup, make the head arrays unconditional, preparation for NUMA
|
|
|
|
* (c) 2002 Manfred Spraul
|
|
|
|
*
|
|
|
|
* An implementation of the Slab Allocator as described in outline in;
|
|
|
|
* UNIX Internals: The New Frontiers by Uresh Vahalia
|
|
|
|
* Pub: Prentice Hall ISBN 0-13-101908-2
|
|
|
|
* or with a little more detail in;
|
|
|
|
* The Slab Allocator: An Object-Caching Kernel Memory Allocator
|
|
|
|
* Jeff Bonwick (Sun Microsystems).
|
|
|
|
* Presented at: USENIX Summer 1994 Technical Conference
|
|
|
|
*
|
|
|
|
* The memory is organized in caches, one cache for each object type.
|
|
|
|
* (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
|
|
|
|
* Each cache consists out of many slabs (they are small (usually one
|
|
|
|
* page long) and always contiguous), and each slab contains multiple
|
|
|
|
* initialized objects.
|
|
|
|
*
|
|
|
|
* This means, that your constructor is used only for newly allocated
|
2007-10-19 17:27:18 -06:00
|
|
|
* slabs and you must pass objects with the same initializations to
|
2005-04-16 16:20:36 -06:00
|
|
|
* kmem_cache_free.
|
|
|
|
*
|
|
|
|
* Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
|
|
|
|
* normal). If you need a special memory type, then must create a new
|
|
|
|
* cache for that memory type.
|
|
|
|
*
|
|
|
|
* In order to reduce fragmentation, the slabs are sorted in 3 groups:
|
|
|
|
* full slabs with 0 free objects
|
|
|
|
* partial slabs
|
|
|
|
* empty slabs with no allocated objects
|
|
|
|
*
|
|
|
|
* If partial slabs exist, then new allocations come from these slabs,
|
|
|
|
* otherwise from empty slabs or new slabs are allocated.
|
|
|
|
*
|
|
|
|
* kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
|
|
|
|
* during kmem_cache_destroy(). The caller must prevent concurrent allocs.
|
|
|
|
*
|
|
|
|
* Each cache has a short per-cpu head array, most allocs
|
|
|
|
* and frees go into that array, and if that array overflows, then 1/2
|
|
|
|
* of the entries in the array are given back into the global cache.
|
|
|
|
* The head array is strictly LIFO and should improve the cache hit rates.
|
|
|
|
* On SMP, it additionally reduces the spinlock operations.
|
|
|
|
*
|
2006-03-22 01:08:11 -07:00
|
|
|
* The c_cpuarray may not be read with enabled local interrupts -
|
2005-04-16 16:20:36 -06:00
|
|
|
* it's changed with a smp_call_function().
|
|
|
|
*
|
|
|
|
* SMP synchronization:
|
|
|
|
* constructors and destructors are called without any locking.
|
2006-02-01 04:05:50 -07:00
|
|
|
* Several members in struct kmem_cache and struct slab never change, they
|
2005-04-16 16:20:36 -06:00
|
|
|
* are accessed without any locking.
|
|
|
|
* The per-cpu arrays are never accessed from the wrong cpu, no locking,
|
|
|
|
* and local interrupts are disabled so slab code is preempt-safe.
|
|
|
|
* The non-constant members are protected with a per-cache irq spinlock.
|
|
|
|
*
|
|
|
|
* Many thanks to Mark Hemment, who wrote another per-cpu slab patch
|
|
|
|
* in 2000 - many ideas in the current implementation are derived from
|
|
|
|
* his patch.
|
|
|
|
*
|
|
|
|
* Further notes from the original documentation:
|
|
|
|
*
|
|
|
|
* 11 April '97. Started multi-threading - markhe
|
2012-07-06 14:25:12 -06:00
|
|
|
* The global cache-chain is protected by the mutex 'slab_mutex'.
|
2005-04-16 16:20:36 -06:00
|
|
|
* The sem is only needed when accessing/extending the cache-chain, which
|
|
|
|
* can never happen inside an interrupt (kmem_cache_create(),
|
|
|
|
* kmem_cache_shrink() and kmem_cache_reap()).
|
|
|
|
*
|
|
|
|
* At present, each engine can be growing a cache. This should be blocked.
|
|
|
|
*
|
2005-09-09 14:03:32 -06:00
|
|
|
* 15 March 2005. NUMA slab allocator.
|
|
|
|
* Shai Fultheim <shai@scalex86.org>.
|
|
|
|
* Shobhit Dayal <shobhit@calsoftinc.com>
|
|
|
|
* Alok N Kataria <alokk@calsoftinc.com>
|
|
|
|
* Christoph Lameter <christoph@lameter.com>
|
|
|
|
*
|
|
|
|
* Modified the slab allocator to be node aware on NUMA systems.
|
|
|
|
* Each node has its own list of partial, free and full slabs.
|
|
|
|
* All object allocations for a node occur from node specific slab lists.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/mm.h>
|
2006-06-27 03:53:52 -06:00
|
|
|
#include <linux/poison.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/swap.h>
|
|
|
|
#include <linux/cache.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/compiler.h>
|
[PATCH] cpuset memory spread slab cache implementation
Provide the slab cache infrastructure to support cpuset memory spreading.
See the previous patches, cpuset_mem_spread, for an explanation of cpuset
memory spreading.
This patch provides a slab cache SLAB_MEM_SPREAD flag. If set in the
kmem_cache_create() call defining a slab cache, then any task marked with the
process state flag PF_MEMSPREAD will spread memory page allocations for that
cache over all the allowed nodes, instead of preferring the local (faulting)
node.
On systems not configured with CONFIG_NUMA, this results in no change to the
page allocation code path for slab caches.
On systems with cpusets configured in the kernel, but the "memory_spread"
cpuset option not enabled for the current tasks cpuset, this adds a call to a
cpuset routine and failed bit test of the processor state flag PF_SPREAD_SLAB.
For tasks so marked, a second inline test is done for the slab cache flag
SLAB_MEM_SPREAD, and if that is set and if the allocation is not
in_interrupt(), this adds a call to to a cpuset routine that computes which of
the tasks mems_allowed nodes should be preferred for this allocation.
==> This patch adds another hook into the performance critical
code path to allocating objects from the slab cache, in the
____cache_alloc() chunk, below. The next patch optimizes this
hook, reducing the impact of the combined mempolicy plus memory
spreading hooks on this critical code path to a single check
against the tasks task_struct flags word.
This patch provides the generic slab flags and logic needed to apply memory
spreading to a particular slab.
A subsequent patch will mark a few specific slab caches for this placement
policy.
Signed-off-by: Paul Jackson <pj@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-24 04:16:07 -07:00
|
|
|
#include <linux/cpuset.h>
|
2008-10-05 14:59:10 -06:00
|
|
|
#include <linux/proc_fs.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/kallsyms.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/sysctl.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/rcupdate.h>
|
2005-06-23 01:09:02 -06:00
|
|
|
#include <linux/string.h>
|
2006-12-06 21:36:41 -07:00
|
|
|
#include <linux/uaccess.h>
|
2005-09-09 14:03:32 -06:00
|
|
|
#include <linux/nodemask.h>
|
2009-06-11 06:22:40 -06:00
|
|
|
#include <linux/kmemleak.h>
|
2006-01-18 18:42:36 -07:00
|
|
|
#include <linux/mempolicy.h>
|
2006-01-18 18:42:33 -07:00
|
|
|
#include <linux/mutex.h>
|
2006-12-08 03:39:44 -07:00
|
|
|
#include <linux/fault-inject.h>
|
2006-06-27 03:54:55 -06:00
|
|
|
#include <linux/rtmutex.h>
|
2006-12-13 01:34:27 -07:00
|
|
|
#include <linux/reciprocal_div.h>
|
2008-04-30 01:55:01 -06:00
|
|
|
#include <linux/debugobjects.h>
|
2008-05-09 12:35:53 -06:00
|
|
|
#include <linux/kmemcheck.h>
|
2010-03-27 20:40:47 -06:00
|
|
|
#include <linux/memory.h>
|
2011-05-20 13:50:29 -06:00
|
|
|
#include <linux/prefetch.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-07-31 17:44:30 -06:00
|
|
|
#include <net/sock.h>
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <asm/cacheflush.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include <asm/page.h>
|
|
|
|
|
2012-01-09 15:15:42 -07:00
|
|
|
#include <trace/events/kmem.h>
|
|
|
|
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
#include "internal.h"
|
|
|
|
|
2012-12-18 15:22:46 -07:00
|
|
|
#include "slab.h"
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
2007-05-06 15:50:16 -06:00
|
|
|
* DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
|
2005-04-16 16:20:36 -06:00
|
|
|
* 0 for faster, smaller code (especially in the critical paths).
|
|
|
|
*
|
|
|
|
* STATS - 1 to collect stats for /proc/slabinfo.
|
|
|
|
* 0 for faster, smaller code (especially in the critical paths).
|
|
|
|
*
|
|
|
|
* FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_SLAB
|
|
|
|
#define DEBUG 1
|
|
|
|
#define STATS 1
|
|
|
|
#define FORCED_DEBUG 1
|
|
|
|
#else
|
|
|
|
#define DEBUG 0
|
|
|
|
#define STATS 0
|
|
|
|
#define FORCED_DEBUG 0
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* Shouldn't this be in a header file somewhere? */
|
|
|
|
#define BYTES_PER_WORD sizeof(void *)
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-04 19:26:44 -06:00
|
|
|
#define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long))
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
#ifndef ARCH_KMALLOC_FLAGS
|
|
|
|
#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
|
|
|
|
#endif
|
|
|
|
|
2013-12-02 01:49:41 -07:00
|
|
|
#define FREELIST_BYTE_INDEX (((PAGE_SIZE >> BITS_PER_BYTE) \
|
|
|
|
<= SLAB_OBJ_MIN_SIZE) ? 1 : 0)
|
|
|
|
|
|
|
|
#if FREELIST_BYTE_INDEX
|
|
|
|
typedef unsigned char freelist_idx_t;
|
|
|
|
#else
|
|
|
|
typedef unsigned short freelist_idx_t;
|
|
|
|
#endif
|
|
|
|
|
2014-05-05 14:20:04 -06:00
|
|
|
#define SLAB_OBJ_MAX_NUM ((1 << sizeof(freelist_idx_t) * BITS_PER_BYTE) - 1)
|
2013-12-02 01:49:41 -07:00
|
|
|
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
/*
|
|
|
|
* true if a page was allocated from pfmemalloc reserves for network-based
|
|
|
|
* swap
|
|
|
|
*/
|
|
|
|
static bool pfmemalloc_active __read_mostly;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* struct array_cache
|
|
|
|
*
|
|
|
|
* Purpose:
|
|
|
|
* - LIFO ordering, to hand out cache-warm objects from _alloc
|
|
|
|
* - reduce the number of linked list operations
|
|
|
|
* - reduce spinlock operations
|
|
|
|
*
|
|
|
|
* The limit is stored in the per-cpu structure to reduce the data cache
|
|
|
|
* footprint.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
struct array_cache {
|
|
|
|
unsigned int avail;
|
|
|
|
unsigned int limit;
|
|
|
|
unsigned int batchcount;
|
|
|
|
unsigned int touched;
|
2007-10-17 00:30:05 -06:00
|
|
|
void *entry[]; /*
|
2006-03-22 01:08:11 -07:00
|
|
|
* Must have this definition in here for the proper
|
|
|
|
* alignment of array_cache. Also simplifies accessing
|
|
|
|
* the entries.
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
*
|
|
|
|
* Entries should not be directly dereferenced as
|
|
|
|
* entries belonging to slabs marked pfmemalloc will
|
|
|
|
* have the lower bits set SLAB_OBJ_PFMEMALLOC
|
2006-03-22 01:08:11 -07:00
|
|
|
*/
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
2014-08-06 17:04:29 -06:00
|
|
|
struct alien_cache {
|
|
|
|
spinlock_t lock;
|
|
|
|
struct array_cache ac;
|
|
|
|
};
|
|
|
|
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
#define SLAB_OBJ_PFMEMALLOC 1
|
|
|
|
static inline bool is_obj_pfmemalloc(void *objp)
|
|
|
|
{
|
|
|
|
return (unsigned long)objp & SLAB_OBJ_PFMEMALLOC;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void set_obj_pfmemalloc(void **objp)
|
|
|
|
{
|
|
|
|
*objp = (void *)((unsigned long)*objp | SLAB_OBJ_PFMEMALLOC);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void clear_obj_pfmemalloc(void **objp)
|
|
|
|
{
|
|
|
|
*objp = (void *)((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC);
|
|
|
|
}
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* bootstrap: The caches do not work without cpuarrays anymore, but the
|
|
|
|
* cpuarrays are allocated from the generic caches...
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
#define BOOT_CPUCACHE_ENTRIES 1
|
|
|
|
struct arraycache_init {
|
|
|
|
struct array_cache cache;
|
2006-01-08 02:00:37 -07:00
|
|
|
void *entries[BOOT_CPUCACHE_ENTRIES];
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
2005-09-09 14:03:32 -06:00
|
|
|
/*
|
|
|
|
* Need this for bootstrapping a per node allocator.
|
|
|
|
*/
|
2014-10-09 16:26:27 -06:00
|
|
|
#define NUM_INIT_LISTS (2 * MAX_NUMNODES)
|
2013-01-10 12:14:19 -07:00
|
|
|
static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS];
|
2005-09-09 14:03:32 -06:00
|
|
|
#define CACHE_CACHE 0
|
2014-10-09 16:26:27 -06:00
|
|
|
#define SIZE_NODE (MAX_NUMNODES)
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2006-06-30 02:55:45 -06:00
|
|
|
static int drain_freelist(struct kmem_cache *cache,
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n, int tofree);
|
2006-06-30 02:55:45 -06:00
|
|
|
static void free_block(struct kmem_cache *cachep, void **objpp, int len,
|
2014-08-06 17:04:25 -06:00
|
|
|
int node, struct list_head *list);
|
|
|
|
static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list);
|
2009-06-10 10:40:04 -06:00
|
|
|
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
|
2006-11-22 07:55:48 -07:00
|
|
|
static void cache_reap(struct work_struct *unused);
|
2006-06-30 02:55:45 -06:00
|
|
|
|
2006-06-23 03:03:46 -06:00
|
|
|
static int slab_early_init = 1;
|
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node))
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
static void kmem_cache_node_init(struct kmem_cache_node *parent)
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
|
|
|
INIT_LIST_HEAD(&parent->slabs_full);
|
|
|
|
INIT_LIST_HEAD(&parent->slabs_partial);
|
|
|
|
INIT_LIST_HEAD(&parent->slabs_free);
|
|
|
|
parent->shared = NULL;
|
|
|
|
parent->alien = NULL;
|
2006-02-05 00:27:56 -07:00
|
|
|
parent->colour_next = 0;
|
2005-09-09 14:03:32 -06:00
|
|
|
spin_lock_init(&parent->list_lock);
|
|
|
|
parent->free_objects = 0;
|
|
|
|
parent->free_touched = 0;
|
|
|
|
}
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
#define MAKE_LIST(cachep, listp, slab, nodeid) \
|
|
|
|
do { \
|
|
|
|
INIT_LIST_HEAD(listp); \
|
2014-08-06 17:04:11 -06:00
|
|
|
list_splice(&get_node(cachep, nodeid)->slab, listp); \
|
2005-09-09 14:03:32 -06:00
|
|
|
} while (0)
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
|
|
|
|
do { \
|
2005-09-09 14:03:32 -06:00
|
|
|
MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
|
|
|
|
MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
|
|
|
|
MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
|
|
|
|
} while (0)
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
#define CFLGS_OFF_SLAB (0x80000000UL)
|
|
|
|
#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
|
|
|
|
|
|
|
|
#define BATCHREFILL_LIMIT 16
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* Optimization question: fewer reaps means less probability for unnessary
|
|
|
|
* cpucache drain/refill cycles.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
2005-11-08 08:44:08 -07:00
|
|
|
* OTOH the cpuarrays can contain lots of objects,
|
2005-04-16 16:20:36 -06:00
|
|
|
* which could lock up otherwise freeable slabs.
|
|
|
|
*/
|
2014-03-30 03:02:20 -06:00
|
|
|
#define REAPTIMEOUT_AC (2*HZ)
|
|
|
|
#define REAPTIMEOUT_NODE (4*HZ)
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
#if STATS
|
|
|
|
#define STATS_INC_ACTIVE(x) ((x)->num_active++)
|
|
|
|
#define STATS_DEC_ACTIVE(x) ((x)->num_active--)
|
|
|
|
#define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
|
|
|
|
#define STATS_INC_GROWN(x) ((x)->grown++)
|
2006-06-30 02:55:45 -06:00
|
|
|
#define STATS_ADD_REAPED(x,y) ((x)->reaped += (y))
|
2006-03-22 01:08:11 -07:00
|
|
|
#define STATS_SET_HIGH(x) \
|
|
|
|
do { \
|
|
|
|
if ((x)->num_active > (x)->high_mark) \
|
|
|
|
(x)->high_mark = (x)->num_active; \
|
|
|
|
} while (0)
|
2005-04-16 16:20:36 -06:00
|
|
|
#define STATS_INC_ERR(x) ((x)->errors++)
|
|
|
|
#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
|
2005-09-09 14:03:32 -06:00
|
|
|
#define STATS_INC_NODEFREES(x) ((x)->node_frees++)
|
2006-04-10 23:52:54 -06:00
|
|
|
#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
|
2006-03-22 01:08:11 -07:00
|
|
|
#define STATS_SET_FREEABLE(x, i) \
|
|
|
|
do { \
|
|
|
|
if ((x)->max_freeable < i) \
|
|
|
|
(x)->max_freeable = i; \
|
|
|
|
} while (0)
|
2005-04-16 16:20:36 -06:00
|
|
|
#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
|
|
|
|
#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
|
|
|
|
#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
|
|
|
|
#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
|
|
|
|
#else
|
|
|
|
#define STATS_INC_ACTIVE(x) do { } while (0)
|
|
|
|
#define STATS_DEC_ACTIVE(x) do { } while (0)
|
|
|
|
#define STATS_INC_ALLOCED(x) do { } while (0)
|
|
|
|
#define STATS_INC_GROWN(x) do { } while (0)
|
2010-08-09 18:19:03 -06:00
|
|
|
#define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0)
|
2005-04-16 16:20:36 -06:00
|
|
|
#define STATS_SET_HIGH(x) do { } while (0)
|
|
|
|
#define STATS_INC_ERR(x) do { } while (0)
|
|
|
|
#define STATS_INC_NODEALLOCS(x) do { } while (0)
|
2005-09-09 14:03:32 -06:00
|
|
|
#define STATS_INC_NODEFREES(x) do { } while (0)
|
2006-04-10 23:52:54 -06:00
|
|
|
#define STATS_INC_ACOVERFLOW(x) do { } while (0)
|
2006-03-22 01:08:11 -07:00
|
|
|
#define STATS_SET_FREEABLE(x, i) do { } while (0)
|
2005-04-16 16:20:36 -06:00
|
|
|
#define STATS_INC_ALLOCHIT(x) do { } while (0)
|
|
|
|
#define STATS_INC_ALLOCMISS(x) do { } while (0)
|
|
|
|
#define STATS_INC_FREEHIT(x) do { } while (0)
|
|
|
|
#define STATS_INC_FREEMISS(x) do { } while (0)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* memory layout of objects:
|
2005-04-16 16:20:36 -06:00
|
|
|
* 0 : objp
|
2006-02-01 04:05:42 -07:00
|
|
|
* 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
|
2005-04-16 16:20:36 -06:00
|
|
|
* the end of an object is aligned with the end of the real
|
|
|
|
* allocation. Catches writes behind the end of the allocation.
|
2006-02-01 04:05:42 -07:00
|
|
|
* cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
|
2005-04-16 16:20:36 -06:00
|
|
|
* redzone word.
|
2006-02-01 04:05:42 -07:00
|
|
|
* cachep->obj_offset: The real object.
|
2012-06-13 09:24:57 -06:00
|
|
|
* cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
|
|
|
|
* cachep->size - 1* BYTES_PER_WORD: last caller address
|
2006-03-22 01:08:11 -07:00
|
|
|
* [BYTES_PER_WORD long]
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2006-02-01 04:05:50 -07:00
|
|
|
static int obj_offset(struct kmem_cache *cachep)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-02-01 04:05:42 -07:00
|
|
|
return cachep->obj_offset;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 01:22:59 -06:00
|
|
|
static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 01:22:59 -06:00
|
|
|
return (unsigned long long*) (objp + obj_offset(cachep) -
|
|
|
|
sizeof(unsigned long long));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 01:22:59 -06:00
|
|
|
static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
|
|
|
|
if (cachep->flags & SLAB_STORE_USER)
|
2012-06-13 09:24:57 -06:00
|
|
|
return (unsigned long long *)(objp + cachep->size -
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 01:22:59 -06:00
|
|
|
sizeof(unsigned long long) -
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-04 19:26:44 -06:00
|
|
|
REDZONE_ALIGN);
|
2012-06-13 09:24:57 -06:00
|
|
|
return (unsigned long long *) (objp + cachep->size -
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 01:22:59 -06:00
|
|
|
sizeof(unsigned long long));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void **dbg_userword(struct kmem_cache *cachep, void *objp)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
BUG_ON(!(cachep->flags & SLAB_STORE_USER));
|
2012-06-13 09:24:57 -06:00
|
|
|
return (void **)(objp + cachep->size - BYTES_PER_WORD);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
2006-02-01 04:05:42 -07:00
|
|
|
#define obj_offset(x) 0
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 01:22:59 -06:00
|
|
|
#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
|
|
|
|
#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
|
2005-04-16 16:20:36 -06:00
|
|
|
#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
#define OBJECT_FREE (0)
|
|
|
|
#define OBJECT_ACTIVE (1)
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_SLAB_LEAK
|
|
|
|
|
|
|
|
static void set_obj_status(struct page *page, int idx, int val)
|
|
|
|
{
|
|
|
|
int freelist_size;
|
|
|
|
char *status;
|
|
|
|
struct kmem_cache *cachep = page->slab_cache;
|
|
|
|
|
|
|
|
freelist_size = cachep->num * sizeof(freelist_idx_t);
|
|
|
|
status = (char *)page->freelist + freelist_size;
|
|
|
|
status[idx] = val;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned int get_obj_status(struct page *page, int idx)
|
|
|
|
{
|
|
|
|
int freelist_size;
|
|
|
|
char *status;
|
|
|
|
struct kmem_cache *cachep = page->slab_cache;
|
|
|
|
|
|
|
|
freelist_size = cachep->num * sizeof(freelist_idx_t);
|
|
|
|
status = (char *)page->freelist + freelist_size;
|
|
|
|
|
|
|
|
return status[idx];
|
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
static inline void set_obj_status(struct page *page, int idx, int val) {}
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
2011-10-18 23:09:28 -06:00
|
|
|
* Do not go above this order unless 0 objects fit into the slab or
|
|
|
|
* overridden on the command line.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2011-10-18 23:09:24 -06:00
|
|
|
#define SLAB_MAX_ORDER_HI 1
|
|
|
|
#define SLAB_MAX_ORDER_LO 0
|
|
|
|
static int slab_max_order = SLAB_MAX_ORDER_LO;
|
2011-10-18 23:09:28 -06:00
|
|
|
static bool slab_max_order_set __initdata;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-02-01 04:05:49 -07:00
|
|
|
static inline struct kmem_cache *virt_to_cache(const void *obj)
|
|
|
|
{
|
2007-05-06 15:49:41 -06:00
|
|
|
struct page *page = virt_to_head_page(obj);
|
2012-06-13 09:24:56 -06:00
|
|
|
return page->slab_cache;
|
2006-02-01 04:05:49 -07:00
|
|
|
}
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
static inline void *index_to_obj(struct kmem_cache *cache, struct page *page,
|
2006-03-22 01:08:10 -07:00
|
|
|
unsigned int idx)
|
|
|
|
{
|
2013-10-23 19:07:49 -06:00
|
|
|
return page->s_mem + cache->size * idx;
|
2006-03-22 01:08:10 -07:00
|
|
|
}
|
|
|
|
|
2006-12-13 01:34:27 -07:00
|
|
|
/*
|
2012-06-13 09:24:57 -06:00
|
|
|
* We want to avoid an expensive divide : (offset / cache->size)
|
|
|
|
* Using the fact that size is a constant for a particular cache,
|
|
|
|
* we can replace (offset / cache->size) by
|
2006-12-13 01:34:27 -07:00
|
|
|
* reciprocal_divide(offset, cache->reciprocal_buffer_size)
|
|
|
|
*/
|
|
|
|
static inline unsigned int obj_to_index(const struct kmem_cache *cache,
|
2013-10-23 19:07:49 -06:00
|
|
|
const struct page *page, void *obj)
|
2006-03-22 01:08:10 -07:00
|
|
|
{
|
2013-10-23 19:07:49 -06:00
|
|
|
u32 offset = (obj - page->s_mem);
|
2006-12-13 01:34:27 -07:00
|
|
|
return reciprocal_divide(offset, cache->reciprocal_buffer_size);
|
2006-03-22 01:08:10 -07:00
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* internal cache of cache description objs */
|
2012-09-04 18:20:33 -06:00
|
|
|
static struct kmem_cache kmem_cache_boot = {
|
2006-01-08 02:00:37 -07:00
|
|
|
.batchcount = 1,
|
|
|
|
.limit = BOOT_CPUCACHE_ENTRIES,
|
|
|
|
.shared = 1,
|
2012-06-13 09:24:57 -06:00
|
|
|
.size = sizeof(struct kmem_cache),
|
2006-01-08 02:00:37 -07:00
|
|
|
.name = "kmem_cache",
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
2014-08-08 15:19:15 -06:00
|
|
|
#define BAD_ALIEN_MAGIC 0x01020304ul
|
|
|
|
|
2009-10-29 07:34:13 -06:00
|
|
|
static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2014-10-09 16:26:27 -06:00
|
|
|
return this_cpu_ptr(cachep->cpu_cache);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
static size_t calculate_freelist_size(int nr_objs, size_t align)
|
|
|
|
{
|
|
|
|
size_t freelist_size;
|
|
|
|
|
|
|
|
freelist_size = nr_objs * sizeof(freelist_idx_t);
|
|
|
|
if (IS_ENABLED(CONFIG_DEBUG_SLAB_LEAK))
|
|
|
|
freelist_size += nr_objs * sizeof(char);
|
|
|
|
|
|
|
|
if (align)
|
|
|
|
freelist_size = ALIGN(freelist_size, align);
|
|
|
|
|
|
|
|
return freelist_size;
|
|
|
|
}
|
|
|
|
|
2013-12-02 01:49:39 -07:00
|
|
|
static int calculate_nr_objs(size_t slab_size, size_t buffer_size,
|
|
|
|
size_t idx_size, size_t align)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2013-12-02 01:49:39 -07:00
|
|
|
int nr_objs;
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
size_t remained_size;
|
2013-12-02 01:49:39 -07:00
|
|
|
size_t freelist_size;
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
int extra_space = 0;
|
2013-12-02 01:49:39 -07:00
|
|
|
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
if (IS_ENABLED(CONFIG_DEBUG_SLAB_LEAK))
|
|
|
|
extra_space = sizeof(char);
|
2013-12-02 01:49:39 -07:00
|
|
|
/*
|
|
|
|
* Ignore padding for the initial guess. The padding
|
|
|
|
* is at most @align-1 bytes, and @buffer_size is at
|
|
|
|
* least @align. In the worst case, this result will
|
|
|
|
* be one greater than the number of objects that fit
|
|
|
|
* into the memory allocation when taking the padding
|
|
|
|
* into account.
|
|
|
|
*/
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
nr_objs = slab_size / (buffer_size + idx_size + extra_space);
|
2013-12-02 01:49:39 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This calculated number will be either the right
|
|
|
|
* amount, or one greater than what we want.
|
|
|
|
*/
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
remained_size = slab_size - nr_objs * buffer_size;
|
|
|
|
freelist_size = calculate_freelist_size(nr_objs, align);
|
|
|
|
if (remained_size < freelist_size)
|
2013-12-02 01:49:39 -07:00
|
|
|
nr_objs--;
|
|
|
|
|
|
|
|
return nr_objs;
|
2006-02-01 04:05:45 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* Calculate the number of objects and left-over bytes for a given buffer size.
|
|
|
|
*/
|
2006-02-01 04:05:45 -07:00
|
|
|
static void cache_estimate(unsigned long gfporder, size_t buffer_size,
|
|
|
|
size_t align, int flags, size_t *left_over,
|
|
|
|
unsigned int *num)
|
|
|
|
{
|
|
|
|
int nr_objs;
|
|
|
|
size_t mgmt_size;
|
|
|
|
size_t slab_size = PAGE_SIZE << gfporder;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-02-01 04:05:45 -07:00
|
|
|
/*
|
|
|
|
* The slab management structure can be either off the slab or
|
|
|
|
* on it. For the latter case, the memory allocated for a
|
|
|
|
* slab is used for:
|
|
|
|
*
|
2013-10-23 19:07:46 -06:00
|
|
|
* - One unsigned int for each object
|
2006-02-01 04:05:45 -07:00
|
|
|
* - Padding to respect alignment of @align
|
|
|
|
* - @buffer_size bytes for each object
|
|
|
|
*
|
|
|
|
* If the slab management structure is off the slab, then the
|
|
|
|
* alignment will already be calculated into the size. Because
|
|
|
|
* the slabs are all pages aligned, the objects will be at the
|
|
|
|
* correct alignment when allocated.
|
|
|
|
*/
|
|
|
|
if (flags & CFLGS_OFF_SLAB) {
|
|
|
|
mgmt_size = 0;
|
|
|
|
nr_objs = slab_size / buffer_size;
|
|
|
|
|
|
|
|
} else {
|
2013-12-02 01:49:39 -07:00
|
|
|
nr_objs = calculate_nr_objs(slab_size, buffer_size,
|
slab: introduce byte sized index for the freelist of a slab
Currently, the freelist of a slab consist of unsigned int sized indexes.
Since most of slabs have less number of objects than 256, large sized
indexes is needless. For example, consider the minimum kmalloc slab. It's
object size is 32 byte and it would consist of one page, so 256 indexes
through byte sized index are enough to contain all possible indexes.
There can be some slabs whose object size is 8 byte. We cannot handle
this case with byte sized index, so we need to restrict minimum
object size. Since these slabs are not major, wasted memory from these
slabs would be negligible.
Some architectures' page size isn't 4096 bytes and rather larger than
4096 bytes (One example is 64KB page size on PPC or IA64) so that
byte sized index doesn't fit to them. In this case, we will use
two bytes sized index.
Below is some number for this patch.
* Before *
kmalloc-512 525 640 512 8 1 : tunables 54 27 0 : slabdata 80 80 0
kmalloc-256 210 210 256 15 1 : tunables 120 60 0 : slabdata 14 14 0
kmalloc-192 1016 1040 192 20 1 : tunables 120 60 0 : slabdata 52 52 0
kmalloc-96 560 620 128 31 1 : tunables 120 60 0 : slabdata 20 20 0
kmalloc-64 2148 2280 64 60 1 : tunables 120 60 0 : slabdata 38 38 0
kmalloc-128 647 682 128 31 1 : tunables 120 60 0 : slabdata 22 22 0
kmalloc-32 11360 11413 32 113 1 : tunables 120 60 0 : slabdata 101 101 0
kmem_cache 197 200 192 20 1 : tunables 120 60 0 : slabdata 10 10 0
* After *
kmalloc-512 521 648 512 8 1 : tunables 54 27 0 : slabdata 81 81 0
kmalloc-256 208 208 256 16 1 : tunables 120 60 0 : slabdata 13 13 0
kmalloc-192 1029 1029 192 21 1 : tunables 120 60 0 : slabdata 49 49 0
kmalloc-96 529 589 128 31 1 : tunables 120 60 0 : slabdata 19 19 0
kmalloc-64 2142 2142 64 63 1 : tunables 120 60 0 : slabdata 34 34 0
kmalloc-128 660 682 128 31 1 : tunables 120 60 0 : slabdata 22 22 0
kmalloc-32 11716 11780 32 124 1 : tunables 120 60 0 : slabdata 95 95 0
kmem_cache 197 210 192 21 1 : tunables 120 60 0 : slabdata 10 10 0
kmem_caches consisting of objects less than or equal to 256 byte have
one or more objects than before. In the case of kmalloc-32, we have 11 more
objects, so 352 bytes (11 * 32) are saved and this is roughly 9% saving of
memory. Of couse, this percentage decreases as the number of objects
in a slab decreases.
Here are the performance results on my 4 cpus machine.
* Before *
Performance counter stats for 'perf bench sched messaging -g 50 -l 1000' (10 runs):
229,945,138 cache-misses ( +- 0.23% )
11.627897174 seconds time elapsed ( +- 0.14% )
* After *
Performance counter stats for 'perf bench sched messaging -g 50 -l 1000' (10 runs):
218,640,472 cache-misses ( +- 0.42% )
11.504999837 seconds time elapsed ( +- 0.21% )
cache-misses are reduced by this patchset, roughly 5%.
And elapsed times are improved by 1%.
Acked-by: Christoph Lameter <cl@linux.com>
Acked-by: David Rientjes <rientjes@google.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>
2013-12-02 01:49:42 -07:00
|
|
|
sizeof(freelist_idx_t), align);
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
mgmt_size = calculate_freelist_size(nr_objs, align);
|
2006-02-01 04:05:45 -07:00
|
|
|
}
|
|
|
|
*num = nr_objs;
|
|
|
|
*left_over = slab_size - nr_objs*buffer_size - mgmt_size;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2012-09-11 13:49:38 -06:00
|
|
|
#if DEBUG
|
2008-04-30 01:55:07 -06:00
|
|
|
#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
static void __slab_error(const char *function, struct kmem_cache *cachep,
|
|
|
|
char *msg)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
|
2006-01-08 02:00:37 -07:00
|
|
|
function, cachep->name, msg);
|
2005-04-16 16:20:36 -06:00
|
|
|
dump_stack();
|
2013-01-20 23:47:39 -07:00
|
|
|
add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2012-09-11 13:49:38 -06:00
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-12-06 21:32:16 -07:00
|
|
|
/*
|
|
|
|
* By default on NUMA we use alien caches to stage the freeing of
|
|
|
|
* objects allocated from other nodes. This causes massive memory
|
|
|
|
* inefficiencies when using fake NUMA setup to split memory into a
|
|
|
|
* large number of small nodes, so it can be disabled on the command
|
|
|
|
* line
|
|
|
|
*/
|
|
|
|
|
|
|
|
static int use_alien_caches __read_mostly = 1;
|
|
|
|
static int __init noaliencache_setup(char *s)
|
|
|
|
{
|
|
|
|
use_alien_caches = 0;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
__setup("noaliencache", noaliencache_setup);
|
|
|
|
|
2011-10-18 23:09:28 -06:00
|
|
|
static int __init slab_max_order_setup(char *str)
|
|
|
|
{
|
|
|
|
get_option(&str, &slab_max_order);
|
|
|
|
slab_max_order = slab_max_order < 0 ? 0 :
|
|
|
|
min(slab_max_order, MAX_ORDER - 1);
|
|
|
|
slab_max_order_set = true;
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
__setup("slab_max_order=", slab_max_order_setup);
|
|
|
|
|
2006-03-09 18:33:54 -07:00
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
/*
|
|
|
|
* Special reaping functions for NUMA systems called from cache_reap().
|
|
|
|
* These take care of doing round robin flushing of alien caches (containing
|
|
|
|
* objects freed on different nodes from which they were allocated) and the
|
|
|
|
* flushing of remote pcps by calling drain_node_pages.
|
|
|
|
*/
|
2009-10-29 07:34:13 -06:00
|
|
|
static DEFINE_PER_CPU(unsigned long, slab_reap_node);
|
2006-03-09 18:33:54 -07:00
|
|
|
|
|
|
|
static void init_reap_node(int cpu)
|
|
|
|
{
|
|
|
|
int node;
|
|
|
|
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
node = next_node(cpu_to_mem(cpu), node_online_map);
|
2006-03-09 18:33:54 -07:00
|
|
|
if (node == MAX_NUMNODES)
|
2006-03-22 01:09:11 -07:00
|
|
|
node = first_node(node_online_map);
|
2006-03-09 18:33:54 -07:00
|
|
|
|
2009-10-29 07:34:13 -06:00
|
|
|
per_cpu(slab_reap_node, cpu) = node;
|
2006-03-09 18:33:54 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static void next_reap_node(void)
|
|
|
|
{
|
2010-12-08 08:22:55 -07:00
|
|
|
int node = __this_cpu_read(slab_reap_node);
|
2006-03-09 18:33:54 -07:00
|
|
|
|
|
|
|
node = next_node(node, node_online_map);
|
|
|
|
if (unlikely(node >= MAX_NUMNODES))
|
|
|
|
node = first_node(node_online_map);
|
2010-12-08 08:22:55 -07:00
|
|
|
__this_cpu_write(slab_reap_node, node);
|
2006-03-09 18:33:54 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
#define init_reap_node(cpu) do { } while (0)
|
|
|
|
#define next_reap_node(void) do { } while (0)
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
|
|
|
|
* via the workqueue/eventd.
|
|
|
|
* Add the CPU number into the expiration time to minimize the possibility of
|
|
|
|
* the CPUs getting into lockstep and contending for the global cache chain
|
|
|
|
* lock.
|
|
|
|
*/
|
2013-06-19 12:53:51 -06:00
|
|
|
static void start_cpu_timer(int cpu)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2009-10-29 07:34:13 -06:00
|
|
|
struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* When this gets called from do_initcalls via cpucache_init(),
|
|
|
|
* init_workqueues() has already run, so keventd will be setup
|
|
|
|
* at that time.
|
|
|
|
*/
|
2006-11-22 07:54:01 -07:00
|
|
|
if (keventd_up() && reap_work->work.func == NULL) {
|
2006-03-09 18:33:54 -07:00
|
|
|
init_reap_node(cpu);
|
2012-08-21 14:18:23 -06:00
|
|
|
INIT_DEFERRABLE_WORK(reap_work, cache_reap);
|
2006-12-10 03:21:28 -07:00
|
|
|
schedule_delayed_work_on(cpu, reap_work,
|
|
|
|
__round_jiffies_relative(HZ, cpu));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-08-06 17:04:27 -06:00
|
|
|
static void init_arraycache(struct array_cache *ac, int limit, int batch)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2009-06-11 06:22:40 -06:00
|
|
|
/*
|
|
|
|
* The array_cache structures contain pointers to free object.
|
2011-03-30 19:57:33 -06:00
|
|
|
* However, when such objects are allocated or transferred to another
|
2009-06-11 06:22:40 -06:00
|
|
|
* cache the pointers are not cleared and they could be counted as
|
|
|
|
* valid references during a kmemleak scan. Therefore, kmemleak must
|
|
|
|
* not scan such objects.
|
|
|
|
*/
|
2014-08-06 17:04:27 -06:00
|
|
|
kmemleak_no_scan(ac);
|
|
|
|
if (ac) {
|
|
|
|
ac->avail = 0;
|
|
|
|
ac->limit = limit;
|
|
|
|
ac->batchcount = batch;
|
|
|
|
ac->touched = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2014-08-06 17:04:27 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct array_cache *alloc_arraycache(int node, int entries,
|
|
|
|
int batchcount, gfp_t gfp)
|
|
|
|
{
|
2014-08-06 17:04:40 -06:00
|
|
|
size_t memsize = sizeof(void *) * entries + sizeof(struct array_cache);
|
2014-08-06 17:04:27 -06:00
|
|
|
struct array_cache *ac = NULL;
|
|
|
|
|
|
|
|
ac = kmalloc_node(memsize, gfp, node);
|
|
|
|
init_arraycache(ac, entries, batchcount);
|
|
|
|
return ac;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
static inline bool is_slab_pfmemalloc(struct page *page)
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
{
|
|
|
|
return PageSlabPfmemalloc(page);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Clears pfmemalloc_active if no slabs have pfmalloc set */
|
|
|
|
static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
|
|
|
|
struct array_cache *ac)
|
|
|
|
{
|
2014-08-06 17:04:11 -06:00
|
|
|
struct kmem_cache_node *n = get_node(cachep, numa_mem_id());
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page;
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
if (!pfmemalloc_active)
|
|
|
|
return;
|
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock_irqsave(&n->list_lock, flags);
|
2013-10-23 19:07:49 -06:00
|
|
|
list_for_each_entry(page, &n->slabs_full, lru)
|
|
|
|
if (is_slab_pfmemalloc(page))
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
goto out;
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
list_for_each_entry(page, &n->slabs_partial, lru)
|
|
|
|
if (is_slab_pfmemalloc(page))
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
goto out;
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
list_for_each_entry(page, &n->slabs_free, lru)
|
|
|
|
if (is_slab_pfmemalloc(page))
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
goto out;
|
|
|
|
|
|
|
|
pfmemalloc_active = false;
|
|
|
|
out:
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock_irqrestore(&n->list_lock, flags);
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
}
|
|
|
|
|
2012-07-31 17:44:30 -06:00
|
|
|
static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac,
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
gfp_t flags, bool force_refill)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
void *objp = ac->entry[--ac->avail];
|
|
|
|
|
|
|
|
/* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
|
|
|
|
if (unlikely(is_obj_pfmemalloc(objp))) {
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
|
|
|
|
if (gfp_pfmemalloc_allowed(flags)) {
|
|
|
|
clear_obj_pfmemalloc(&objp);
|
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* The caller cannot use PFMEMALLOC objects, find another one */
|
2012-09-17 15:09:06 -06:00
|
|
|
for (i = 0; i < ac->avail; i++) {
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
/* If a !PFMEMALLOC object is found, swap them */
|
|
|
|
if (!is_obj_pfmemalloc(ac->entry[i])) {
|
|
|
|
objp = ac->entry[i];
|
|
|
|
ac->entry[i] = ac->entry[ac->avail];
|
|
|
|
ac->entry[ac->avail] = objp;
|
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there are empty slabs on the slabs_free list and we are
|
|
|
|
* being forced to refill the cache, mark this one !pfmemalloc.
|
|
|
|
*/
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, numa_mem_id());
|
2013-01-10 12:14:19 -07:00
|
|
|
if (!list_empty(&n->slabs_free) && force_refill) {
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page = virt_to_head_page(objp);
|
2013-10-23 19:07:50 -06:00
|
|
|
ClearPageSlabPfmemalloc(page);
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
clear_obj_pfmemalloc(&objp);
|
|
|
|
recheck_pfmemalloc_active(cachep, ac);
|
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* No !PFMEMALLOC objects available */
|
|
|
|
ac->avail++;
|
|
|
|
objp = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
2012-07-31 17:44:30 -06:00
|
|
|
static inline void *ac_get_obj(struct kmem_cache *cachep,
|
|
|
|
struct array_cache *ac, gfp_t flags, bool force_refill)
|
|
|
|
{
|
|
|
|
void *objp;
|
|
|
|
|
|
|
|
if (unlikely(sk_memalloc_socks()))
|
|
|
|
objp = __ac_get_obj(cachep, ac, flags, force_refill);
|
|
|
|
else
|
|
|
|
objp = ac->entry[--ac->avail];
|
|
|
|
|
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
2014-10-09 16:26:06 -06:00
|
|
|
static noinline void *__ac_put_obj(struct kmem_cache *cachep,
|
|
|
|
struct array_cache *ac, void *objp)
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
{
|
|
|
|
if (unlikely(pfmemalloc_active)) {
|
|
|
|
/* Some pfmemalloc slabs exist, check if this is one */
|
2012-09-17 15:09:03 -06:00
|
|
|
struct page *page = virt_to_head_page(objp);
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
if (PageSlabPfmemalloc(page))
|
|
|
|
set_obj_pfmemalloc(&objp);
|
|
|
|
}
|
|
|
|
|
2012-07-31 17:44:30 -06:00
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac,
|
|
|
|
void *objp)
|
|
|
|
{
|
|
|
|
if (unlikely(sk_memalloc_socks()))
|
|
|
|
objp = __ac_put_obj(cachep, ac, objp);
|
|
|
|
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
ac->entry[ac->avail++] = objp;
|
|
|
|
}
|
|
|
|
|
2006-03-25 04:06:44 -07:00
|
|
|
/*
|
|
|
|
* Transfer objects in one arraycache to another.
|
|
|
|
* Locking must be handled by the caller.
|
|
|
|
*
|
|
|
|
* Return the number of entries transferred.
|
|
|
|
*/
|
|
|
|
static int transfer_objects(struct array_cache *to,
|
|
|
|
struct array_cache *from, unsigned int max)
|
|
|
|
{
|
|
|
|
/* Figure out how many entries to transfer */
|
2010-10-26 15:22:23 -06:00
|
|
|
int nr = min3(from->avail, max, to->limit - to->avail);
|
2006-03-25 04:06:44 -07:00
|
|
|
|
|
|
|
if (!nr)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
memcpy(to->entry + to->avail, from->entry + from->avail -nr,
|
|
|
|
sizeof(void *) *nr);
|
|
|
|
|
|
|
|
from->avail -= nr;
|
|
|
|
to->avail += nr;
|
|
|
|
return nr;
|
|
|
|
}
|
|
|
|
|
2006-09-27 02:50:08 -06:00
|
|
|
#ifndef CONFIG_NUMA
|
|
|
|
|
|
|
|
#define drain_alien_cache(cachep, alien) do { } while (0)
|
2013-01-10 12:14:19 -07:00
|
|
|
#define reap_alien(cachep, n) do { } while (0)
|
2006-09-27 02:50:08 -06:00
|
|
|
|
2014-08-06 17:04:29 -06:00
|
|
|
static inline struct alien_cache **alloc_alien_cache(int node,
|
|
|
|
int limit, gfp_t gfp)
|
2006-09-27 02:50:08 -06:00
|
|
|
{
|
2014-08-08 15:19:15 -06:00
|
|
|
return (struct alien_cache **)BAD_ALIEN_MAGIC;
|
2006-09-27 02:50:08 -06:00
|
|
|
}
|
|
|
|
|
2014-08-06 17:04:29 -06:00
|
|
|
static inline void free_alien_cache(struct alien_cache **ac_ptr)
|
2006-09-27 02:50:08 -06:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void *alternate_node_alloc(struct kmem_cache *cachep,
|
|
|
|
gfp_t flags)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2006-12-06 21:32:30 -07:00
|
|
|
static inline void *____cache_alloc_node(struct kmem_cache *cachep,
|
2006-09-27 02:50:08 -06:00
|
|
|
gfp_t flags, int nodeid)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
#else /* CONFIG_NUMA */
|
|
|
|
|
2006-12-06 21:32:30 -07:00
|
|
|
static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);
|
2006-03-24 04:16:08 -07:00
|
|
|
static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
|
2006-01-18 18:42:36 -07:00
|
|
|
|
2014-08-06 17:04:29 -06:00
|
|
|
static struct alien_cache *__alloc_alien_cache(int node, int entries,
|
|
|
|
int batch, gfp_t gfp)
|
|
|
|
{
|
2014-08-06 17:04:40 -06:00
|
|
|
size_t memsize = sizeof(void *) * entries + sizeof(struct alien_cache);
|
2014-08-06 17:04:29 -06:00
|
|
|
struct alien_cache *alc = NULL;
|
|
|
|
|
|
|
|
alc = kmalloc_node(memsize, gfp, node);
|
|
|
|
init_arraycache(&alc->ac, entries, batch);
|
2014-08-06 17:04:31 -06:00
|
|
|
spin_lock_init(&alc->lock);
|
2014-08-06 17:04:29 -06:00
|
|
|
return alc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct alien_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
2014-08-06 17:04:29 -06:00
|
|
|
struct alien_cache **alc_ptr;
|
2014-08-06 17:04:40 -06:00
|
|
|
size_t memsize = sizeof(void *) * nr_node_ids;
|
2005-09-09 14:03:32 -06:00
|
|
|
int i;
|
|
|
|
|
|
|
|
if (limit > 1)
|
|
|
|
limit = 12;
|
2014-08-06 17:04:29 -06:00
|
|
|
alc_ptr = kzalloc_node(memsize, gfp, node);
|
|
|
|
if (!alc_ptr)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
for_each_node(i) {
|
|
|
|
if (i == node || !node_online(i))
|
|
|
|
continue;
|
|
|
|
alc_ptr[i] = __alloc_alien_cache(node, limit, 0xbaadf00d, gfp);
|
|
|
|
if (!alc_ptr[i]) {
|
|
|
|
for (i--; i >= 0; i--)
|
|
|
|
kfree(alc_ptr[i]);
|
|
|
|
kfree(alc_ptr);
|
|
|
|
return NULL;
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
|
|
|
}
|
2014-08-06 17:04:29 -06:00
|
|
|
return alc_ptr;
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
|
|
|
|
2014-08-06 17:04:29 -06:00
|
|
|
static void free_alien_cache(struct alien_cache **alc_ptr)
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2014-08-06 17:04:29 -06:00
|
|
|
if (!alc_ptr)
|
2005-09-09 14:03:32 -06:00
|
|
|
return;
|
|
|
|
for_each_node(i)
|
2014-08-06 17:04:29 -06:00
|
|
|
kfree(alc_ptr[i]);
|
|
|
|
kfree(alc_ptr);
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void __drain_alien_cache(struct kmem_cache *cachep,
|
2014-08-06 17:04:33 -06:00
|
|
|
struct array_cache *ac, int node,
|
|
|
|
struct list_head *list)
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
2014-08-06 17:04:11 -06:00
|
|
|
struct kmem_cache_node *n = get_node(cachep, node);
|
2005-09-09 14:03:32 -06:00
|
|
|
|
|
|
|
if (ac->avail) {
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock(&n->list_lock);
|
2006-03-25 04:06:45 -07:00
|
|
|
/*
|
|
|
|
* Stuff objects into the remote nodes shared array first.
|
|
|
|
* That way we could avoid the overhead of putting the objects
|
|
|
|
* into the free lists and getting them back later.
|
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
if (n->shared)
|
|
|
|
transfer_objects(n->shared, ac, ac->limit);
|
2006-03-25 04:06:45 -07:00
|
|
|
|
2014-08-06 17:04:33 -06:00
|
|
|
free_block(cachep, ac->entry, ac->avail, node, list);
|
2005-09-09 14:03:32 -06:00
|
|
|
ac->avail = 0;
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock(&n->list_lock);
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-03-09 18:33:54 -07:00
|
|
|
/*
|
|
|
|
* Called from cache_reap() to regularly drain alien caches round robin.
|
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n)
|
2006-03-09 18:33:54 -07:00
|
|
|
{
|
2010-12-08 08:22:55 -07:00
|
|
|
int node = __this_cpu_read(slab_reap_node);
|
2006-03-09 18:33:54 -07:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
if (n->alien) {
|
2014-08-06 17:04:29 -06:00
|
|
|
struct alien_cache *alc = n->alien[node];
|
|
|
|
struct array_cache *ac;
|
|
|
|
|
|
|
|
if (alc) {
|
|
|
|
ac = &alc->ac;
|
2014-08-06 17:04:31 -06:00
|
|
|
if (ac->avail && spin_trylock_irq(&alc->lock)) {
|
2014-08-06 17:04:33 -06:00
|
|
|
LIST_HEAD(list);
|
|
|
|
|
|
|
|
__drain_alien_cache(cachep, ac, node, &list);
|
2014-08-06 17:04:31 -06:00
|
|
|
spin_unlock_irq(&alc->lock);
|
2014-08-06 17:04:33 -06:00
|
|
|
slabs_destroy(cachep, &list);
|
2014-08-06 17:04:29 -06:00
|
|
|
}
|
2006-03-09 18:33:54 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
static void drain_alien_cache(struct kmem_cache *cachep,
|
2014-08-06 17:04:29 -06:00
|
|
|
struct alien_cache **alien)
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
2006-01-08 02:00:37 -07:00
|
|
|
int i = 0;
|
2014-08-06 17:04:29 -06:00
|
|
|
struct alien_cache *alc;
|
2005-09-09 14:03:32 -06:00
|
|
|
struct array_cache *ac;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
for_each_online_node(i) {
|
2014-08-06 17:04:29 -06:00
|
|
|
alc = alien[i];
|
|
|
|
if (alc) {
|
2014-08-06 17:04:33 -06:00
|
|
|
LIST_HEAD(list);
|
|
|
|
|
2014-08-06 17:04:29 -06:00
|
|
|
ac = &alc->ac;
|
2014-08-06 17:04:31 -06:00
|
|
|
spin_lock_irqsave(&alc->lock, flags);
|
2014-08-06 17:04:33 -06:00
|
|
|
__drain_alien_cache(cachep, ac, i, &list);
|
2014-08-06 17:04:31 -06:00
|
|
|
spin_unlock_irqrestore(&alc->lock, flags);
|
2014-08-06 17:04:33 -06:00
|
|
|
slabs_destroy(cachep, &list);
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2006-06-23 03:03:05 -06:00
|
|
|
|
2014-10-09 16:26:09 -06:00
|
|
|
static int __cache_free_alien(struct kmem_cache *cachep, void *objp,
|
|
|
|
int node, int page_node)
|
2006-06-23 03:03:05 -06:00
|
|
|
{
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2014-08-06 17:04:29 -06:00
|
|
|
struct alien_cache *alien = NULL;
|
|
|
|
struct array_cache *ac;
|
2014-08-06 17:04:25 -06:00
|
|
|
LIST_HEAD(list);
|
2006-10-06 01:43:52 -06:00
|
|
|
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
2006-06-23 03:03:05 -06:00
|
|
|
STATS_INC_NODEFREES(cachep);
|
2014-10-09 16:26:09 -06:00
|
|
|
if (n->alien && n->alien[page_node]) {
|
|
|
|
alien = n->alien[page_node];
|
2014-08-06 17:04:29 -06:00
|
|
|
ac = &alien->ac;
|
2014-08-06 17:04:31 -06:00
|
|
|
spin_lock(&alien->lock);
|
2014-08-06 17:04:29 -06:00
|
|
|
if (unlikely(ac->avail == ac->limit)) {
|
2006-06-23 03:03:05 -06:00
|
|
|
STATS_INC_ACOVERFLOW(cachep);
|
2014-10-09 16:26:09 -06:00
|
|
|
__drain_alien_cache(cachep, ac, page_node, &list);
|
2006-06-23 03:03:05 -06:00
|
|
|
}
|
2014-08-06 17:04:29 -06:00
|
|
|
ac_put_obj(cachep, ac, objp);
|
2014-08-06 17:04:31 -06:00
|
|
|
spin_unlock(&alien->lock);
|
2014-08-06 17:04:33 -06:00
|
|
|
slabs_destroy(cachep, &list);
|
2006-06-23 03:03:05 -06:00
|
|
|
} else {
|
2014-10-09 16:26:09 -06:00
|
|
|
n = get_node(cachep, page_node);
|
2014-08-06 17:04:11 -06:00
|
|
|
spin_lock(&n->list_lock);
|
2014-10-09 16:26:09 -06:00
|
|
|
free_block(cachep, &objp, 1, page_node, &list);
|
2014-08-06 17:04:11 -06:00
|
|
|
spin_unlock(&n->list_lock);
|
2014-08-06 17:04:25 -06:00
|
|
|
slabs_destroy(cachep, &list);
|
2006-06-23 03:03:05 -06:00
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
2014-10-09 16:26:09 -06:00
|
|
|
|
|
|
|
static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
|
|
|
|
{
|
|
|
|
int page_node = page_to_nid(virt_to_page(objp));
|
|
|
|
int node = numa_mem_id();
|
|
|
|
/*
|
|
|
|
* Make sure we are not freeing a object from another node to the array
|
|
|
|
* cache on this cpu.
|
|
|
|
*/
|
|
|
|
if (likely(node == page_node))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return __cache_free_alien(cachep, objp, node, page_node);
|
|
|
|
}
|
2005-09-09 14:03:32 -06:00
|
|
|
#endif
|
|
|
|
|
2010-03-27 20:40:47 -06:00
|
|
|
/*
|
2013-01-10 12:14:19 -07:00
|
|
|
* Allocates and initializes node for a node on each slab cache, used for
|
2013-01-10 12:14:19 -07:00
|
|
|
* either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node
|
2010-03-27 20:40:47 -06:00
|
|
|
* will be allocated off-node since memory is not yet online for the new node.
|
2013-01-10 12:14:19 -07:00
|
|
|
* When hotplugging memory or a cpu, existing node are not replaced if
|
2010-03-27 20:40:47 -06:00
|
|
|
* already in use.
|
|
|
|
*
|
2012-07-06 14:25:12 -06:00
|
|
|
* Must hold slab_mutex.
|
2010-03-27 20:40:47 -06:00
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
static int init_cache_node_node(int node)
|
2010-03-27 20:40:47 -06:00
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2014-08-06 17:04:40 -06:00
|
|
|
const size_t memsize = sizeof(struct kmem_cache_node);
|
2010-03-27 20:40:47 -06:00
|
|
|
|
2012-07-06 14:25:12 -06:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2010-03-27 20:40:47 -06:00
|
|
|
/*
|
2014-03-30 03:02:20 -06:00
|
|
|
* Set up the kmem_cache_node for cpu before we can
|
2010-03-27 20:40:47 -06:00
|
|
|
* begin anything. Make sure some other cpu on this
|
|
|
|
* node has not already allocated this
|
|
|
|
*/
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
|
|
|
if (!n) {
|
2013-01-10 12:14:19 -07:00
|
|
|
n = kmalloc_node(memsize, GFP_KERNEL, node);
|
|
|
|
if (!n)
|
2010-03-27 20:40:47 -06:00
|
|
|
return -ENOMEM;
|
2013-01-10 12:14:19 -07:00
|
|
|
kmem_cache_node_init(n);
|
2014-03-30 03:02:20 -06:00
|
|
|
n->next_reap = jiffies + REAPTIMEOUT_NODE +
|
|
|
|
((unsigned long)cachep) % REAPTIMEOUT_NODE;
|
2010-03-27 20:40:47 -06:00
|
|
|
|
|
|
|
/*
|
2014-03-30 03:02:20 -06:00
|
|
|
* The kmem_cache_nodes don't come and go as CPUs
|
|
|
|
* come and go. slab_mutex is sufficient
|
2010-03-27 20:40:47 -06:00
|
|
|
* protection here.
|
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
cachep->node[node] = n;
|
2010-03-27 20:40:47 -06:00
|
|
|
}
|
|
|
|
|
2014-08-06 17:04:11 -06:00
|
|
|
spin_lock_irq(&n->list_lock);
|
|
|
|
n->free_limit =
|
2010-03-27 20:40:47 -06:00
|
|
|
(1 + nr_cpus_node(node)) *
|
|
|
|
cachep->batchcount + cachep->num;
|
2014-08-06 17:04:11 -06:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2010-03-27 20:40:47 -06:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-07-03 18:33:22 -06:00
|
|
|
static inline int slabs_tofree(struct kmem_cache *cachep,
|
|
|
|
struct kmem_cache_node *n)
|
|
|
|
{
|
|
|
|
return (n->free_objects + cachep->num - 1) / cachep->num;
|
|
|
|
}
|
|
|
|
|
2013-06-19 12:53:51 -06:00
|
|
|
static void cpuup_canceled(long cpu)
|
2007-10-18 04:05:09 -06:00
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n = NULL;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
int node = cpu_to_mem(cpu);
|
2009-03-12 22:19:46 -06:00
|
|
|
const struct cpumask *mask = cpumask_of_node(node);
|
2007-10-18 04:05:09 -06:00
|
|
|
|
2012-07-06 14:25:12 -06:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2007-10-18 04:05:09 -06:00
|
|
|
struct array_cache *nc;
|
|
|
|
struct array_cache *shared;
|
2014-08-06 17:04:29 -06:00
|
|
|
struct alien_cache **alien;
|
2014-08-06 17:04:25 -06:00
|
|
|
LIST_HEAD(list);
|
2007-10-18 04:05:09 -06:00
|
|
|
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
2013-01-10 12:14:19 -07:00
|
|
|
if (!n)
|
2014-10-09 16:26:27 -06:00
|
|
|
continue;
|
2007-10-18 04:05:09 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock_irq(&n->list_lock);
|
2007-10-18 04:05:09 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
/* Free limit for this kmem_cache_node */
|
|
|
|
n->free_limit -= cachep->batchcount;
|
2014-10-09 16:26:27 -06:00
|
|
|
|
|
|
|
/* cpu is dead; no one can alloc from it. */
|
|
|
|
nc = per_cpu_ptr(cachep->cpu_cache, cpu);
|
|
|
|
if (nc) {
|
2014-08-06 17:04:25 -06:00
|
|
|
free_block(cachep, nc->entry, nc->avail, node, &list);
|
2014-10-09 16:26:27 -06:00
|
|
|
nc->avail = 0;
|
|
|
|
}
|
2007-10-18 04:05:09 -06:00
|
|
|
|
2009-12-17 10:43:12 -07:00
|
|
|
if (!cpumask_empty(mask)) {
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2014-10-09 16:26:27 -06:00
|
|
|
goto free_slab;
|
2007-10-18 04:05:09 -06:00
|
|
|
}
|
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
shared = n->shared;
|
2007-10-18 04:05:09 -06:00
|
|
|
if (shared) {
|
|
|
|
free_block(cachep, shared->entry,
|
2014-08-06 17:04:25 -06:00
|
|
|
shared->avail, node, &list);
|
2013-01-10 12:14:19 -07:00
|
|
|
n->shared = NULL;
|
2007-10-18 04:05:09 -06:00
|
|
|
}
|
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
alien = n->alien;
|
|
|
|
n->alien = NULL;
|
2007-10-18 04:05:09 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2007-10-18 04:05:09 -06:00
|
|
|
|
|
|
|
kfree(shared);
|
|
|
|
if (alien) {
|
|
|
|
drain_alien_cache(cachep, alien);
|
|
|
|
free_alien_cache(alien);
|
|
|
|
}
|
2014-10-09 16:26:27 -06:00
|
|
|
|
|
|
|
free_slab:
|
2014-08-06 17:04:25 -06:00
|
|
|
slabs_destroy(cachep, &list);
|
2007-10-18 04:05:09 -06:00
|
|
|
}
|
|
|
|
/*
|
|
|
|
* In the previous loop, all the objects were freed to
|
|
|
|
* the respective cache's slabs, now we can go ahead and
|
|
|
|
* shrink each nodelist to its limit.
|
|
|
|
*/
|
2012-07-06 14:25:12 -06:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
2013-01-10 12:14:19 -07:00
|
|
|
if (!n)
|
2007-10-18 04:05:09 -06:00
|
|
|
continue;
|
2013-07-03 18:33:22 -06:00
|
|
|
drain_freelist(cachep, n, slabs_tofree(cachep, n));
|
2007-10-18 04:05:09 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-06-19 12:53:51 -06:00
|
|
|
static int cpuup_prepare(long cpu)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-02-01 04:05:50 -07:00
|
|
|
struct kmem_cache *cachep;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n = NULL;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
int node = cpu_to_mem(cpu);
|
2010-03-27 20:40:47 -06:00
|
|
|
int err;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-18 04:05:09 -06:00
|
|
|
/*
|
|
|
|
* We need to do this right in the beginning since
|
|
|
|
* alloc_arraycache's are going to use this list.
|
|
|
|
* kmalloc_node allows us to add the slab to the right
|
2013-01-10 12:14:19 -07:00
|
|
|
* kmem_cache_node and not this cpu's kmem_cache_node
|
2007-10-18 04:05:09 -06:00
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
err = init_cache_node_node(node);
|
2010-03-27 20:40:47 -06:00
|
|
|
if (err < 0)
|
|
|
|
goto bad;
|
2007-10-18 04:05:09 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Now we can go ahead with allocating the shared arrays and
|
|
|
|
* array caches
|
|
|
|
*/
|
2012-07-06 14:25:12 -06:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2007-10-18 04:05:09 -06:00
|
|
|
struct array_cache *shared = NULL;
|
2014-08-06 17:04:29 -06:00
|
|
|
struct alien_cache **alien = NULL;
|
2007-10-18 04:05:09 -06:00
|
|
|
|
|
|
|
if (cachep->shared) {
|
|
|
|
shared = alloc_arraycache(node,
|
|
|
|
cachep->shared * cachep->batchcount,
|
2009-06-10 10:40:04 -06:00
|
|
|
0xbaadf00d, GFP_KERNEL);
|
2014-10-09 16:26:27 -06:00
|
|
|
if (!shared)
|
2005-04-16 16:20:36 -06:00
|
|
|
goto bad;
|
2007-10-18 04:05:09 -06:00
|
|
|
}
|
|
|
|
if (use_alien_caches) {
|
2009-06-10 10:40:04 -06:00
|
|
|
alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL);
|
2007-10-18 04:05:11 -06:00
|
|
|
if (!alien) {
|
|
|
|
kfree(shared);
|
2007-10-18 04:05:09 -06:00
|
|
|
goto bad;
|
2007-10-18 04:05:11 -06:00
|
|
|
}
|
2007-10-18 04:05:09 -06:00
|
|
|
}
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
2013-01-10 12:14:19 -07:00
|
|
|
BUG_ON(!n);
|
2007-10-18 04:05:09 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock_irq(&n->list_lock);
|
|
|
|
if (!n->shared) {
|
2007-10-18 04:05:09 -06:00
|
|
|
/*
|
|
|
|
* We are serialised from CPU_DEAD or
|
|
|
|
* CPU_UP_CANCELLED by the cpucontrol lock
|
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
n->shared = shared;
|
2007-10-18 04:05:09 -06:00
|
|
|
shared = NULL;
|
|
|
|
}
|
[PATCH] NUMA slab locking fixes: fix cpu down and up locking
This fixes locking and bugs in cpu_down and cpu_up paths of the NUMA slab
allocator. Sonny Rao <sonny@burdell.org> reported problems sometime back on
POWER5 boxes, when the last cpu on the nodes were being offlined. We could
not reproduce the same on x86_64 because the cpumask (node_to_cpumask) was not
being updated on cpu down. Since that issue is now fixed, we can reproduce
Sonny's problems on x86_64 NUMA, and here is the fix.
The problem earlier was on CPU_DOWN, if it was the last cpu on the node to go
down, the array_caches (shared, alien) and the kmem_list3 of the node were
being freed (kfree) with the kmem_list3 lock held. If the l3 or the
array_caches were to come from the same cache being cleared, we hit on
badness.
This patch cleans up the locking in cpu_up and cpu_down path. We cannot
really free l3 on cpu down because, there is no node offlining yet and even
though a cpu is not yet up, node local memory can be allocated for it. So l3s
are usually allocated at keme_cache_create and destroyed at
kmem_cache_destroy. Hence, we don't need cachep->spinlock protection to get
to the cachep->nodelist[nodeid] either.
Patch survived onlining and offlining on a 4 core 2 node Tyan box with a 4
dbench process running all the time.
Signed-off-by: Alok N Kataria <alokk@calsoftinc.com>
Signed-off-by: Ravikiran Thirumalai <kiran@scalex86.org>
Cc: Christoph Lameter <christoph@lameter.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-02-05 00:27:59 -07:00
|
|
|
#ifdef CONFIG_NUMA
|
2013-01-10 12:14:19 -07:00
|
|
|
if (!n->alien) {
|
|
|
|
n->alien = alien;
|
2007-10-18 04:05:09 -06:00
|
|
|
alien = NULL;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2007-10-18 04:05:09 -06:00
|
|
|
#endif
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2007-10-18 04:05:09 -06:00
|
|
|
kfree(shared);
|
|
|
|
free_alien_cache(alien);
|
|
|
|
}
|
2009-11-23 13:01:15 -07:00
|
|
|
|
2007-10-18 04:05:09 -06:00
|
|
|
return 0;
|
|
|
|
bad:
|
2007-10-18 04:05:11 -06:00
|
|
|
cpuup_canceled(cpu);
|
2007-10-18 04:05:09 -06:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2013-06-19 12:53:51 -06:00
|
|
|
static int cpuup_callback(struct notifier_block *nfb,
|
2007-10-18 04:05:09 -06:00
|
|
|
unsigned long action, void *hcpu)
|
|
|
|
{
|
|
|
|
long cpu = (long)hcpu;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
switch (action) {
|
|
|
|
case CPU_UP_PREPARE:
|
|
|
|
case CPU_UP_PREPARE_FROZEN:
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_lock(&slab_mutex);
|
2007-10-18 04:05:09 -06:00
|
|
|
err = cpuup_prepare(cpu);
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_unlock(&slab_mutex);
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
|
|
|
case CPU_ONLINE:
|
2007-05-09 03:35:10 -06:00
|
|
|
case CPU_ONLINE_FROZEN:
|
2005-04-16 16:20:36 -06:00
|
|
|
start_cpu_timer(cpu);
|
|
|
|
break;
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
2007-05-09 03:34:22 -06:00
|
|
|
case CPU_DOWN_PREPARE:
|
2007-05-09 03:35:10 -06:00
|
|
|
case CPU_DOWN_PREPARE_FROZEN:
|
2007-05-09 03:34:22 -06:00
|
|
|
/*
|
2012-07-06 14:25:12 -06:00
|
|
|
* Shutdown cache reaper. Note that the slab_mutex is
|
2007-05-09 03:34:22 -06:00
|
|
|
* held so that if cache_reap() is invoked it cannot do
|
|
|
|
* anything expensive but will only modify reap_work
|
|
|
|
* and reschedule the timer.
|
|
|
|
*/
|
2010-12-14 08:21:17 -07:00
|
|
|
cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
|
2007-05-09 03:34:22 -06:00
|
|
|
/* Now the cache_reaper is guaranteed to be not running. */
|
2009-10-29 07:34:13 -06:00
|
|
|
per_cpu(slab_reap_work, cpu).work.func = NULL;
|
2007-05-09 03:34:22 -06:00
|
|
|
break;
|
|
|
|
case CPU_DOWN_FAILED:
|
2007-05-09 03:35:10 -06:00
|
|
|
case CPU_DOWN_FAILED_FROZEN:
|
2007-05-09 03:34:22 -06:00
|
|
|
start_cpu_timer(cpu);
|
|
|
|
break;
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_DEAD:
|
2007-05-09 03:35:10 -06:00
|
|
|
case CPU_DEAD_FROZEN:
|
[PATCH] NUMA slab locking fixes: fix cpu down and up locking
This fixes locking and bugs in cpu_down and cpu_up paths of the NUMA slab
allocator. Sonny Rao <sonny@burdell.org> reported problems sometime back on
POWER5 boxes, when the last cpu on the nodes were being offlined. We could
not reproduce the same on x86_64 because the cpumask (node_to_cpumask) was not
being updated on cpu down. Since that issue is now fixed, we can reproduce
Sonny's problems on x86_64 NUMA, and here is the fix.
The problem earlier was on CPU_DOWN, if it was the last cpu on the node to go
down, the array_caches (shared, alien) and the kmem_list3 of the node were
being freed (kfree) with the kmem_list3 lock held. If the l3 or the
array_caches were to come from the same cache being cleared, we hit on
badness.
This patch cleans up the locking in cpu_up and cpu_down path. We cannot
really free l3 on cpu down because, there is no node offlining yet and even
though a cpu is not yet up, node local memory can be allocated for it. So l3s
are usually allocated at keme_cache_create and destroyed at
kmem_cache_destroy. Hence, we don't need cachep->spinlock protection to get
to the cachep->nodelist[nodeid] either.
Patch survived onlining and offlining on a 4 core 2 node Tyan box with a 4
dbench process running all the time.
Signed-off-by: Alok N Kataria <alokk@calsoftinc.com>
Signed-off-by: Ravikiran Thirumalai <kiran@scalex86.org>
Cc: Christoph Lameter <christoph@lameter.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-02-05 00:27:59 -07:00
|
|
|
/*
|
|
|
|
* Even if all the cpus of a node are down, we don't free the
|
2013-01-10 12:14:19 -07:00
|
|
|
* kmem_cache_node of any cache. This to avoid a race between
|
[PATCH] NUMA slab locking fixes: fix cpu down and up locking
This fixes locking and bugs in cpu_down and cpu_up paths of the NUMA slab
allocator. Sonny Rao <sonny@burdell.org> reported problems sometime back on
POWER5 boxes, when the last cpu on the nodes were being offlined. We could
not reproduce the same on x86_64 because the cpumask (node_to_cpumask) was not
being updated on cpu down. Since that issue is now fixed, we can reproduce
Sonny's problems on x86_64 NUMA, and here is the fix.
The problem earlier was on CPU_DOWN, if it was the last cpu on the node to go
down, the array_caches (shared, alien) and the kmem_list3 of the node were
being freed (kfree) with the kmem_list3 lock held. If the l3 or the
array_caches were to come from the same cache being cleared, we hit on
badness.
This patch cleans up the locking in cpu_up and cpu_down path. We cannot
really free l3 on cpu down because, there is no node offlining yet and even
though a cpu is not yet up, node local memory can be allocated for it. So l3s
are usually allocated at keme_cache_create and destroyed at
kmem_cache_destroy. Hence, we don't need cachep->spinlock protection to get
to the cachep->nodelist[nodeid] either.
Patch survived onlining and offlining on a 4 core 2 node Tyan box with a 4
dbench process running all the time.
Signed-off-by: Alok N Kataria <alokk@calsoftinc.com>
Signed-off-by: Ravikiran Thirumalai <kiran@scalex86.org>
Cc: Christoph Lameter <christoph@lameter.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-02-05 00:27:59 -07:00
|
|
|
* cpu_down, and a kmalloc allocation from another cpu for
|
2013-01-10 12:14:19 -07:00
|
|
|
* memory from the node of the cpu going down. The node
|
[PATCH] NUMA slab locking fixes: fix cpu down and up locking
This fixes locking and bugs in cpu_down and cpu_up paths of the NUMA slab
allocator. Sonny Rao <sonny@burdell.org> reported problems sometime back on
POWER5 boxes, when the last cpu on the nodes were being offlined. We could
not reproduce the same on x86_64 because the cpumask (node_to_cpumask) was not
being updated on cpu down. Since that issue is now fixed, we can reproduce
Sonny's problems on x86_64 NUMA, and here is the fix.
The problem earlier was on CPU_DOWN, if it was the last cpu on the node to go
down, the array_caches (shared, alien) and the kmem_list3 of the node were
being freed (kfree) with the kmem_list3 lock held. If the l3 or the
array_caches were to come from the same cache being cleared, we hit on
badness.
This patch cleans up the locking in cpu_up and cpu_down path. We cannot
really free l3 on cpu down because, there is no node offlining yet and even
though a cpu is not yet up, node local memory can be allocated for it. So l3s
are usually allocated at keme_cache_create and destroyed at
kmem_cache_destroy. Hence, we don't need cachep->spinlock protection to get
to the cachep->nodelist[nodeid] either.
Patch survived onlining and offlining on a 4 core 2 node Tyan box with a 4
dbench process running all the time.
Signed-off-by: Alok N Kataria <alokk@calsoftinc.com>
Signed-off-by: Ravikiran Thirumalai <kiran@scalex86.org>
Cc: Christoph Lameter <christoph@lameter.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-02-05 00:27:59 -07:00
|
|
|
* structure is usually allocated from kmem_cache_create() and
|
|
|
|
* gets destroyed at kmem_cache_destroy().
|
|
|
|
*/
|
2007-10-19 17:27:18 -06:00
|
|
|
/* fall through */
|
[PATCH] mm: slab: eliminate lock_cpu_hotplug from slab
Here's an attempt towards doing away with lock_cpu_hotplug in the slab
subsystem. This approach also fixes a bug which shows up when cpus are
being offlined/onlined and slab caches are being tuned simultaneously.
http://marc.theaimsgroup.com/?l=linux-kernel&m=116098888100481&w=2
The patch has been stress tested overnight on a 2 socket 4 core AMD box with
repeated cpu online and offline, while dbench and kernbench process are
running, and slab caches being tuned at the same time.
There were no lockdep warnings either. (This test on 2,6.18 as 2.6.19-rc
crashes at __drain_pages
http://marc.theaimsgroup.com/?l=linux-kernel&m=116172164217678&w=2 )
The approach here is to hold cache_chain_mutex from CPU_UP_PREPARE until
CPU_ONLINE (similar in approach as worqueue_mutex) . Slab code sensitive
to cpu_online_map (kmem_cache_create, kmem_cache_destroy, slabinfo_write,
__cache_shrink) is already serialized with cache_chain_mutex. (This patch
lengthens cache_chain_mutex hold time at kmem_cache_destroy to cover this).
This patch also takes the cache_chain_sem at kmem_cache_shrink to protect
sanity of cpu_online_map at __cache_shrink, as viewed by slab.
(kmem_cache_shrink->__cache_shrink->drain_cpu_caches). But, really,
kmem_cache_shrink is used at just one place in the acpi subsystem! Do we
really need to keep kmem_cache_shrink at all?
Another note. Looks like a cpu hotplug event can send CPU_UP_CANCELED to
a registered subsystem even if the subsystem did not receive CPU_UP_PREPARE.
This could be due to a subsystem registered for notification earlier than
the current subsystem crapping out with NOTIFY_BAD. Badness can occur with
in the CPU_UP_CANCELED code path at slab if this happens (The same would
apply for workqueue.c as well). To overcome this, we might have to use either
a) a per subsystem flag and avoid handling of CPU_UP_CANCELED, or
b) Use a special notifier events like LOCK_ACQUIRE/RELEASE as Gautham was
using in his experiments, or
c) Do not send CPU_UP_CANCELED to a subsystem which did not receive
CPU_UP_PREPARE.
I would prefer c).
Signed-off-by: Ravikiran Thirumalai <kiran@scalex86.org>
Signed-off-by: Shai Fultheim <shai@scalex86.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-06 21:32:14 -07:00
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
case CPU_UP_CANCELED:
|
2007-05-09 03:35:10 -06:00
|
|
|
case CPU_UP_CANCELED_FROZEN:
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_lock(&slab_mutex);
|
2007-10-18 04:05:09 -06:00
|
|
|
cpuup_canceled(cpu);
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_unlock(&slab_mutex);
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
|
|
|
}
|
2010-05-26 15:43:32 -06:00
|
|
|
return notifier_from_errno(err);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2013-06-19 12:53:51 -06:00
|
|
|
static struct notifier_block cpucache_notifier = {
|
2006-06-27 03:54:09 -06:00
|
|
|
&cpuup_callback, NULL, 0
|
|
|
|
};
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-03-27 20:40:47 -06:00
|
|
|
#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
|
|
|
|
/*
|
|
|
|
* Drains freelist for a node on each slab cache, used for memory hot-remove.
|
|
|
|
* Returns -EBUSY if all objects cannot be drained so that the node is not
|
|
|
|
* removed.
|
|
|
|
*
|
2012-07-06 14:25:12 -06:00
|
|
|
* Must hold slab_mutex.
|
2010-03-27 20:40:47 -06:00
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
static int __meminit drain_cache_node_node(int node)
|
2010-03-27 20:40:47 -06:00
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
|
|
|
int ret = 0;
|
|
|
|
|
2012-07-06 14:25:12 -06:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2010-03-27 20:40:47 -06:00
|
|
|
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
2013-01-10 12:14:19 -07:00
|
|
|
if (!n)
|
2010-03-27 20:40:47 -06:00
|
|
|
continue;
|
|
|
|
|
2013-07-03 18:33:22 -06:00
|
|
|
drain_freelist(cachep, n, slabs_tofree(cachep, n));
|
2010-03-27 20:40:47 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
if (!list_empty(&n->slabs_full) ||
|
|
|
|
!list_empty(&n->slabs_partial)) {
|
2010-03-27 20:40:47 -06:00
|
|
|
ret = -EBUSY;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __meminit slab_memory_callback(struct notifier_block *self,
|
|
|
|
unsigned long action, void *arg)
|
|
|
|
{
|
|
|
|
struct memory_notify *mnb = arg;
|
|
|
|
int ret = 0;
|
|
|
|
int nid;
|
|
|
|
|
|
|
|
nid = mnb->status_change_nid;
|
|
|
|
if (nid < 0)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
switch (action) {
|
|
|
|
case MEM_GOING_ONLINE:
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_lock(&slab_mutex);
|
2013-01-10 12:14:19 -07:00
|
|
|
ret = init_cache_node_node(nid);
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_unlock(&slab_mutex);
|
2010-03-27 20:40:47 -06:00
|
|
|
break;
|
|
|
|
case MEM_GOING_OFFLINE:
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_lock(&slab_mutex);
|
2013-01-10 12:14:19 -07:00
|
|
|
ret = drain_cache_node_node(nid);
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_unlock(&slab_mutex);
|
2010-03-27 20:40:47 -06:00
|
|
|
break;
|
|
|
|
case MEM_ONLINE:
|
|
|
|
case MEM_OFFLINE:
|
|
|
|
case MEM_CANCEL_ONLINE:
|
|
|
|
case MEM_CANCEL_OFFLINE:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
out:
|
2011-03-22 17:30:49 -06:00
|
|
|
return notifier_from_errno(ret);
|
2010-03-27 20:40:47 -06:00
|
|
|
}
|
|
|
|
#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
|
|
|
|
|
2005-09-09 14:03:32 -06:00
|
|
|
/*
|
2013-01-10 12:14:19 -07:00
|
|
|
* swap the static kmem_cache_node with kmalloced memory
|
2005-09-09 14:03:32 -06:00
|
|
|
*/
|
2013-01-10 12:12:17 -07:00
|
|
|
static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
|
2010-03-27 20:40:47 -06:00
|
|
|
int nodeid)
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
2013-01-10 12:12:17 -07:00
|
|
|
struct kmem_cache_node *ptr;
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2013-01-10 12:12:17 -07:00
|
|
|
ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
|
2005-09-09 14:03:32 -06:00
|
|
|
BUG_ON(!ptr);
|
|
|
|
|
2013-01-10 12:12:17 -07:00
|
|
|
memcpy(ptr, list, sizeof(struct kmem_cache_node));
|
2006-07-03 01:25:28 -06:00
|
|
|
/*
|
|
|
|
* Do not assume that spinlocks can be initialized via memcpy:
|
|
|
|
*/
|
|
|
|
spin_lock_init(&ptr->list_lock);
|
|
|
|
|
2005-09-09 14:03:32 -06:00
|
|
|
MAKE_ALL_LISTS(cachep, ptr, nodeid);
|
2013-01-10 12:14:19 -07:00
|
|
|
cachep->node[nodeid] = ptr;
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
|
|
|
|
2008-01-24 23:20:51 -07:00
|
|
|
/*
|
2013-01-10 12:14:19 -07:00
|
|
|
* For setting up all the kmem_cache_node for cache whose buffer_size is same as
|
|
|
|
* size of kmem_cache_node.
|
2008-01-24 23:20:51 -07:00
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
static void __init set_up_node(struct kmem_cache *cachep, int index)
|
2008-01-24 23:20:51 -07:00
|
|
|
{
|
|
|
|
int node;
|
|
|
|
|
|
|
|
for_each_online_node(node) {
|
2013-01-10 12:14:19 -07:00
|
|
|
cachep->node[node] = &init_kmem_cache_node[index + node];
|
2013-01-10 12:14:19 -07:00
|
|
|
cachep->node[node]->next_reap = jiffies +
|
2014-03-30 03:02:20 -06:00
|
|
|
REAPTIMEOUT_NODE +
|
|
|
|
((unsigned long)cachep) % REAPTIMEOUT_NODE;
|
2008-01-24 23:20:51 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* Initialisation. Called after the page allocator have been initialised and
|
|
|
|
* before smp_init().
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
void __init kmem_cache_init(void)
|
|
|
|
{
|
2005-09-09 14:03:32 -06:00
|
|
|
int i;
|
|
|
|
|
2013-10-23 19:07:42 -06:00
|
|
|
BUILD_BUG_ON(sizeof(((struct page *)NULL)->lru) <
|
|
|
|
sizeof(struct rcu_head));
|
2012-09-04 18:20:33 -06:00
|
|
|
kmem_cache = &kmem_cache_boot;
|
|
|
|
|
2009-06-16 16:32:16 -06:00
|
|
|
if (num_possible_nodes() == 1)
|
2007-05-02 11:27:18 -06:00
|
|
|
use_alien_caches = 0;
|
|
|
|
|
2012-11-28 09:23:01 -07:00
|
|
|
for (i = 0; i < NUM_INIT_LISTS; i++)
|
2013-01-10 12:14:19 -07:00
|
|
|
kmem_cache_node_init(&init_kmem_cache_node[i]);
|
2012-11-28 09:23:01 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* Fragmentation resistance on low memory - only use bigger
|
2011-10-18 23:09:28 -06:00
|
|
|
* page orders on machines with more than 32MB of memory if
|
|
|
|
* not overridden on the command line.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2011-10-18 23:09:28 -06:00
|
|
|
if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
|
2011-10-18 23:09:24 -06:00
|
|
|
slab_max_order = SLAB_MAX_ORDER_HI;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Bootstrap is tricky, because several objects are allocated
|
|
|
|
* from caches that do not exist yet:
|
2012-09-04 18:20:33 -06:00
|
|
|
* 1) initialize the kmem_cache cache: it contains the struct
|
|
|
|
* kmem_cache structures of all caches, except kmem_cache itself:
|
|
|
|
* kmem_cache is statically allocated.
|
2005-09-09 14:03:32 -06:00
|
|
|
* Initially an __init data area is used for the head array and the
|
2013-01-10 12:14:19 -07:00
|
|
|
* kmem_cache_node structures, it's replaced with a kmalloc allocated
|
2005-09-09 14:03:32 -06:00
|
|
|
* array at the end of the bootstrap.
|
2005-04-16 16:20:36 -06:00
|
|
|
* 2) Create the first kmalloc cache.
|
2006-02-01 04:05:50 -07:00
|
|
|
* The struct kmem_cache for the new cache is allocated normally.
|
2005-09-09 14:03:32 -06:00
|
|
|
* An __init data area is used for the head array.
|
|
|
|
* 3) Create the remaining kmalloc caches, with minimally sized
|
|
|
|
* head arrays.
|
2012-09-04 18:20:33 -06:00
|
|
|
* 4) Replace the __init data head arrays for kmem_cache and the first
|
2005-04-16 16:20:36 -06:00
|
|
|
* kmalloc cache with kmalloc allocated arrays.
|
2013-01-10 12:14:19 -07:00
|
|
|
* 5) Replace the __init data for kmem_cache_node for kmem_cache and
|
2005-09-09 14:03:32 -06:00
|
|
|
* the other cache's with kmalloc allocated memory.
|
|
|
|
* 6) Resize the head arrays of the kmalloc caches to their final sizes.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
|
2012-09-04 18:20:33 -06:00
|
|
|
/* 1) create the kmem_cache */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-05-06 15:49:29 -06:00
|
|
|
/*
|
2011-07-20 11:04:23 -06:00
|
|
|
* struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
|
2007-05-06 15:49:29 -06:00
|
|
|
*/
|
2012-11-28 09:23:09 -07:00
|
|
|
create_boot_cache(kmem_cache, "kmem_cache",
|
2014-10-09 16:26:27 -06:00
|
|
|
offsetof(struct kmem_cache, node) +
|
2013-01-10 12:12:17 -07:00
|
|
|
nr_node_ids * sizeof(struct kmem_cache_node *),
|
2012-11-28 09:23:09 -07:00
|
|
|
SLAB_HWCACHE_ALIGN);
|
|
|
|
list_add(&kmem_cache->list, &slab_caches);
|
2014-10-09 16:26:27 -06:00
|
|
|
slab_state = PARTIAL;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
2014-10-09 16:26:27 -06:00
|
|
|
* Initialize the caches that provide memory for the kmem_cache_node
|
|
|
|
* structures first. Without this, further allocations will bug.
|
2005-09-09 14:03:32 -06:00
|
|
|
*/
|
2014-10-09 16:26:27 -06:00
|
|
|
kmalloc_caches[INDEX_NODE] = create_kmalloc_cache("kmalloc-node",
|
2013-01-10 12:14:19 -07:00
|
|
|
kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS);
|
2014-10-09 16:26:27 -06:00
|
|
|
slab_state = PARTIAL_NODE;
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2006-06-23 03:03:46 -06:00
|
|
|
slab_early_init = 0;
|
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
/* 5) Replace the bootstrap kmem_cache_node */
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
2006-10-06 01:43:52 -06:00
|
|
|
int nid;
|
|
|
|
|
2008-01-24 06:49:54 -07:00
|
|
|
for_each_online_node(nid) {
|
2013-01-10 12:14:19 -07:00
|
|
|
init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid);
|
2008-01-24 23:20:51 -07:00
|
|
|
|
2014-10-09 16:26:27 -06:00
|
|
|
init_list(kmalloc_caches[INDEX_NODE],
|
2013-01-10 12:14:19 -07:00
|
|
|
&init_kmem_cache_node[SIZE_NODE + nid], nid);
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-01-10 12:12:17 -07:00
|
|
|
create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
|
2009-06-12 06:58:59 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void __init kmem_cache_init_late(void)
|
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
|
|
|
|
2012-07-06 14:25:11 -06:00
|
|
|
slab_state = UP;
|
2011-11-28 13:12:40 -07:00
|
|
|
|
2009-06-12 06:58:59 -06:00
|
|
|
/* 6) resize the head arrays to their final sizes */
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_lock(&slab_mutex);
|
|
|
|
list_for_each_entry(cachep, &slab_caches, list)
|
2009-06-12 06:58:59 -06:00
|
|
|
if (enable_cpucache(cachep, GFP_NOWAIT))
|
|
|
|
BUG();
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_unlock(&slab_mutex);
|
2006-09-26 00:31:38 -06:00
|
|
|
|
2012-07-06 14:25:11 -06:00
|
|
|
/* Done! */
|
|
|
|
slab_state = FULL;
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* Register a cpu startup notifier callback that initializes
|
|
|
|
* cpu_cache_get for all new cpus
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
register_cpu_notifier(&cpucache_notifier);
|
|
|
|
|
2010-03-27 20:40:47 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
/*
|
|
|
|
* Register a memory hotplug callback that initializes and frees
|
2013-01-10 12:14:19 -07:00
|
|
|
* node.
|
2010-03-27 20:40:47 -06:00
|
|
|
*/
|
|
|
|
hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
|
|
|
|
#endif
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* The reap timers are started later, with a module init call: That part
|
|
|
|
* of the kernel is not yet operational.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init cpucache_init(void)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* Register the timers that return unneeded pages to the page allocator
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2005-09-09 14:03:32 -06:00
|
|
|
for_each_online_cpu(cpu)
|
2006-03-22 01:08:11 -07:00
|
|
|
start_cpu_timer(cpu);
|
slab: move FULL state transition to an initcall
During kmem_cache_init_late(), we transition to the LATE state,
and after some more work, to the FULL state, its last state
This is quite different from slub, that will only transition to
its last state (previously SYSFS), in a (late)initcall, after a lot
more of the kernel is ready.
This means that in slab, we have no way to taking actions dependent
on the initialization of other pieces of the kernel that are supposed
to start way after kmem_init_late(), such as cgroups initialization.
To achieve more consistency in this behavior, that patch only
transitions to the UP state in kmem_init_late. In my analysis,
setup_cpu_cache() should be happy to test for >= UP, instead of
== FULL. It also has passed some tests I've made.
We then only mark FULL state after the reap timers are in place,
meaning that no further setup is expected.
Signed-off-by: Glauber Costa <glommer@parallels.com>
Acked-by: Christoph Lameter <cl@linux.com>
Acked-by: David Rientjes <rientjes@google.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>
2012-06-20 14:59:18 -06:00
|
|
|
|
|
|
|
/* Done! */
|
2012-07-06 14:25:11 -06:00
|
|
|
slab_state = FULL;
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
__initcall(cpucache_init);
|
|
|
|
|
2012-03-09 13:27:27 -07:00
|
|
|
static noinline void
|
|
|
|
slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
|
|
|
|
{
|
mm, slab: suppress out of memory warning unless debug is enabled
When the slab or slub allocators cannot allocate additional slab pages,
they emit diagnostic information to the kernel log such as current
number of slabs, number of objects, active objects, etc. This is always
coupled with a page allocation failure warning since it is controlled by
!__GFP_NOWARN.
Suppress this out of memory warning if the allocator is configured
without debug supported. The page allocation failure warning will
indicate it is a failed slab allocation, the order, and the gfp mask, so
this is only useful to diagnose allocator issues.
Since CONFIG_SLUB_DEBUG is already enabled by default for the slub
allocator, there is no functional change with this patch. If debug is
disabled, however, the warnings are now suppressed.
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: Pekka Enberg <penberg@kernel.org>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 17:06:36 -06:00
|
|
|
#if DEBUG
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page;
|
2012-03-09 13:27:27 -07:00
|
|
|
unsigned long flags;
|
|
|
|
int node;
|
mm, slab: suppress out of memory warning unless debug is enabled
When the slab or slub allocators cannot allocate additional slab pages,
they emit diagnostic information to the kernel log such as current
number of slabs, number of objects, active objects, etc. This is always
coupled with a page allocation failure warning since it is controlled by
!__GFP_NOWARN.
Suppress this out of memory warning if the allocator is configured
without debug supported. The page allocation failure warning will
indicate it is a failed slab allocation, the order, and the gfp mask, so
this is only useful to diagnose allocator issues.
Since CONFIG_SLUB_DEBUG is already enabled by default for the slub
allocator, there is no functional change with this patch. If debug is
disabled, however, the warnings are now suppressed.
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: Pekka Enberg <penberg@kernel.org>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 17:06:36 -06:00
|
|
|
static DEFINE_RATELIMIT_STATE(slab_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
|
|
|
|
DEFAULT_RATELIMIT_BURST);
|
|
|
|
|
|
|
|
if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slab_oom_rs))
|
|
|
|
return;
|
2012-03-09 13:27:27 -07:00
|
|
|
|
|
|
|
printk(KERN_WARNING
|
|
|
|
"SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n",
|
|
|
|
nodeid, gfpflags);
|
|
|
|
printk(KERN_WARNING " cache: %s, object size: %d, order: %d\n",
|
2012-06-13 09:24:57 -06:00
|
|
|
cachep->name, cachep->size, cachep->gfporder);
|
2012-03-09 13:27:27 -07:00
|
|
|
|
2014-08-06 17:04:11 -06:00
|
|
|
for_each_kmem_cache_node(cachep, node, n) {
|
2012-03-09 13:27:27 -07:00
|
|
|
unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
|
|
|
|
unsigned long active_slabs = 0, num_slabs = 0;
|
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock_irqsave(&n->list_lock, flags);
|
2013-10-23 19:07:49 -06:00
|
|
|
list_for_each_entry(page, &n->slabs_full, lru) {
|
2012-03-09 13:27:27 -07:00
|
|
|
active_objs += cachep->num;
|
|
|
|
active_slabs++;
|
|
|
|
}
|
2013-10-23 19:07:49 -06:00
|
|
|
list_for_each_entry(page, &n->slabs_partial, lru) {
|
|
|
|
active_objs += page->active;
|
2012-03-09 13:27:27 -07:00
|
|
|
active_slabs++;
|
|
|
|
}
|
2013-10-23 19:07:49 -06:00
|
|
|
list_for_each_entry(page, &n->slabs_free, lru)
|
2012-03-09 13:27:27 -07:00
|
|
|
num_slabs++;
|
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
free_objects += n->free_objects;
|
|
|
|
spin_unlock_irqrestore(&n->list_lock, flags);
|
2012-03-09 13:27:27 -07:00
|
|
|
|
|
|
|
num_slabs += active_slabs;
|
|
|
|
num_objs = num_slabs * cachep->num;
|
|
|
|
printk(KERN_WARNING
|
|
|
|
" node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",
|
|
|
|
node, active_slabs, num_slabs, active_objs, num_objs,
|
|
|
|
free_objects);
|
|
|
|
}
|
mm, slab: suppress out of memory warning unless debug is enabled
When the slab or slub allocators cannot allocate additional slab pages,
they emit diagnostic information to the kernel log such as current
number of slabs, number of objects, active objects, etc. This is always
coupled with a page allocation failure warning since it is controlled by
!__GFP_NOWARN.
Suppress this out of memory warning if the allocator is configured
without debug supported. The page allocation failure warning will
indicate it is a failed slab allocation, the order, and the gfp mask, so
this is only useful to diagnose allocator issues.
Since CONFIG_SLUB_DEBUG is already enabled by default for the slub
allocator, there is no functional change with this patch. If debug is
disabled, however, the warnings are now suppressed.
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: Pekka Enberg <penberg@kernel.org>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 17:06:36 -06:00
|
|
|
#endif
|
2012-03-09 13:27:27 -07:00
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
2014-08-06 17:04:46 -06:00
|
|
|
* Interface to system's page allocator. No need to hold the
|
|
|
|
* kmem_cache_node ->list_lock.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* If we requested dmaable memory, we will get it. Even if we
|
|
|
|
* did not request dmaable memory, we might get it, but that
|
|
|
|
* would be relatively rare and ignorable.
|
|
|
|
*/
|
2013-10-23 19:07:38 -06:00
|
|
|
static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
|
|
|
|
int nodeid)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct page *page;
|
2006-06-23 03:03:17 -06:00
|
|
|
int nr_pages;
|
2006-09-27 02:50:08 -06:00
|
|
|
|
2012-06-14 06:17:21 -06:00
|
|
|
flags |= cachep->allocflags;
|
2007-10-16 02:25:52 -06:00
|
|
|
if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
|
|
|
|
flags |= __GFP_RECLAIMABLE;
|
2006-06-23 03:03:17 -06:00
|
|
|
|
2014-06-04 17:06:38 -06:00
|
|
|
if (memcg_charge_slab(cachep, flags, cachep->gfporder))
|
|
|
|
return NULL;
|
|
|
|
|
2009-06-16 20:50:13 -06:00
|
|
|
page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
|
2012-03-09 13:27:27 -07:00
|
|
|
if (!page) {
|
2014-06-04 17:06:38 -06:00
|
|
|
memcg_uncharge_slab(cachep, cachep->gfporder);
|
mm, slab: suppress out of memory warning unless debug is enabled
When the slab or slub allocators cannot allocate additional slab pages,
they emit diagnostic information to the kernel log such as current
number of slabs, number of objects, active objects, etc. This is always
coupled with a page allocation failure warning since it is controlled by
!__GFP_NOWARN.
Suppress this out of memory warning if the allocator is configured
without debug supported. The page allocation failure warning will
indicate it is a failed slab allocation, the order, and the gfp mask, so
this is only useful to diagnose allocator issues.
Since CONFIG_SLUB_DEBUG is already enabled by default for the slub
allocator, there is no functional change with this patch. If debug is
disabled, however, the warnings are now suppressed.
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: Pekka Enberg <penberg@kernel.org>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 17:06:36 -06:00
|
|
|
slab_out_of_memory(cachep, flags, nodeid);
|
2005-04-16 16:20:36 -06:00
|
|
|
return NULL;
|
2012-03-09 13:27:27 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-07-31 17:44:03 -06:00
|
|
|
/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
if (unlikely(page->pfmemalloc))
|
|
|
|
pfmemalloc_active = true;
|
|
|
|
|
2006-06-23 03:03:17 -06:00
|
|
|
nr_pages = (1 << cachep->gfporder);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
|
2006-09-26 00:31:51 -06:00
|
|
|
add_zone_page_state(page_zone(page),
|
|
|
|
NR_SLAB_RECLAIMABLE, nr_pages);
|
|
|
|
else
|
|
|
|
add_zone_page_state(page_zone(page),
|
|
|
|
NR_SLAB_UNRECLAIMABLE, nr_pages);
|
2013-10-23 19:07:44 -06:00
|
|
|
__SetPageSlab(page);
|
|
|
|
if (page->pfmemalloc)
|
|
|
|
SetPageSlabPfmemalloc(page);
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
|
2008-11-25 08:55:53 -07:00
|
|
|
if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
|
|
|
|
kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
|
|
|
|
|
|
|
|
if (cachep->ctor)
|
|
|
|
kmemcheck_mark_uninitialized_pages(page, nr_pages);
|
|
|
|
else
|
|
|
|
kmemcheck_mark_unallocated_pages(page, nr_pages);
|
|
|
|
}
|
2008-05-09 12:35:53 -06:00
|
|
|
|
2013-10-23 19:07:38 -06:00
|
|
|
return page;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Interface to system's page release.
|
|
|
|
*/
|
2013-10-23 19:07:38 -06:00
|
|
|
static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2013-10-23 19:07:44 -06:00
|
|
|
const unsigned long nr_freed = (1 << cachep->gfporder);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-11-25 08:55:53 -07:00
|
|
|
kmemcheck_free_shadow(page, cachep->gfporder);
|
2008-05-09 12:35:53 -06:00
|
|
|
|
2006-09-26 00:31:51 -06:00
|
|
|
if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
|
|
|
|
sub_zone_page_state(page_zone(page),
|
|
|
|
NR_SLAB_RECLAIMABLE, nr_freed);
|
|
|
|
else
|
|
|
|
sub_zone_page_state(page_zone(page),
|
|
|
|
NR_SLAB_UNRECLAIMABLE, nr_freed);
|
2013-10-23 19:07:37 -06:00
|
|
|
|
2013-10-23 19:07:44 -06:00
|
|
|
BUG_ON(!PageSlab(page));
|
2013-10-23 19:07:37 -06:00
|
|
|
__ClearPageSlabPfmemalloc(page);
|
2013-10-23 19:07:44 -06:00
|
|
|
__ClearPageSlab(page);
|
2013-10-23 19:07:49 -06:00
|
|
|
page_mapcount_reset(page);
|
|
|
|
page->mapping = NULL;
|
2012-12-18 15:22:50 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (current->reclaim_state)
|
|
|
|
current->reclaim_state->reclaimed_slab += nr_freed;
|
2014-06-04 17:06:38 -06:00
|
|
|
__free_pages(page, cachep->gfporder);
|
|
|
|
memcg_uncharge_slab(cachep, cachep->gfporder);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void kmem_rcu_free(struct rcu_head *head)
|
|
|
|
{
|
2013-10-23 19:07:42 -06:00
|
|
|
struct kmem_cache *cachep;
|
|
|
|
struct page *page;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-10-23 19:07:42 -06:00
|
|
|
page = container_of(head, struct page, rcu_head);
|
|
|
|
cachep = page->slab_cache;
|
|
|
|
|
|
|
|
kmem_freepages(cachep, page);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
2006-02-01 04:05:50 -07:00
|
|
|
static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
|
2006-01-08 02:00:37 -07:00
|
|
|
unsigned long caller)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2012-06-13 09:24:58 -06:00
|
|
|
int size = cachep->object_size;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-02-01 04:05:42 -07:00
|
|
|
addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-01-08 02:00:37 -07:00
|
|
|
if (size < 5 * sizeof(unsigned long))
|
2005-04-16 16:20:36 -06:00
|
|
|
return;
|
|
|
|
|
2006-01-08 02:00:37 -07:00
|
|
|
*addr++ = 0x12345678;
|
|
|
|
*addr++ = caller;
|
|
|
|
*addr++ = smp_processor_id();
|
|
|
|
size -= 3 * sizeof(unsigned long);
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
unsigned long *sptr = &caller;
|
|
|
|
unsigned long svalue;
|
|
|
|
|
|
|
|
while (!kstack_end(sptr)) {
|
|
|
|
svalue = *sptr++;
|
|
|
|
if (kernel_text_address(svalue)) {
|
2006-01-08 02:00:37 -07:00
|
|
|
*addr++ = svalue;
|
2005-04-16 16:20:36 -06:00
|
|
|
size -= sizeof(unsigned long);
|
|
|
|
if (size <= sizeof(unsigned long))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
2006-01-08 02:00:37 -07:00
|
|
|
*addr++ = 0x87654321;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2012-06-13 09:24:58 -06:00
|
|
|
int size = cachep->object_size;
|
2006-02-01 04:05:42 -07:00
|
|
|
addr = &((char *)addr)[obj_offset(cachep)];
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
memset(addr, val, size);
|
2006-01-08 02:00:37 -07:00
|
|
|
*(unsigned char *)(addr + size - 1) = POISON_END;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void dump_line(char *data, int offset, int limit)
|
|
|
|
{
|
|
|
|
int i;
|
2006-09-29 02:59:51 -06:00
|
|
|
unsigned char error = 0;
|
|
|
|
int bad_count = 0;
|
|
|
|
|
2011-07-29 10:22:13 -06:00
|
|
|
printk(KERN_ERR "%03x: ", offset);
|
2006-09-29 02:59:51 -06:00
|
|
|
for (i = 0; i < limit; i++) {
|
|
|
|
if (data[offset + i] != POISON_FREE) {
|
|
|
|
error = data[offset + i];
|
|
|
|
bad_count++;
|
|
|
|
}
|
|
|
|
}
|
2011-07-29 10:22:13 -06:00
|
|
|
print_hex_dump(KERN_CONT, "", 0, 16, 1,
|
|
|
|
&data[offset], limit, 1);
|
2006-09-29 02:59:51 -06:00
|
|
|
|
|
|
|
if (bad_count == 1) {
|
|
|
|
error ^= POISON_FREE;
|
|
|
|
if (!(error & (error - 1))) {
|
|
|
|
printk(KERN_ERR "Single bit error detected. Probably "
|
|
|
|
"bad RAM.\n");
|
|
|
|
#ifdef CONFIG_X86
|
|
|
|
printk(KERN_ERR "Run memtest86+ or a similar memory "
|
|
|
|
"test tool.\n");
|
|
|
|
#else
|
|
|
|
printk(KERN_ERR "Run a memory test tool.\n");
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int i, size;
|
|
|
|
char *realobj;
|
|
|
|
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 01:22:59 -06:00
|
|
|
printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n",
|
2006-03-22 01:08:11 -07:00
|
|
|
*dbg_redzone1(cachep, objp),
|
|
|
|
*dbg_redzone2(cachep, objp));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
if (cachep->flags & SLAB_STORE_USER) {
|
2012-12-12 11:19:12 -07:00
|
|
|
printk(KERN_ERR "Last user: [<%p>](%pSR)\n",
|
|
|
|
*dbg_userword(cachep, objp),
|
|
|
|
*dbg_userword(cachep, objp));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2006-02-01 04:05:42 -07:00
|
|
|
realobj = (char *)objp + obj_offset(cachep);
|
2012-06-13 09:24:58 -06:00
|
|
|
size = cachep->object_size;
|
2006-01-08 02:00:37 -07:00
|
|
|
for (i = 0; i < size && lines; i += 16, lines--) {
|
2005-04-16 16:20:36 -06:00
|
|
|
int limit;
|
|
|
|
limit = 16;
|
2006-01-08 02:00:37 -07:00
|
|
|
if (i + limit > size)
|
|
|
|
limit = size - i;
|
2005-04-16 16:20:36 -06:00
|
|
|
dump_line(realobj, i, limit);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void check_poison_obj(struct kmem_cache *cachep, void *objp)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
char *realobj;
|
|
|
|
int size, i;
|
|
|
|
int lines = 0;
|
|
|
|
|
2006-02-01 04:05:42 -07:00
|
|
|
realobj = (char *)objp + obj_offset(cachep);
|
2012-06-13 09:24:58 -06:00
|
|
|
size = cachep->object_size;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-01-08 02:00:37 -07:00
|
|
|
for (i = 0; i < size; i++) {
|
2005-04-16 16:20:36 -06:00
|
|
|
char exp = POISON_FREE;
|
2006-01-08 02:00:37 -07:00
|
|
|
if (i == size - 1)
|
2005-04-16 16:20:36 -06:00
|
|
|
exp = POISON_END;
|
|
|
|
if (realobj[i] != exp) {
|
|
|
|
int limit;
|
|
|
|
/* Mismatch ! */
|
|
|
|
/* Print header */
|
|
|
|
if (lines == 0) {
|
2006-01-08 02:00:37 -07:00
|
|
|
printk(KERN_ERR
|
2011-11-15 16:03:52 -07:00
|
|
|
"Slab corruption (%s): %s start=%p, len=%d\n",
|
|
|
|
print_tainted(), cachep->name, realobj, size);
|
2005-04-16 16:20:36 -06:00
|
|
|
print_objinfo(cachep, objp, 0);
|
|
|
|
}
|
|
|
|
/* Hexdump the affected line */
|
2006-01-08 02:00:37 -07:00
|
|
|
i = (i / 16) * 16;
|
2005-04-16 16:20:36 -06:00
|
|
|
limit = 16;
|
2006-01-08 02:00:37 -07:00
|
|
|
if (i + limit > size)
|
|
|
|
limit = size - i;
|
2005-04-16 16:20:36 -06:00
|
|
|
dump_line(realobj, i, limit);
|
|
|
|
i += 16;
|
|
|
|
lines++;
|
|
|
|
/* Limit to 5 lines */
|
|
|
|
if (lines > 5)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (lines != 0) {
|
|
|
|
/* Print some data about the neighboring objects, if they
|
|
|
|
* exist:
|
|
|
|
*/
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page = virt_to_head_page(objp);
|
2006-03-22 01:08:10 -07:00
|
|
|
unsigned int objnr;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
objnr = obj_to_index(cachep, page, objp);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (objnr) {
|
2013-10-23 19:07:49 -06:00
|
|
|
objp = index_to_obj(cachep, page, objnr - 1);
|
2006-02-01 04:05:42 -07:00
|
|
|
realobj = (char *)objp + obj_offset(cachep);
|
2005-04-16 16:20:36 -06:00
|
|
|
printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
|
2006-01-08 02:00:37 -07:00
|
|
|
realobj, size);
|
2005-04-16 16:20:36 -06:00
|
|
|
print_objinfo(cachep, objp, 2);
|
|
|
|
}
|
2006-01-08 02:00:37 -07:00
|
|
|
if (objnr + 1 < cachep->num) {
|
2013-10-23 19:07:49 -06:00
|
|
|
objp = index_to_obj(cachep, page, objnr + 1);
|
2006-02-01 04:05:42 -07:00
|
|
|
realobj = (char *)objp + obj_offset(cachep);
|
2005-04-16 16:20:36 -06:00
|
|
|
printk(KERN_ERR "Next obj: start=%p, len=%d\n",
|
2006-01-08 02:00:37 -07:00
|
|
|
realobj, size);
|
2005-04-16 16:20:36 -06:00
|
|
|
print_objinfo(cachep, objp, 2);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2006-02-01 04:05:46 -07:00
|
|
|
#if DEBUG
|
2013-10-23 19:07:49 -06:00
|
|
|
static void slab_destroy_debugcheck(struct kmem_cache *cachep,
|
|
|
|
struct page *page)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
for (i = 0; i < cachep->num; i++) {
|
2013-10-23 19:07:49 -06:00
|
|
|
void *objp = index_to_obj(cachep, page, i);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (cachep->flags & SLAB_POISON) {
|
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
2012-06-13 09:24:57 -06:00
|
|
|
if (cachep->size % PAGE_SIZE == 0 &&
|
2006-03-22 01:08:11 -07:00
|
|
|
OFF_SLAB(cachep))
|
2006-01-08 02:00:37 -07:00
|
|
|
kernel_map_pages(virt_to_page(objp),
|
2012-06-13 09:24:57 -06:00
|
|
|
cachep->size / PAGE_SIZE, 1);
|
2005-04-16 16:20:36 -06:00
|
|
|
else
|
|
|
|
check_poison_obj(cachep, objp);
|
|
|
|
#else
|
|
|
|
check_poison_obj(cachep, objp);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
|
|
|
if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
|
|
|
|
slab_error(cachep, "start of a freed object "
|
2006-01-08 02:00:37 -07:00
|
|
|
"was overwritten");
|
2005-04-16 16:20:36 -06:00
|
|
|
if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
|
|
|
|
slab_error(cachep, "end of a freed object "
|
2006-01-08 02:00:37 -07:00
|
|
|
"was overwritten");
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
2006-02-01 04:05:46 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
#else
|
2013-10-23 19:07:49 -06:00
|
|
|
static void slab_destroy_debugcheck(struct kmem_cache *cachep,
|
|
|
|
struct page *page)
|
2006-02-01 04:05:46 -07:00
|
|
|
{
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
|
2006-03-22 01:08:14 -07:00
|
|
|
/**
|
|
|
|
* slab_destroy - destroy and release all objects in a slab
|
|
|
|
* @cachep: cache pointer being destroyed
|
2014-01-27 10:57:08 -07:00
|
|
|
* @page: page pointer being destroyed
|
2006-03-22 01:08:14 -07:00
|
|
|
*
|
2014-08-06 17:04:46 -06:00
|
|
|
* Destroy all the objs in a slab page, and release the mem back to the system.
|
|
|
|
* Before calling the slab page must have been unlinked from the cache. The
|
|
|
|
* kmem_cache_node ->list_lock is not held/needed.
|
2006-02-01 04:05:46 -07:00
|
|
|
*/
|
2013-10-23 19:07:49 -06:00
|
|
|
static void slab_destroy(struct kmem_cache *cachep, struct page *page)
|
2006-02-01 04:05:46 -07:00
|
|
|
{
|
2013-10-30 04:04:01 -06:00
|
|
|
void *freelist;
|
2006-02-01 04:05:46 -07:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
freelist = page->freelist;
|
|
|
|
slab_destroy_debugcheck(cachep, page);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
|
2013-10-23 19:07:42 -06:00
|
|
|
struct rcu_head *head;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* RCU free overloads the RCU head over the LRU.
|
|
|
|
* slab_page has been overloeaded over the LRU,
|
|
|
|
* however it is not used from now on so that
|
|
|
|
* we can use it safely.
|
|
|
|
*/
|
|
|
|
head = (void *)&page->rcu_head;
|
|
|
|
call_rcu(head, kmem_rcu_free);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
} else {
|
2013-10-23 19:07:38 -06:00
|
|
|
kmem_freepages(cachep, page);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2013-10-23 19:07:42 -06:00
|
|
|
|
|
|
|
/*
|
2013-10-23 19:07:49 -06:00
|
|
|
* From now on, we don't use freelist
|
2013-10-23 19:07:42 -06:00
|
|
|
* although actual page can be freed in rcu context
|
|
|
|
*/
|
|
|
|
if (OFF_SLAB(cachep))
|
2013-10-23 19:07:49 -06:00
|
|
|
kmem_cache_free(cachep->freelist_cache, freelist);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2014-08-06 17:04:25 -06:00
|
|
|
static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list)
|
|
|
|
{
|
|
|
|
struct page *page, *n;
|
|
|
|
|
|
|
|
list_for_each_entry_safe(page, n, list, lru) {
|
|
|
|
list_del(&page->lru);
|
|
|
|
slab_destroy(cachep, page);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-01-08 02:00:36 -07:00
|
|
|
/**
|
2006-02-01 04:05:52 -07:00
|
|
|
* calculate_slab_order - calculate size (page order) of slabs
|
|
|
|
* @cachep: pointer to the cache that is being created
|
|
|
|
* @size: size of objects to be created in this cache.
|
|
|
|
* @align: required alignment for the objects.
|
|
|
|
* @flags: slab allocation flags
|
|
|
|
*
|
|
|
|
* Also calculates the number of objects per slab.
|
2006-01-08 02:00:36 -07:00
|
|
|
*
|
|
|
|
* This could be made much more intelligent. For now, try to avoid using
|
|
|
|
* high order pages for slabs. When the gfp() functions are more friendly
|
|
|
|
* towards high-order requests, this should be changed.
|
|
|
|
*/
|
2006-03-22 01:08:11 -07:00
|
|
|
static size_t calculate_slab_order(struct kmem_cache *cachep,
|
2006-02-01 04:05:53 -07:00
|
|
|
size_t size, size_t align, unsigned long flags)
|
2006-01-08 02:00:36 -07:00
|
|
|
{
|
2006-06-02 07:44:58 -06:00
|
|
|
unsigned long offslab_limit;
|
2006-01-08 02:00:36 -07:00
|
|
|
size_t left_over = 0;
|
2006-03-06 18:44:43 -07:00
|
|
|
int gfporder;
|
2006-01-08 02:00:36 -07:00
|
|
|
|
2007-05-16 23:11:01 -06:00
|
|
|
for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
|
2006-01-08 02:00:36 -07:00
|
|
|
unsigned int num;
|
|
|
|
size_t remainder;
|
|
|
|
|
2006-03-06 18:44:43 -07:00
|
|
|
cache_estimate(gfporder, size, align, flags, &remainder, &num);
|
2006-01-08 02:00:36 -07:00
|
|
|
if (!num)
|
|
|
|
continue;
|
2006-03-06 18:44:43 -07:00
|
|
|
|
2013-12-02 01:49:41 -07:00
|
|
|
/* Can't handle number of objects more than SLAB_OBJ_MAX_NUM */
|
|
|
|
if (num > SLAB_OBJ_MAX_NUM)
|
|
|
|
break;
|
|
|
|
|
2006-06-02 07:44:58 -06:00
|
|
|
if (flags & CFLGS_OFF_SLAB) {
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
size_t freelist_size_per_obj = sizeof(freelist_idx_t);
|
2006-06-02 07:44:58 -06:00
|
|
|
/*
|
|
|
|
* Max number of objs-per-slab for caches which
|
|
|
|
* use off-slab slabs. Needed to avoid a possible
|
|
|
|
* looping condition in cache_grow().
|
|
|
|
*/
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
if (IS_ENABLED(CONFIG_DEBUG_SLAB_LEAK))
|
|
|
|
freelist_size_per_obj += sizeof(char);
|
2013-10-23 19:07:49 -06:00
|
|
|
offslab_limit = size;
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
offslab_limit /= freelist_size_per_obj;
|
2006-06-02 07:44:58 -06:00
|
|
|
|
|
|
|
if (num > offslab_limit)
|
|
|
|
break;
|
|
|
|
}
|
2006-01-08 02:00:36 -07:00
|
|
|
|
2006-03-06 18:44:43 -07:00
|
|
|
/* Found something acceptable - save it away */
|
2006-01-08 02:00:36 -07:00
|
|
|
cachep->num = num;
|
2006-03-06 18:44:43 -07:00
|
|
|
cachep->gfporder = gfporder;
|
2006-01-08 02:00:36 -07:00
|
|
|
left_over = remainder;
|
|
|
|
|
2006-03-08 11:33:05 -07:00
|
|
|
/*
|
|
|
|
* A VFS-reclaimable slab tends to have most allocations
|
|
|
|
* as GFP_NOFS and we really don't want to have to be allocating
|
|
|
|
* higher-order pages when we are unable to shrink dcache.
|
|
|
|
*/
|
|
|
|
if (flags & SLAB_RECLAIM_ACCOUNT)
|
|
|
|
break;
|
|
|
|
|
2006-01-08 02:00:36 -07:00
|
|
|
/*
|
|
|
|
* Large number of objects is good, but very large slabs are
|
|
|
|
* currently bad for the gfp()s.
|
|
|
|
*/
|
2011-10-18 23:09:24 -06:00
|
|
|
if (gfporder >= slab_max_order)
|
2006-01-08 02:00:36 -07:00
|
|
|
break;
|
|
|
|
|
2006-03-06 18:44:43 -07:00
|
|
|
/*
|
|
|
|
* Acceptable internal fragmentation?
|
|
|
|
*/
|
2006-03-22 01:08:11 -07:00
|
|
|
if (left_over * 8 <= (PAGE_SIZE << gfporder))
|
2006-01-08 02:00:36 -07:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
return left_over;
|
|
|
|
}
|
|
|
|
|
2014-10-09 16:26:27 -06:00
|
|
|
static struct array_cache __percpu *alloc_kmem_cache_cpus(
|
|
|
|
struct kmem_cache *cachep, int entries, int batchcount)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
size_t size;
|
|
|
|
struct array_cache __percpu *cpu_cache;
|
|
|
|
|
|
|
|
size = sizeof(void *) * entries + sizeof(struct array_cache);
|
2014-10-13 16:51:01 -06:00
|
|
|
cpu_cache = __alloc_percpu(size, sizeof(void *));
|
2014-10-09 16:26:27 -06:00
|
|
|
|
|
|
|
if (!cpu_cache)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
init_arraycache(per_cpu_ptr(cpu_cache, cpu),
|
|
|
|
entries, batchcount);
|
|
|
|
}
|
|
|
|
|
|
|
|
return cpu_cache;
|
|
|
|
}
|
|
|
|
|
2009-06-10 10:40:04 -06:00
|
|
|
static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
|
2006-03-22 01:08:11 -07:00
|
|
|
{
|
2012-07-06 14:25:11 -06:00
|
|
|
if (slab_state >= FULL)
|
2009-06-10 10:40:04 -06:00
|
|
|
return enable_cpucache(cachep, gfp);
|
2006-09-26 00:31:38 -06:00
|
|
|
|
2014-10-09 16:26:27 -06:00
|
|
|
cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1);
|
|
|
|
if (!cachep->cpu_cache)
|
|
|
|
return 1;
|
|
|
|
|
2012-07-06 14:25:11 -06:00
|
|
|
if (slab_state == DOWN) {
|
2014-10-09 16:26:27 -06:00
|
|
|
/* Creation of first cache (kmem_cache). */
|
|
|
|
set_up_node(kmem_cache, CACHE_CACHE);
|
2012-11-28 09:23:09 -07:00
|
|
|
} else if (slab_state == PARTIAL) {
|
2014-10-09 16:26:27 -06:00
|
|
|
/* For kmem_cache_node */
|
|
|
|
set_up_node(cachep, SIZE_NODE);
|
2006-03-22 01:08:11 -07:00
|
|
|
} else {
|
2014-10-09 16:26:27 -06:00
|
|
|
int node;
|
2006-03-22 01:08:11 -07:00
|
|
|
|
2014-10-09 16:26:27 -06:00
|
|
|
for_each_online_node(node) {
|
|
|
|
cachep->node[node] = kmalloc_node(
|
|
|
|
sizeof(struct kmem_cache_node), gfp, node);
|
|
|
|
BUG_ON(!cachep->node[node]);
|
|
|
|
kmem_cache_node_init(cachep->node[node]);
|
2006-03-22 01:08:11 -07:00
|
|
|
}
|
|
|
|
}
|
2014-10-09 16:26:27 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
cachep->node[numa_mem_id()]->next_reap =
|
2014-03-30 03:02:20 -06:00
|
|
|
jiffies + REAPTIMEOUT_NODE +
|
|
|
|
((unsigned long)cachep) % REAPTIMEOUT_NODE;
|
2006-03-22 01:08:11 -07:00
|
|
|
|
|
|
|
cpu_cache_get(cachep)->avail = 0;
|
|
|
|
cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
|
|
|
|
cpu_cache_get(cachep)->batchcount = 1;
|
|
|
|
cpu_cache_get(cachep)->touched = 0;
|
|
|
|
cachep->batchcount = 1;
|
|
|
|
cachep->limit = BOOT_CPUCACHE_ENTRIES;
|
2006-09-26 00:31:38 -06:00
|
|
|
return 0;
|
2006-03-22 01:08:11 -07:00
|
|
|
}
|
|
|
|
|
2014-10-09 16:26:24 -06:00
|
|
|
unsigned long kmem_cache_flags(unsigned long object_size,
|
|
|
|
unsigned long flags, const char *name,
|
|
|
|
void (*ctor)(void *))
|
|
|
|
{
|
|
|
|
return flags;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct kmem_cache *
|
|
|
|
__kmem_cache_alias(const char *name, size_t size, size_t align,
|
|
|
|
unsigned long flags, void (*ctor)(void *))
|
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
|
|
|
|
|
|
|
cachep = find_mergeable(size, align, flags, name, ctor);
|
|
|
|
if (cachep) {
|
|
|
|
cachep->refcount++;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Adjust the object sizes so that we clear
|
|
|
|
* the complete object on kzalloc.
|
|
|
|
*/
|
|
|
|
cachep->object_size = max_t(int, cachep->object_size, size);
|
|
|
|
}
|
|
|
|
return cachep;
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/**
|
2012-07-06 14:25:10 -06:00
|
|
|
* __kmem_cache_create - Create a cache.
|
2012-11-06 18:10:10 -07:00
|
|
|
* @cachep: cache management descriptor
|
2005-04-16 16:20:36 -06:00
|
|
|
* @flags: SLAB flags
|
|
|
|
*
|
|
|
|
* Returns a ptr to the cache on success, NULL on failure.
|
|
|
|
* Cannot be called within a int, but can be interrupted.
|
2007-07-19 19:11:58 -06:00
|
|
|
* The @ctor is run when new pages are allocated by the cache.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* The flags are
|
|
|
|
*
|
|
|
|
* %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
|
|
|
|
* to catch references to uninitialised memory.
|
|
|
|
*
|
|
|
|
* %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
|
|
|
|
* for buffer overruns.
|
|
|
|
*
|
|
|
|
* %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
|
|
|
|
* cacheline. This can be beneficial if you're counting cycles as closely
|
|
|
|
* as davem.
|
|
|
|
*/
|
2012-09-04 18:20:34 -06:00
|
|
|
int
|
2012-09-04 17:18:33 -06:00
|
|
|
__kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2014-09-25 17:05:20 -06:00
|
|
|
size_t left_over, freelist_size;
|
|
|
|
size_t ralign = BYTES_PER_WORD;
|
2009-06-10 10:40:04 -06:00
|
|
|
gfp_t gfp;
|
2012-09-04 18:20:34 -06:00
|
|
|
int err;
|
2012-09-04 17:18:33 -06:00
|
|
|
size_t size = cachep->size;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
#if FORCED_DEBUG
|
|
|
|
/*
|
|
|
|
* Enable redzoning and last user accounting, except for caches with
|
|
|
|
* large objects, if the increased size would increase the object size
|
|
|
|
* above the next power of two: caches with object sizes just above a
|
|
|
|
* power of two have a significant amount of internal fragmentation.
|
|
|
|
*/
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-04 19:26:44 -06:00
|
|
|
if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
|
|
|
|
2 * sizeof(unsigned long long)))
|
2006-01-08 02:00:37 -07:00
|
|
|
flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!(flags & SLAB_DESTROY_BY_RCU))
|
|
|
|
flags |= SLAB_POISON;
|
|
|
|
#endif
|
|
|
|
if (flags & SLAB_DESTROY_BY_RCU)
|
|
|
|
BUG_ON(flags & SLAB_POISON);
|
|
|
|
#endif
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* Check that size is in terms of words. This is needed to avoid
|
2005-04-16 16:20:36 -06:00
|
|
|
* unaligned accesses for some archs when redzoning is used, and makes
|
|
|
|
* sure any on-slab bufctl's are also correctly aligned.
|
|
|
|
*/
|
2006-01-08 02:00:37 -07:00
|
|
|
if (size & (BYTES_PER_WORD - 1)) {
|
|
|
|
size += (BYTES_PER_WORD - 1);
|
|
|
|
size &= ~(BYTES_PER_WORD - 1);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-04 19:26:44 -06:00
|
|
|
if (flags & SLAB_RED_ZONE) {
|
|
|
|
ralign = REDZONE_ALIGN;
|
|
|
|
/* If redzoning, ensure that the second redzone is suitably
|
|
|
|
* aligned, by adjusting the object size accordingly. */
|
|
|
|
size += REDZONE_ALIGN - 1;
|
|
|
|
size &= ~(REDZONE_ALIGN - 1);
|
|
|
|
}
|
2006-09-26 00:31:25 -06:00
|
|
|
|
2006-12-06 21:32:11 -07:00
|
|
|
/* 3) caller mandated alignment */
|
2012-09-04 17:18:33 -06:00
|
|
|
if (ralign < cachep->align) {
|
|
|
|
ralign = cachep->align;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2011-02-14 08:46:21 -07:00
|
|
|
/* disable debug if necessary */
|
|
|
|
if (ralign > __alignof__(unsigned long long))
|
2006-12-06 21:32:11 -07:00
|
|
|
flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
2006-09-26 00:31:25 -06:00
|
|
|
* 4) Store it.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2012-09-04 17:18:33 -06:00
|
|
|
cachep->align = ralign;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-06-10 10:40:04 -06:00
|
|
|
if (slab_is_available())
|
|
|
|
gfp = GFP_KERNEL;
|
|
|
|
else
|
|
|
|
gfp = GFP_NOWAIT;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#if DEBUG
|
|
|
|
|
2006-09-26 00:31:25 -06:00
|
|
|
/*
|
|
|
|
* Both debugging options require word-alignment which is calculated
|
|
|
|
* into align above.
|
|
|
|
*/
|
2005-04-16 16:20:36 -06:00
|
|
|
if (flags & SLAB_RED_ZONE) {
|
|
|
|
/* add space for red zone words */
|
2011-02-14 08:46:21 -07:00
|
|
|
cachep->obj_offset += sizeof(unsigned long long);
|
|
|
|
size += 2 * sizeof(unsigned long long);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
if (flags & SLAB_STORE_USER) {
|
2006-09-26 00:31:25 -06:00
|
|
|
/* user store requires one word storage behind the end of
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-04 19:26:44 -06:00
|
|
|
* the real object. But if the second red zone needs to be
|
|
|
|
* aligned to 64 bits, we must allow that much space.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
Fix slab redzone alignment
Commit b46b8f19c9cd435ecac4d9d12b39d78c137ecd66 fixed a couple of bugs
by switching the redzone to 64 bits. Unfortunately, it neglected to
ensure that the _second_ redzone, after the slab object, is aligned
correctly. This caused illegal instruction faults on sparc32, which for
some reason not entirely clear to me are not trapped and fixed up.
Two things need to be done to fix this:
- increase the object size, rounding up to alignof(long long) so
that the second redzone can be aligned correctly.
- If SLAB_STORE_USER is set but alignof(long long)==8, allow a
full 64 bits of space for the user word at the end of the buffer,
even though we may not _use_ the whole 64 bits.
This patch should be a no-op on any 64-bit architecture or any 32-bit
architecture where alignof(long long) == 4. Of the others, it's tested
on ppc32 by myself and a very similar patch was tested on sparc32 by
Mark Fortescue, who reported the new problem.
Also, fix the conditions for FORCED_DEBUG, which hadn't been adjusted to
the new sizes. Again noticed by Mark.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-04 19:26:44 -06:00
|
|
|
if (flags & SLAB_RED_ZONE)
|
|
|
|
size += REDZONE_ALIGN;
|
|
|
|
else
|
|
|
|
size += BYTES_PER_WORD;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
|
2013-01-10 12:14:19 -07:00
|
|
|
if (size >= kmalloc_size(INDEX_NODE + 1)
|
2012-09-30 02:28:25 -06:00
|
|
|
&& cachep->object_size > cache_line_size()
|
|
|
|
&& ALIGN(size, cachep->align) < PAGE_SIZE) {
|
|
|
|
cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align);
|
2005-04-16 16:20:36 -06:00
|
|
|
size = PAGE_SIZE;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
2006-06-23 03:03:46 -06:00
|
|
|
/*
|
|
|
|
* Determine if the slab management is 'on' or 'off' slab.
|
|
|
|
* (bootstrapping cannot cope with offslab caches so don't do
|
2009-10-28 07:33:08 -06:00
|
|
|
* it too early on. Always use on-slab management when
|
|
|
|
* SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak)
|
2006-06-23 03:03:46 -06:00
|
|
|
*/
|
2013-12-02 01:49:43 -07:00
|
|
|
if ((size >= (PAGE_SIZE >> 5)) && !slab_early_init &&
|
2009-10-28 07:33:08 -06:00
|
|
|
!(flags & SLAB_NOLEAKTRACE))
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* Size is large, assume best to place the slab management obj
|
|
|
|
* off-slab (should allow better packing of objs).
|
|
|
|
*/
|
|
|
|
flags |= CFLGS_OFF_SLAB;
|
|
|
|
|
2012-09-04 17:18:33 -06:00
|
|
|
size = ALIGN(size, cachep->align);
|
2013-12-02 01:49:41 -07:00
|
|
|
/*
|
|
|
|
* We should restrict the number of objects in a slab to implement
|
|
|
|
* byte sized index. Refer comment on SLAB_OBJ_MIN_SIZE definition.
|
|
|
|
*/
|
|
|
|
if (FREELIST_BYTE_INDEX && size < SLAB_OBJ_MIN_SIZE)
|
|
|
|
size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-09-04 17:18:33 -06:00
|
|
|
left_over = calculate_slab_order(cachep, size, cachep->align, flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-09-04 17:18:33 -06:00
|
|
|
if (!cachep->num)
|
2012-09-04 18:20:34 -06:00
|
|
|
return -E2BIG;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
freelist_size = calculate_freelist_size(cachep->num, cachep->align);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If the slab has been placed off-slab, and we have enough space then
|
|
|
|
* move it on-slab. This is at the expense of any extra colouring.
|
|
|
|
*/
|
2013-10-23 19:07:49 -06:00
|
|
|
if (flags & CFLGS_OFF_SLAB && left_over >= freelist_size) {
|
2005-04-16 16:20:36 -06:00
|
|
|
flags &= ~CFLGS_OFF_SLAB;
|
2013-10-23 19:07:49 -06:00
|
|
|
left_over -= freelist_size;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
if (flags & CFLGS_OFF_SLAB) {
|
|
|
|
/* really off slab. No need for manual alignment */
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
freelist_size = calculate_freelist_size(cachep->num, 0);
|
2009-05-21 13:28:22 -06:00
|
|
|
|
|
|
|
#ifdef CONFIG_PAGE_POISONING
|
|
|
|
/* If we're going to use the generic kernel_map_pages()
|
|
|
|
* poisoning, then it's going to smash the contents of
|
|
|
|
* the redzone and userword anyhow, so switch them off.
|
|
|
|
*/
|
|
|
|
if (size % PAGE_SIZE == 0 && flags & SLAB_POISON)
|
|
|
|
flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
|
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
cachep->colour_off = cache_line_size();
|
|
|
|
/* Offset must be a multiple of the alignment. */
|
2012-09-04 17:18:33 -06:00
|
|
|
if (cachep->colour_off < cachep->align)
|
|
|
|
cachep->colour_off = cachep->align;
|
2006-01-08 02:00:37 -07:00
|
|
|
cachep->colour = left_over / cachep->colour_off;
|
2013-10-23 19:07:49 -06:00
|
|
|
cachep->freelist_size = freelist_size;
|
2005-04-16 16:20:36 -06:00
|
|
|
cachep->flags = flags;
|
2013-10-23 19:07:44 -06:00
|
|
|
cachep->allocflags = __GFP_COMP;
|
2007-02-10 02:43:10 -07:00
|
|
|
if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
|
2012-06-14 06:17:21 -06:00
|
|
|
cachep->allocflags |= GFP_DMA;
|
2012-06-13 09:24:57 -06:00
|
|
|
cachep->size = size;
|
2006-12-13 01:34:27 -07:00
|
|
|
cachep->reciprocal_buffer_size = reciprocal_value(size);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-09-26 00:31:34 -06:00
|
|
|
if (flags & CFLGS_OFF_SLAB) {
|
2013-10-23 19:07:49 -06:00
|
|
|
cachep->freelist_cache = kmalloc_slab(freelist_size, 0u);
|
2006-09-26 00:31:34 -06:00
|
|
|
/*
|
2014-03-30 03:02:20 -06:00
|
|
|
* This is a possibility for one of the kmalloc_{dma,}_caches.
|
2006-09-26 00:31:34 -06:00
|
|
|
* But since we go off slab only for object size greater than
|
2014-03-30 03:02:20 -06:00
|
|
|
* PAGE_SIZE/8, and kmalloc_{dma,}_caches get created
|
|
|
|
* in ascending order,this should not happen at all.
|
2006-09-26 00:31:34 -06:00
|
|
|
* But leave a BUG_ON for some lucky dude.
|
|
|
|
*/
|
2013-10-23 19:07:49 -06:00
|
|
|
BUG_ON(ZERO_OR_NULL_PTR(cachep->freelist_cache));
|
2006-09-26 00:31:34 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-09-04 18:20:34 -06:00
|
|
|
err = setup_cpu_cache(cachep, gfp);
|
|
|
|
if (err) {
|
2012-09-04 17:38:33 -06:00
|
|
|
__kmem_cache_shutdown(cachep);
|
2012-09-04 18:20:34 -06:00
|
|
|
return err;
|
2006-09-26 00:31:38 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-09-04 18:20:34 -06:00
|
|
|
return 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
static void check_irq_off(void)
|
|
|
|
{
|
|
|
|
BUG_ON(!irqs_disabled());
|
|
|
|
}
|
|
|
|
|
|
|
|
static void check_irq_on(void)
|
|
|
|
{
|
|
|
|
BUG_ON(irqs_disabled());
|
|
|
|
}
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void check_spinlock_acquired(struct kmem_cache *cachep)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
check_irq_off();
|
2014-08-06 17:04:11 -06:00
|
|
|
assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
}
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
check_irq_off();
|
2014-08-06 17:04:11 -06:00
|
|
|
assert_spin_locked(&get_node(cachep, node)->list_lock);
|
2005-09-09 14:03:32 -06:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#else
|
|
|
|
#define check_irq_off() do { } while(0)
|
|
|
|
#define check_irq_on() do { } while(0)
|
|
|
|
#define check_spinlock_acquired(x) do { } while(0)
|
2005-09-09 14:03:32 -06:00
|
|
|
#define check_spinlock_acquired_node(x, y) do { } while(0)
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
|
2006-03-22 01:09:06 -07:00
|
|
|
struct array_cache *ac,
|
|
|
|
int force, int node);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static void do_drain(void *arg)
|
|
|
|
{
|
2006-03-22 01:08:11 -07:00
|
|
|
struct kmem_cache *cachep = arg;
|
2005-04-16 16:20:36 -06:00
|
|
|
struct array_cache *ac;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
int node = numa_mem_id();
|
2014-08-06 17:04:11 -06:00
|
|
|
struct kmem_cache_node *n;
|
2014-08-06 17:04:25 -06:00
|
|
|
LIST_HEAD(list);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
check_irq_off();
|
2006-02-01 04:05:49 -07:00
|
|
|
ac = cpu_cache_get(cachep);
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
|
|
|
spin_lock(&n->list_lock);
|
2014-08-06 17:04:25 -06:00
|
|
|
free_block(cachep, ac->entry, ac->avail, node, &list);
|
2014-08-06 17:04:11 -06:00
|
|
|
spin_unlock(&n->list_lock);
|
2014-08-06 17:04:25 -06:00
|
|
|
slabs_destroy(cachep, &list);
|
2005-04-16 16:20:36 -06:00
|
|
|
ac->avail = 0;
|
|
|
|
}
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void drain_cpu_caches(struct kmem_cache *cachep)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2005-09-09 14:03:32 -06:00
|
|
|
int node;
|
|
|
|
|
2008-05-09 01:39:44 -06:00
|
|
|
on_each_cpu(do_drain, cachep, 1);
|
2005-04-16 16:20:36 -06:00
|
|
|
check_irq_on();
|
2014-08-06 17:04:11 -06:00
|
|
|
for_each_kmem_cache_node(cachep, node, n)
|
|
|
|
if (n->alien)
|
2013-01-10 12:14:19 -07:00
|
|
|
drain_alien_cache(cachep, n->alien);
|
2006-05-15 12:41:00 -06:00
|
|
|
|
2014-08-06 17:04:11 -06:00
|
|
|
for_each_kmem_cache_node(cachep, node, n)
|
|
|
|
drain_array(cachep, n, n->shared, 1, node);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-06-30 02:55:45 -06:00
|
|
|
/*
|
|
|
|
* Remove slabs from the list of free slabs.
|
|
|
|
* Specify the number of slabs to drain in tofree.
|
|
|
|
*
|
|
|
|
* Returns the actual number of slabs released.
|
|
|
|
*/
|
|
|
|
static int drain_freelist(struct kmem_cache *cache,
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n, int tofree)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-06-30 02:55:45 -06:00
|
|
|
struct list_head *p;
|
|
|
|
int nr_freed;
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-06-30 02:55:45 -06:00
|
|
|
nr_freed = 0;
|
2013-01-10 12:14:19 -07:00
|
|
|
while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock_irq(&n->list_lock);
|
|
|
|
p = n->slabs_free.prev;
|
|
|
|
if (p == &n->slabs_free) {
|
|
|
|
spin_unlock_irq(&n->list_lock);
|
2006-06-30 02:55:45 -06:00
|
|
|
goto out;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
page = list_entry(p, struct page, lru);
|
2005-04-16 16:20:36 -06:00
|
|
|
#if DEBUG
|
2013-10-23 19:07:49 -06:00
|
|
|
BUG_ON(page->active);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
2013-10-23 19:07:49 -06:00
|
|
|
list_del(&page->lru);
|
2006-06-30 02:55:45 -06:00
|
|
|
/*
|
|
|
|
* Safe to drop the lock. The slab is no longer linked
|
|
|
|
* to the cache.
|
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
n->free_objects -= cache->num;
|
|
|
|
spin_unlock_irq(&n->list_lock);
|
2013-10-23 19:07:49 -06:00
|
|
|
slab_destroy(cache, page);
|
2006-06-30 02:55:45 -06:00
|
|
|
nr_freed++;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2006-06-30 02:55:45 -06:00
|
|
|
out:
|
|
|
|
return nr_freed;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
slab: get_online_mems for kmem_cache_{create,destroy,shrink}
When we create a sl[au]b cache, we allocate kmem_cache_node structures
for each online NUMA node. To handle nodes taken online/offline, we
register memory hotplug notifier and allocate/free kmem_cache_node
corresponding to the node that changes its state for each kmem cache.
To synchronize between the two paths we hold the slab_mutex during both
the cache creationg/destruction path and while tuning per-node parts of
kmem caches in memory hotplug handler, but that's not quite right,
because it does not guarantee that a newly created cache will have all
kmem_cache_nodes initialized in case it races with memory hotplug. For
instance, in case of slub:
CPU0 CPU1
---- ----
kmem_cache_create: online_pages:
__kmem_cache_create: slab_memory_callback:
slab_mem_going_online_callback:
lock slab_mutex
for each slab_caches list entry
allocate kmem_cache node
unlock slab_mutex
lock slab_mutex
init_kmem_cache_nodes:
for_each_node_state(node, N_NORMAL_MEMORY)
allocate kmem_cache node
add kmem_cache to slab_caches list
unlock slab_mutex
online_pages (continued):
node_states_set_node
As a result we'll get a kmem cache with not all kmem_cache_nodes
allocated.
To avoid issues like that we should hold get/put_online_mems() during
the whole kmem cache creation/destruction/shrink paths, just like we
deal with cpu hotplug. This patch does the trick.
Note, that after it's applied, there is no need in taking the slab_mutex
for kmem_cache_shrink any more, so it is removed from there.
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Toshi Kani <toshi.kani@hp.com>
Cc: Xishi Qiu <qiuxishi@huawei.com>
Cc: Jiang Liu <liuj97@gmail.com>
Cc: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 17:07:20 -06:00
|
|
|
int __kmem_cache_shrink(struct kmem_cache *cachep)
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
2014-08-06 17:04:11 -06:00
|
|
|
int ret = 0;
|
|
|
|
int node;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2005-09-09 14:03:32 -06:00
|
|
|
|
|
|
|
drain_cpu_caches(cachep);
|
|
|
|
|
|
|
|
check_irq_on();
|
2014-08-06 17:04:11 -06:00
|
|
|
for_each_kmem_cache_node(cachep, node, n) {
|
2013-07-03 18:33:22 -06:00
|
|
|
drain_freelist(cachep, n, slabs_tofree(cachep, n));
|
2006-06-30 02:55:45 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
ret += !list_empty(&n->slabs_full) ||
|
|
|
|
!list_empty(&n->slabs_partial);
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
|
|
|
return (ret ? 1 : 0);
|
|
|
|
}
|
|
|
|
|
2012-09-04 17:18:33 -06:00
|
|
|
int __kmem_cache_shutdown(struct kmem_cache *cachep)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2012-09-04 17:38:33 -06:00
|
|
|
int i;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
slab: get_online_mems for kmem_cache_{create,destroy,shrink}
When we create a sl[au]b cache, we allocate kmem_cache_node structures
for each online NUMA node. To handle nodes taken online/offline, we
register memory hotplug notifier and allocate/free kmem_cache_node
corresponding to the node that changes its state for each kmem cache.
To synchronize between the two paths we hold the slab_mutex during both
the cache creationg/destruction path and while tuning per-node parts of
kmem caches in memory hotplug handler, but that's not quite right,
because it does not guarantee that a newly created cache will have all
kmem_cache_nodes initialized in case it races with memory hotplug. For
instance, in case of slub:
CPU0 CPU1
---- ----
kmem_cache_create: online_pages:
__kmem_cache_create: slab_memory_callback:
slab_mem_going_online_callback:
lock slab_mutex
for each slab_caches list entry
allocate kmem_cache node
unlock slab_mutex
lock slab_mutex
init_kmem_cache_nodes:
for_each_node_state(node, N_NORMAL_MEMORY)
allocate kmem_cache node
add kmem_cache to slab_caches list
unlock slab_mutex
online_pages (continued):
node_states_set_node
As a result we'll get a kmem cache with not all kmem_cache_nodes
allocated.
To avoid issues like that we should hold get/put_online_mems() during
the whole kmem cache creation/destruction/shrink paths, just like we
deal with cpu hotplug. This patch does the trick.
Note, that after it's applied, there is no need in taking the slab_mutex
for kmem_cache_shrink any more, so it is removed from there.
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Cc: Toshi Kani <toshi.kani@hp.com>
Cc: Xishi Qiu <qiuxishi@huawei.com>
Cc: Jiang Liu <liuj97@gmail.com>
Cc: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-04 17:07:20 -06:00
|
|
|
int rc = __kmem_cache_shrink(cachep);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-09-04 17:38:33 -06:00
|
|
|
if (rc)
|
|
|
|
return rc;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2014-10-09 16:26:27 -06:00
|
|
|
free_percpu(cachep->cpu_cache);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
/* NUMA: free the node structures */
|
2014-08-06 17:04:11 -06:00
|
|
|
for_each_kmem_cache_node(cachep, i, n) {
|
|
|
|
kfree(n->shared);
|
|
|
|
free_alien_cache(n->alien);
|
|
|
|
kfree(n);
|
|
|
|
cachep->node[i] = NULL;
|
2012-09-04 17:38:33 -06:00
|
|
|
}
|
|
|
|
return 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-09-26 00:31:34 -06:00
|
|
|
/*
|
|
|
|
* Get the memory for a slab management obj.
|
2014-03-30 03:02:20 -06:00
|
|
|
*
|
|
|
|
* For a slab cache when the slab descriptor is off-slab, the
|
|
|
|
* slab descriptor can't come from the same cache which is being created,
|
|
|
|
* Because if it is the case, that means we defer the creation of
|
|
|
|
* the kmalloc_{dma,}_cache of size sizeof(slab descriptor) to this point.
|
|
|
|
* And we eventually call down to __kmem_cache_create(), which
|
|
|
|
* in turn looks up in the kmalloc_{dma,}_caches for the disired-size one.
|
|
|
|
* This is a "chicken-and-egg" problem.
|
|
|
|
*
|
|
|
|
* So the off-slab slab descriptor shall come from the kmalloc_{dma,}_caches,
|
|
|
|
* which are all initialized during kmem_cache_init().
|
2006-09-26 00:31:34 -06:00
|
|
|
*/
|
2013-10-30 04:04:01 -06:00
|
|
|
static void *alloc_slabmgmt(struct kmem_cache *cachep,
|
2013-10-23 19:07:38 -06:00
|
|
|
struct page *page, int colour_off,
|
|
|
|
gfp_t local_flags, int nodeid)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2013-10-30 04:04:01 -06:00
|
|
|
void *freelist;
|
2013-10-23 19:07:38 -06:00
|
|
|
void *addr = page_address(page);
|
2006-01-08 02:00:37 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (OFF_SLAB(cachep)) {
|
|
|
|
/* Slab management obj is off-slab. */
|
2013-10-23 19:07:49 -06:00
|
|
|
freelist = kmem_cache_alloc_node(cachep->freelist_cache,
|
2008-11-26 01:01:31 -07:00
|
|
|
local_flags, nodeid);
|
2013-10-23 19:07:49 -06:00
|
|
|
if (!freelist)
|
2005-04-16 16:20:36 -06:00
|
|
|
return NULL;
|
|
|
|
} else {
|
2013-10-23 19:07:49 -06:00
|
|
|
freelist = addr + colour_off;
|
|
|
|
colour_off += cachep->freelist_size;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2013-10-23 19:07:49 -06:00
|
|
|
page->active = 0;
|
|
|
|
page->s_mem = addr + colour_off;
|
|
|
|
return freelist;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2014-04-18 01:24:09 -06:00
|
|
|
static inline freelist_idx_t get_free_obj(struct page *page, unsigned int idx)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
slab: introduce byte sized index for the freelist of a slab
Currently, the freelist of a slab consist of unsigned int sized indexes.
Since most of slabs have less number of objects than 256, large sized
indexes is needless. For example, consider the minimum kmalloc slab. It's
object size is 32 byte and it would consist of one page, so 256 indexes
through byte sized index are enough to contain all possible indexes.
There can be some slabs whose object size is 8 byte. We cannot handle
this case with byte sized index, so we need to restrict minimum
object size. Since these slabs are not major, wasted memory from these
slabs would be negligible.
Some architectures' page size isn't 4096 bytes and rather larger than
4096 bytes (One example is 64KB page size on PPC or IA64) so that
byte sized index doesn't fit to them. In this case, we will use
two bytes sized index.
Below is some number for this patch.
* Before *
kmalloc-512 525 640 512 8 1 : tunables 54 27 0 : slabdata 80 80 0
kmalloc-256 210 210 256 15 1 : tunables 120 60 0 : slabdata 14 14 0
kmalloc-192 1016 1040 192 20 1 : tunables 120 60 0 : slabdata 52 52 0
kmalloc-96 560 620 128 31 1 : tunables 120 60 0 : slabdata 20 20 0
kmalloc-64 2148 2280 64 60 1 : tunables 120 60 0 : slabdata 38 38 0
kmalloc-128 647 682 128 31 1 : tunables 120 60 0 : slabdata 22 22 0
kmalloc-32 11360 11413 32 113 1 : tunables 120 60 0 : slabdata 101 101 0
kmem_cache 197 200 192 20 1 : tunables 120 60 0 : slabdata 10 10 0
* After *
kmalloc-512 521 648 512 8 1 : tunables 54 27 0 : slabdata 81 81 0
kmalloc-256 208 208 256 16 1 : tunables 120 60 0 : slabdata 13 13 0
kmalloc-192 1029 1029 192 21 1 : tunables 120 60 0 : slabdata 49 49 0
kmalloc-96 529 589 128 31 1 : tunables 120 60 0 : slabdata 19 19 0
kmalloc-64 2142 2142 64 63 1 : tunables 120 60 0 : slabdata 34 34 0
kmalloc-128 660 682 128 31 1 : tunables 120 60 0 : slabdata 22 22 0
kmalloc-32 11716 11780 32 124 1 : tunables 120 60 0 : slabdata 95 95 0
kmem_cache 197 210 192 21 1 : tunables 120 60 0 : slabdata 10 10 0
kmem_caches consisting of objects less than or equal to 256 byte have
one or more objects than before. In the case of kmalloc-32, we have 11 more
objects, so 352 bytes (11 * 32) are saved and this is roughly 9% saving of
memory. Of couse, this percentage decreases as the number of objects
in a slab decreases.
Here are the performance results on my 4 cpus machine.
* Before *
Performance counter stats for 'perf bench sched messaging -g 50 -l 1000' (10 runs):
229,945,138 cache-misses ( +- 0.23% )
11.627897174 seconds time elapsed ( +- 0.14% )
* After *
Performance counter stats for 'perf bench sched messaging -g 50 -l 1000' (10 runs):
218,640,472 cache-misses ( +- 0.42% )
11.504999837 seconds time elapsed ( +- 0.21% )
cache-misses are reduced by this patchset, roughly 5%.
And elapsed times are improved by 1%.
Acked-by: Christoph Lameter <cl@linux.com>
Acked-by: David Rientjes <rientjes@google.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>
2013-12-02 01:49:42 -07:00
|
|
|
return ((freelist_idx_t *)page->freelist)[idx];
|
2013-12-02 01:49:40 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void set_free_obj(struct page *page,
|
2014-04-18 01:24:09 -06:00
|
|
|
unsigned int idx, freelist_idx_t val)
|
2013-12-02 01:49:40 -07:00
|
|
|
{
|
slab: introduce byte sized index for the freelist of a slab
Currently, the freelist of a slab consist of unsigned int sized indexes.
Since most of slabs have less number of objects than 256, large sized
indexes is needless. For example, consider the minimum kmalloc slab. It's
object size is 32 byte and it would consist of one page, so 256 indexes
through byte sized index are enough to contain all possible indexes.
There can be some slabs whose object size is 8 byte. We cannot handle
this case with byte sized index, so we need to restrict minimum
object size. Since these slabs are not major, wasted memory from these
slabs would be negligible.
Some architectures' page size isn't 4096 bytes and rather larger than
4096 bytes (One example is 64KB page size on PPC or IA64) so that
byte sized index doesn't fit to them. In this case, we will use
two bytes sized index.
Below is some number for this patch.
* Before *
kmalloc-512 525 640 512 8 1 : tunables 54 27 0 : slabdata 80 80 0
kmalloc-256 210 210 256 15 1 : tunables 120 60 0 : slabdata 14 14 0
kmalloc-192 1016 1040 192 20 1 : tunables 120 60 0 : slabdata 52 52 0
kmalloc-96 560 620 128 31 1 : tunables 120 60 0 : slabdata 20 20 0
kmalloc-64 2148 2280 64 60 1 : tunables 120 60 0 : slabdata 38 38 0
kmalloc-128 647 682 128 31 1 : tunables 120 60 0 : slabdata 22 22 0
kmalloc-32 11360 11413 32 113 1 : tunables 120 60 0 : slabdata 101 101 0
kmem_cache 197 200 192 20 1 : tunables 120 60 0 : slabdata 10 10 0
* After *
kmalloc-512 521 648 512 8 1 : tunables 54 27 0 : slabdata 81 81 0
kmalloc-256 208 208 256 16 1 : tunables 120 60 0 : slabdata 13 13 0
kmalloc-192 1029 1029 192 21 1 : tunables 120 60 0 : slabdata 49 49 0
kmalloc-96 529 589 128 31 1 : tunables 120 60 0 : slabdata 19 19 0
kmalloc-64 2142 2142 64 63 1 : tunables 120 60 0 : slabdata 34 34 0
kmalloc-128 660 682 128 31 1 : tunables 120 60 0 : slabdata 22 22 0
kmalloc-32 11716 11780 32 124 1 : tunables 120 60 0 : slabdata 95 95 0
kmem_cache 197 210 192 21 1 : tunables 120 60 0 : slabdata 10 10 0
kmem_caches consisting of objects less than or equal to 256 byte have
one or more objects than before. In the case of kmalloc-32, we have 11 more
objects, so 352 bytes (11 * 32) are saved and this is roughly 9% saving of
memory. Of couse, this percentage decreases as the number of objects
in a slab decreases.
Here are the performance results on my 4 cpus machine.
* Before *
Performance counter stats for 'perf bench sched messaging -g 50 -l 1000' (10 runs):
229,945,138 cache-misses ( +- 0.23% )
11.627897174 seconds time elapsed ( +- 0.14% )
* After *
Performance counter stats for 'perf bench sched messaging -g 50 -l 1000' (10 runs):
218,640,472 cache-misses ( +- 0.42% )
11.504999837 seconds time elapsed ( +- 0.21% )
cache-misses are reduced by this patchset, roughly 5%.
And elapsed times are improved by 1%.
Acked-by: Christoph Lameter <cl@linux.com>
Acked-by: David Rientjes <rientjes@google.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>
2013-12-02 01:49:42 -07:00
|
|
|
((freelist_idx_t *)(page->freelist))[idx] = val;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void cache_init_objs(struct kmem_cache *cachep,
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < cachep->num; i++) {
|
2013-10-23 19:07:49 -06:00
|
|
|
void *objp = index_to_obj(cachep, page, i);
|
2005-04-16 16:20:36 -06:00
|
|
|
#if DEBUG
|
|
|
|
/* need to poison the objs? */
|
|
|
|
if (cachep->flags & SLAB_POISON)
|
|
|
|
poison_obj(cachep, objp, POISON_FREE);
|
|
|
|
if (cachep->flags & SLAB_STORE_USER)
|
|
|
|
*dbg_userword(cachep, objp) = NULL;
|
|
|
|
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
|
|
|
*dbg_redzone1(cachep, objp) = RED_INACTIVE;
|
|
|
|
*dbg_redzone2(cachep, objp) = RED_INACTIVE;
|
|
|
|
}
|
|
|
|
/*
|
2006-03-22 01:08:11 -07:00
|
|
|
* Constructors are not allowed to allocate memory from the same
|
|
|
|
* cache which they are a constructor for. Otherwise, deadlock.
|
|
|
|
* They must also be threaded.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
if (cachep->ctor && !(cachep->flags & SLAB_POISON))
|
2008-07-25 20:45:34 -06:00
|
|
|
cachep->ctor(objp + obj_offset(cachep));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
|
|
|
if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
|
|
|
|
slab_error(cachep, "constructor overwrote the"
|
2006-01-08 02:00:37 -07:00
|
|
|
" end of an object");
|
2005-04-16 16:20:36 -06:00
|
|
|
if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
|
|
|
|
slab_error(cachep, "constructor overwrote the"
|
2006-01-08 02:00:37 -07:00
|
|
|
" start of an object");
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2012-06-13 09:24:57 -06:00
|
|
|
if ((cachep->size % PAGE_SIZE) == 0 &&
|
2006-03-22 01:08:11 -07:00
|
|
|
OFF_SLAB(cachep) && cachep->flags & SLAB_POISON)
|
2006-01-08 02:00:37 -07:00
|
|
|
kernel_map_pages(virt_to_page(objp),
|
2012-06-13 09:24:57 -06:00
|
|
|
cachep->size / PAGE_SIZE, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
#else
|
|
|
|
if (cachep->ctor)
|
2008-07-25 20:45:34 -06:00
|
|
|
cachep->ctor(objp);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
set_obj_status(page, i, OBJECT_FREE);
|
2013-12-02 01:49:40 -07:00
|
|
|
set_free_obj(page, i, i);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-02-10 02:43:10 -07:00
|
|
|
if (CONFIG_ZONE_DMA_FLAG) {
|
|
|
|
if (flags & GFP_DMA)
|
2012-06-14 06:17:21 -06:00
|
|
|
BUG_ON(!(cachep->allocflags & GFP_DMA));
|
2007-02-10 02:43:10 -07:00
|
|
|
else
|
2012-06-14 06:17:21 -06:00
|
|
|
BUG_ON(cachep->allocflags & GFP_DMA);
|
2007-02-10 02:43:10 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
static void *slab_get_obj(struct kmem_cache *cachep, struct page *page,
|
2006-03-22 01:08:11 -07:00
|
|
|
int nodeid)
|
2006-02-01 04:05:47 -07:00
|
|
|
{
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-23 19:07:45 -06:00
|
|
|
void *objp;
|
2006-02-01 04:05:47 -07:00
|
|
|
|
2013-12-02 01:49:40 -07:00
|
|
|
objp = index_to_obj(cachep, page, get_free_obj(page, page->active));
|
2013-10-23 19:07:49 -06:00
|
|
|
page->active++;
|
2006-02-01 04:05:47 -07:00
|
|
|
#if DEBUG
|
2013-10-23 19:07:40 -06:00
|
|
|
WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);
|
2006-02-01 04:05:47 -07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
static void slab_put_obj(struct kmem_cache *cachep, struct page *page,
|
2006-03-22 01:08:11 -07:00
|
|
|
void *objp, int nodeid)
|
2006-02-01 04:05:47 -07:00
|
|
|
{
|
2013-10-23 19:07:49 -06:00
|
|
|
unsigned int objnr = obj_to_index(cachep, page, objp);
|
2006-02-01 04:05:47 -07:00
|
|
|
#if DEBUG
|
2013-10-23 19:07:46 -06:00
|
|
|
unsigned int i;
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-23 19:07:45 -06:00
|
|
|
|
2006-02-01 04:05:47 -07:00
|
|
|
/* Verify that the slab belongs to the intended node */
|
2013-10-23 19:07:40 -06:00
|
|
|
WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);
|
2006-02-01 04:05:47 -07:00
|
|
|
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-23 19:07:45 -06:00
|
|
|
/* Verify double free bug */
|
2013-10-23 19:07:49 -06:00
|
|
|
for (i = page->active; i < cachep->num; i++) {
|
2013-12-02 01:49:40 -07:00
|
|
|
if (get_free_obj(page, i) == objnr) {
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-23 19:07:45 -06:00
|
|
|
printk(KERN_ERR "slab: double free detected in cache "
|
|
|
|
"'%s', objp %p\n", cachep->name, objp);
|
|
|
|
BUG();
|
|
|
|
}
|
2006-02-01 04:05:47 -07:00
|
|
|
}
|
|
|
|
#endif
|
2013-10-23 19:07:49 -06:00
|
|
|
page->active--;
|
2013-12-02 01:49:40 -07:00
|
|
|
set_free_obj(page, page->active, objnr);
|
2006-02-01 04:05:47 -07:00
|
|
|
}
|
|
|
|
|
2006-06-23 03:03:07 -06:00
|
|
|
/*
|
|
|
|
* Map pages beginning at addr to the given cache and slab. This is required
|
|
|
|
* for the slab allocator to be able to lookup the cache and slab of a
|
2011-01-06 23:49:17 -07:00
|
|
|
* virtual address for kfree, ksize, and slab debugging.
|
2006-06-23 03:03:07 -06:00
|
|
|
*/
|
2013-10-23 19:07:49 -06:00
|
|
|
static void slab_map_pages(struct kmem_cache *cache, struct page *page,
|
2013-10-30 04:04:01 -06:00
|
|
|
void *freelist)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2013-10-23 19:07:44 -06:00
|
|
|
page->slab_cache = cache;
|
2013-10-23 19:07:49 -06:00
|
|
|
page->freelist = freelist;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Grow (by 1) the number of slabs within a cache. This is called by
|
|
|
|
* kmem_cache_alloc() when there are no active objs left in a cache.
|
|
|
|
*/
|
2006-12-06 21:33:29 -07:00
|
|
|
static int cache_grow(struct kmem_cache *cachep,
|
2013-10-23 19:07:38 -06:00
|
|
|
gfp_t flags, int nodeid, struct page *page)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2013-10-30 04:04:01 -06:00
|
|
|
void *freelist;
|
2006-01-08 02:00:37 -07:00
|
|
|
size_t offset;
|
|
|
|
gfp_t local_flags;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* Be lazy and only check for valid flags here, keeping it out of the
|
|
|
|
* critical path in kmem_cache_alloc().
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2007-10-16 02:25:41 -06:00
|
|
|
BUG_ON(flags & GFP_SLAB_BUG_MASK);
|
|
|
|
local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
/* Take the node list lock to change the colour_next on this node */
|
2005-04-16 16:20:36 -06:00
|
|
|
check_irq_off();
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, nodeid);
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock(&n->list_lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Get colour for the slab, and cal the next value. */
|
2013-01-10 12:14:19 -07:00
|
|
|
offset = n->colour_next;
|
|
|
|
n->colour_next++;
|
|
|
|
if (n->colour_next >= cachep->colour)
|
|
|
|
n->colour_next = 0;
|
|
|
|
spin_unlock(&n->list_lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-02-05 00:27:56 -07:00
|
|
|
offset *= cachep->colour_off;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (local_flags & __GFP_WAIT)
|
|
|
|
local_irq_enable();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The test for missing atomic flag is performed here, rather than
|
|
|
|
* the more obvious place, simply to reduce the critical path length
|
|
|
|
* in kmem_cache_alloc(). If a caller is seriously mis-behaving they
|
|
|
|
* will eventually be caught here (where it matters).
|
|
|
|
*/
|
|
|
|
kmem_flagcheck(cachep, flags);
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* Get mem for the objs. Attempt to allocate a physical page from
|
|
|
|
* 'nodeid'.
|
2005-09-09 14:03:32 -06:00
|
|
|
*/
|
2013-10-23 19:07:38 -06:00
|
|
|
if (!page)
|
|
|
|
page = kmem_getpages(cachep, local_flags, nodeid);
|
|
|
|
if (!page)
|
2005-04-16 16:20:36 -06:00
|
|
|
goto failed;
|
|
|
|
|
|
|
|
/* Get slab management. */
|
2013-10-23 19:07:49 -06:00
|
|
|
freelist = alloc_slabmgmt(cachep, page, offset,
|
2007-10-16 02:25:41 -06:00
|
|
|
local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
|
2013-10-23 19:07:49 -06:00
|
|
|
if (!freelist)
|
2005-04-16 16:20:36 -06:00
|
|
|
goto opps1;
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
slab_map_pages(cachep, page, freelist);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
cache_init_objs(cachep, page);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (local_flags & __GFP_WAIT)
|
|
|
|
local_irq_disable();
|
|
|
|
check_irq_off();
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock(&n->list_lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Make slab active. */
|
2013-10-23 19:07:49 -06:00
|
|
|
list_add_tail(&page->lru, &(n->slabs_free));
|
2005-04-16 16:20:36 -06:00
|
|
|
STATS_INC_GROWN(cachep);
|
2013-01-10 12:14:19 -07:00
|
|
|
n->free_objects += cachep->num;
|
|
|
|
spin_unlock(&n->list_lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
return 1;
|
2006-03-22 01:08:11 -07:00
|
|
|
opps1:
|
2013-10-23 19:07:38 -06:00
|
|
|
kmem_freepages(cachep, page);
|
2006-03-22 01:08:11 -07:00
|
|
|
failed:
|
2005-04-16 16:20:36 -06:00
|
|
|
if (local_flags & __GFP_WAIT)
|
|
|
|
local_irq_disable();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Perform extra freeing checks:
|
|
|
|
* - detect bad pointers.
|
|
|
|
* - POISON/RED_ZONE checking
|
|
|
|
*/
|
|
|
|
static void kfree_debugcheck(const void *objp)
|
|
|
|
{
|
|
|
|
if (!virt_addr_valid(objp)) {
|
|
|
|
printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
|
2006-01-08 02:00:37 -07:00
|
|
|
(unsigned long)objp);
|
|
|
|
BUG();
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-06-23 03:03:24 -06:00
|
|
|
static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
|
|
|
|
{
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 01:22:59 -06:00
|
|
|
unsigned long long redzone1, redzone2;
|
2006-06-23 03:03:24 -06:00
|
|
|
|
|
|
|
redzone1 = *dbg_redzone1(cache, obj);
|
|
|
|
redzone2 = *dbg_redzone2(cache, obj);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Redzone is ok.
|
|
|
|
*/
|
|
|
|
if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
|
|
|
|
slab_error(cache, "double free detected");
|
|
|
|
else
|
|
|
|
slab_error(cache, "memory outside object was overwritten");
|
|
|
|
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 01:22:59 -06:00
|
|
|
printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n",
|
2006-06-23 03:03:24 -06:00
|
|
|
obj, redzone1, redzone2);
|
|
|
|
}
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
|
2012-09-08 14:47:55 -06:00
|
|
|
unsigned long caller)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
unsigned int objnr;
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-11-29 12:05:13 -07:00
|
|
|
BUG_ON(virt_to_cache(objp) != cachep);
|
|
|
|
|
2006-02-01 04:05:42 -07:00
|
|
|
objp -= obj_offset(cachep);
|
2005-04-16 16:20:36 -06:00
|
|
|
kfree_debugcheck(objp);
|
2007-05-06 15:49:41 -06:00
|
|
|
page = virt_to_head_page(objp);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
2006-06-23 03:03:24 -06:00
|
|
|
verify_redzone_free(cachep, objp);
|
2005-04-16 16:20:36 -06:00
|
|
|
*dbg_redzone1(cachep, objp) = RED_INACTIVE;
|
|
|
|
*dbg_redzone2(cachep, objp) = RED_INACTIVE;
|
|
|
|
}
|
|
|
|
if (cachep->flags & SLAB_STORE_USER)
|
2012-09-08 14:47:55 -06:00
|
|
|
*dbg_userword(cachep, objp) = (void *)caller;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
objnr = obj_to_index(cachep, page, objp);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
BUG_ON(objnr >= cachep->num);
|
2013-10-23 19:07:49 -06:00
|
|
|
BUG_ON(objp != index_to_obj(cachep, page, objnr));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
set_obj_status(page, objnr, OBJECT_FREE);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (cachep->flags & SLAB_POISON) {
|
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
2012-06-13 09:24:57 -06:00
|
|
|
if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
|
2012-09-08 14:47:55 -06:00
|
|
|
store_stackinfo(cachep, objp, caller);
|
2006-01-08 02:00:37 -07:00
|
|
|
kernel_map_pages(virt_to_page(objp),
|
2012-06-13 09:24:57 -06:00
|
|
|
cachep->size / PAGE_SIZE, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
} else {
|
|
|
|
poison_obj(cachep, objp, POISON_FREE);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
poison_obj(cachep, objp, POISON_FREE);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
#define kfree_debugcheck(x) do { } while(0)
|
|
|
|
#define cache_free_debugcheck(x,objp,z) (objp)
|
|
|
|
#endif
|
|
|
|
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags,
|
|
|
|
bool force_refill)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int batchcount;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2005-04-16 16:20:36 -06:00
|
|
|
struct array_cache *ac;
|
2006-10-06 01:43:52 -06:00
|
|
|
int node;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
check_irq_off();
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
node = numa_mem_id();
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
if (unlikely(force_refill))
|
|
|
|
goto force_grow;
|
|
|
|
retry:
|
2006-02-01 04:05:49 -07:00
|
|
|
ac = cpu_cache_get(cachep);
|
2005-04-16 16:20:36 -06:00
|
|
|
batchcount = ac->batchcount;
|
|
|
|
if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* If there was little recent activity on this cache, then
|
|
|
|
* perform only a partial refill. Otherwise we could generate
|
|
|
|
* refill bouncing.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
batchcount = BATCHREFILL_LIMIT;
|
|
|
|
}
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
BUG_ON(ac->avail > 0 || !n);
|
|
|
|
spin_lock(&n->list_lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-03-25 04:06:44 -07:00
|
|
|
/* See if we can refill from the shared array */
|
2013-01-10 12:14:19 -07:00
|
|
|
if (n->shared && transfer_objects(ac, n->shared, batchcount)) {
|
|
|
|
n->shared->touched = 1;
|
2006-03-25 04:06:44 -07:00
|
|
|
goto alloc_done;
|
2010-01-27 04:27:40 -07:00
|
|
|
}
|
2006-03-25 04:06:44 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
while (batchcount > 0) {
|
|
|
|
struct list_head *entry;
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page;
|
2005-04-16 16:20:36 -06:00
|
|
|
/* Get slab alloc is to come from. */
|
2013-01-10 12:14:19 -07:00
|
|
|
entry = n->slabs_partial.next;
|
|
|
|
if (entry == &n->slabs_partial) {
|
|
|
|
n->free_touched = 1;
|
|
|
|
entry = n->slabs_free.next;
|
|
|
|
if (entry == &n->slabs_free)
|
2005-04-16 16:20:36 -06:00
|
|
|
goto must_grow;
|
|
|
|
}
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
page = list_entry(entry, struct page, lru);
|
2005-04-16 16:20:36 -06:00
|
|
|
check_spinlock_acquired(cachep);
|
2007-05-06 15:49:03 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The slab was either on partial or free list so
|
|
|
|
* there must be at least one object available for
|
|
|
|
* allocation.
|
|
|
|
*/
|
2013-10-23 19:07:49 -06:00
|
|
|
BUG_ON(page->active >= cachep->num);
|
2007-05-06 15:49:03 -06:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
while (page->active < cachep->num && batchcount--) {
|
2005-04-16 16:20:36 -06:00
|
|
|
STATS_INC_ALLOCED(cachep);
|
|
|
|
STATS_INC_ACTIVE(cachep);
|
|
|
|
STATS_SET_HIGH(cachep);
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
ac_put_obj(cachep, ac, slab_get_obj(cachep, page,
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
node));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/* move slabp to correct slabp list: */
|
2013-10-23 19:07:49 -06:00
|
|
|
list_del(&page->lru);
|
|
|
|
if (page->active == cachep->num)
|
2014-04-08 14:44:27 -06:00
|
|
|
list_add(&page->lru, &n->slabs_full);
|
2005-04-16 16:20:36 -06:00
|
|
|
else
|
2014-04-08 14:44:27 -06:00
|
|
|
list_add(&page->lru, &n->slabs_partial);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
must_grow:
|
2013-01-10 12:14:19 -07:00
|
|
|
n->free_objects -= ac->avail;
|
2006-03-22 01:08:11 -07:00
|
|
|
alloc_done:
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock(&n->list_lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (unlikely(!ac->avail)) {
|
|
|
|
int x;
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
force_grow:
|
2006-12-06 21:33:29 -07:00
|
|
|
x = cache_grow(cachep, flags | GFP_THISNODE, node, NULL);
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/* cache_grow can reenable interrupts, then ac could change. */
|
2006-02-01 04:05:49 -07:00
|
|
|
ac = cpu_cache_get(cachep);
|
2012-08-28 20:57:21 -06:00
|
|
|
node = numa_mem_id();
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
|
|
|
|
/* no objects in sight? abort */
|
|
|
|
if (!x && (ac->avail == 0 || force_refill))
|
2005-04-16 16:20:36 -06:00
|
|
|
return NULL;
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
if (!ac->avail) /* objects refilled by interrupt? */
|
2005-04-16 16:20:36 -06:00
|
|
|
goto retry;
|
|
|
|
}
|
|
|
|
ac->touched = 1;
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
|
|
|
|
return ac_get_obj(cachep, ac, flags, force_refill);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
|
|
|
|
gfp_t flags)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
might_sleep_if(flags & __GFP_WAIT);
|
|
|
|
#if DEBUG
|
|
|
|
kmem_flagcheck(cachep, flags);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
#if DEBUG
|
2006-03-22 01:08:11 -07:00
|
|
|
static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
|
2012-09-08 14:47:55 -06:00
|
|
|
gfp_t flags, void *objp, unsigned long caller)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
struct page *page;
|
|
|
|
|
2006-01-08 02:00:37 -07:00
|
|
|
if (!objp)
|
2005-04-16 16:20:36 -06:00
|
|
|
return objp;
|
2006-01-08 02:00:37 -07:00
|
|
|
if (cachep->flags & SLAB_POISON) {
|
2005-04-16 16:20:36 -06:00
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
2012-06-13 09:24:57 -06:00
|
|
|
if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
|
2006-01-08 02:00:37 -07:00
|
|
|
kernel_map_pages(virt_to_page(objp),
|
2012-06-13 09:24:57 -06:00
|
|
|
cachep->size / PAGE_SIZE, 1);
|
2005-04-16 16:20:36 -06:00
|
|
|
else
|
|
|
|
check_poison_obj(cachep, objp);
|
|
|
|
#else
|
|
|
|
check_poison_obj(cachep, objp);
|
|
|
|
#endif
|
|
|
|
poison_obj(cachep, objp, POISON_INUSE);
|
|
|
|
}
|
|
|
|
if (cachep->flags & SLAB_STORE_USER)
|
2012-09-08 14:47:55 -06:00
|
|
|
*dbg_userword(cachep, objp) = (void *)caller;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (cachep->flags & SLAB_RED_ZONE) {
|
2006-03-22 01:08:11 -07:00
|
|
|
if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
|
|
|
|
*dbg_redzone2(cachep, objp) != RED_INACTIVE) {
|
|
|
|
slab_error(cachep, "double free, or memory outside"
|
|
|
|
" object was overwritten");
|
2006-01-08 02:00:37 -07:00
|
|
|
printk(KERN_ERR
|
Increase slab redzone to 64bits
There are two problems with the existing redzone implementation.
Firstly, it's causing misalignment of structures which contain a 64-bit
integer, such as netfilter's 'struct ipt_entry' -- causing netfilter
modules to fail to load because of the misalignment. (In particular, the
first check in
net/ipv4/netfilter/ip_tables.c::check_entry_size_and_hooks())
On ppc32 and sparc32, amongst others, __alignof__(uint64_t) == 8.
With slab debugging, we use 32-bit redzones. And allocated slab objects
aren't sufficiently aligned to hold a structure containing a uint64_t.
By _just_ setting ARCH_KMALLOC_MINALIGN to __alignof__(u64) we'd disable
redzone checks on those architectures. By using 64-bit redzones we avoid that
loss of debugging, and also fix the other problem while we're at it.
When investigating this, I noticed that on 64-bit platforms we're using a
32-bit value of RED_ACTIVE/RED_INACTIVE in the 64-bit memory location set
aside for the redzone. Which means that the four bytes immediately before
or after the allocated object at 0x00,0x00,0x00,0x00 for LE and BE
machines, respectively. Which is probably not the most useful choice of
poison value.
One way to fix both of those at once is just to switch to 64-bit
redzones in all cases.
Signed-off-by: David Woodhouse <dwmw2@infradead.org>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Christoph Lameter <clameter@engr.sgi.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 01:22:59 -06:00
|
|
|
"%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
|
2006-03-22 01:08:11 -07:00
|
|
|
objp, *dbg_redzone1(cachep, objp),
|
|
|
|
*dbg_redzone2(cachep, objp));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
*dbg_redzone1(cachep, objp) = RED_ACTIVE;
|
|
|
|
*dbg_redzone2(cachep, objp) = RED_ACTIVE;
|
|
|
|
}
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
|
|
|
|
page = virt_to_head_page(objp);
|
|
|
|
set_obj_status(page, obj_to_index(cachep, page, objp), OBJECT_ACTIVE);
|
2006-02-01 04:05:42 -07:00
|
|
|
objp += obj_offset(cachep);
|
2007-05-06 15:50:17 -06:00
|
|
|
if (cachep->ctor && cachep->flags & SLAB_POISON)
|
2008-07-25 20:45:34 -06:00
|
|
|
cachep->ctor(objp);
|
2011-07-20 18:42:45 -06:00
|
|
|
if (ARCH_SLAB_MINALIGN &&
|
|
|
|
((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
|
2006-12-06 21:32:11 -07:00
|
|
|
printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
|
2011-07-11 14:35:08 -06:00
|
|
|
objp, (int)ARCH_SLAB_MINALIGN);
|
2006-12-06 21:32:11 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
|
|
|
|
#endif
|
|
|
|
|
2008-12-23 03:37:01 -07:00
|
|
|
static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
|
2006-12-08 03:39:44 -07:00
|
|
|
{
|
slab: add unlikely macro to help compiler
This patchset does some cleanup and tries to remove lockdep annotation.
Patches 1~2 are just for really really minor improvement.
Patches 3~9 are for clean-up and removing lockdep annotation.
There are two cases that lockdep annotation is needed in SLAB.
1) holding two node locks
2) holding two array cache(alien cache) locks
I looked at the code and found that we can avoid these cases without any
negative effect.
1) occurs if freeing object makes new free slab and we decide to
destroy it. Although we don't need to hold the lock during destroying
a slab, current code do that. Destroying a slab without holding the
lock would help the reduction of the lock contention. To do it, I
change the implementation that new free slab is destroyed after
releasing the lock.
2) occurs on similar situation. When we free object from non-local
node, we put this object to alien cache with holding the alien cache
lock. If alien cache is full, we try to flush alien cache to proper
node cache, and, in this time, new free slab could be made. Destroying
it would be started and we will free metadata object which comes from
another node. In this case, we need another node's alien cache lock to
free object. This forces us to hold two array cache locks and then we
need lockdep annotation although they are always different locks and
deadlock cannot be possible. To prevent this situation, I use same way
as 1).
In this way, we can avoid 1) and 2) cases, and then, can remove lockdep
annotation. As short stat noted, this makes SLAB code much simpler.
This patch (of 9):
slab_should_failslab() is called on every allocation, so to optimize it
is reasonable. We normally don't allocate from kmem_cache. It is just
used when new kmem_cache is created, so it's very rare case. Therefore,
add unlikely macro to help compiler optimization.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: David Rientjes <rientjes@google.com>
Acked-by: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-08-06 17:04:20 -06:00
|
|
|
if (unlikely(cachep == kmem_cache))
|
2008-12-23 03:37:01 -07:00
|
|
|
return false;
|
2006-12-08 03:39:44 -07:00
|
|
|
|
2012-06-13 09:24:58 -06:00
|
|
|
return should_failslab(cachep->object_size, flags, cachep->flags);
|
2006-12-08 03:39:44 -07:00
|
|
|
}
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-01-08 02:00:37 -07:00
|
|
|
void *objp;
|
2005-04-16 16:20:36 -06:00
|
|
|
struct array_cache *ac;
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
bool force_refill = false;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2005-09-27 22:45:46 -06:00
|
|
|
check_irq_off();
|
2006-12-08 03:39:44 -07:00
|
|
|
|
2006-02-01 04:05:49 -07:00
|
|
|
ac = cpu_cache_get(cachep);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (likely(ac->avail)) {
|
|
|
|
ac->touched = 1;
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
objp = ac_get_obj(cachep, ac, flags, false);
|
|
|
|
|
2009-12-02 00:55:50 -07:00
|
|
|
/*
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
* Allow for the possibility all avail objects are not allowed
|
|
|
|
* by the current flags
|
2009-12-02 00:55:50 -07:00
|
|
|
*/
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
if (objp) {
|
|
|
|
STATS_INC_ALLOCHIT(cachep);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
force_refill = true;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
|
|
|
|
STATS_INC_ALLOCMISS(cachep);
|
|
|
|
objp = cache_alloc_refill(cachep, flags, force_refill);
|
|
|
|
/*
|
|
|
|
* the 'ac' may be updated by cache_alloc_refill(),
|
|
|
|
* and kmemleak_erase() requires its correct value.
|
|
|
|
*/
|
|
|
|
ac = cpu_cache_get(cachep);
|
|
|
|
|
|
|
|
out:
|
2009-06-11 06:22:40 -06:00
|
|
|
/*
|
|
|
|
* To avoid a false negative, if an object that is in one of the
|
|
|
|
* per-CPU caches is leaked, we need to make sure kmemleak doesn't
|
|
|
|
* treat the array pointers as a reference to the object.
|
|
|
|
*/
|
2009-12-02 00:55:49 -07:00
|
|
|
if (objp)
|
|
|
|
kmemleak_erase(&ac->entry[ac->avail]);
|
2005-09-27 22:45:46 -06:00
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
|
2005-09-09 14:03:32 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
2006-03-24 04:16:08 -07:00
|
|
|
/*
|
2014-09-24 19:41:02 -06:00
|
|
|
* Try allocating on another node if PFA_SPREAD_SLAB is a mempolicy is set.
|
2006-03-24 04:16:08 -07:00
|
|
|
*
|
|
|
|
* If we are in_interrupt, then process context, including cpusets and
|
|
|
|
* mempolicy, may not apply and should not be used for allocation policy.
|
|
|
|
*/
|
|
|
|
static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
|
|
|
|
{
|
|
|
|
int nid_alloc, nid_here;
|
|
|
|
|
2006-09-27 02:50:08 -06:00
|
|
|
if (in_interrupt() || (flags & __GFP_THISNODE))
|
2006-03-24 04:16:08 -07:00
|
|
|
return NULL;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
nid_alloc = nid_here = numa_mem_id();
|
2006-03-24 04:16:08 -07:00
|
|
|
if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
|
2010-05-26 15:42:49 -06:00
|
|
|
nid_alloc = cpuset_slab_spread_node();
|
2006-03-24 04:16:08 -07:00
|
|
|
else if (current->mempolicy)
|
2014-04-07 16:37:29 -06:00
|
|
|
nid_alloc = mempolicy_slab_node();
|
2006-03-24 04:16:08 -07:00
|
|
|
if (nid_alloc != nid_here)
|
2006-12-06 21:32:30 -07:00
|
|
|
return ____cache_alloc_node(cachep, flags, nid_alloc);
|
2006-03-24 04:16:08 -07:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2006-09-27 02:50:08 -06:00
|
|
|
/*
|
|
|
|
* Fallback function if there was no memory available and no objects on a
|
2006-12-06 21:33:29 -07:00
|
|
|
* certain node and fall back is permitted. First we scan all the
|
2013-01-10 12:14:19 -07:00
|
|
|
* available node for available objects. If that fails then we
|
2006-12-06 21:33:29 -07:00
|
|
|
* perform an allocation without specifying a node. This allows the page
|
|
|
|
* allocator to do its reclaim / fallback magic. We then insert the
|
|
|
|
* slab into the proper nodelist and then allocate from it.
|
2006-09-27 02:50:08 -06:00
|
|
|
*/
|
2007-02-10 02:42:53 -07:00
|
|
|
static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
|
2006-09-27 02:50:08 -06:00
|
|
|
{
|
2007-02-10 02:42:53 -07:00
|
|
|
struct zonelist *zonelist;
|
|
|
|
gfp_t local_flags;
|
2008-04-28 03:12:17 -06:00
|
|
|
struct zoneref *z;
|
2008-04-28 03:12:16 -06:00
|
|
|
struct zone *zone;
|
|
|
|
enum zone_type high_zoneidx = gfp_zone(flags);
|
2006-09-27 02:50:08 -06:00
|
|
|
void *obj = NULL;
|
2006-12-06 21:33:29 -07:00
|
|
|
int nid;
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
unsigned int cpuset_mems_cookie;
|
2007-02-10 02:42:53 -07:00
|
|
|
|
|
|
|
if (flags & __GFP_THISNODE)
|
|
|
|
return NULL;
|
|
|
|
|
2007-10-16 02:25:41 -06:00
|
|
|
local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
|
2006-09-27 02:50:08 -06:00
|
|
|
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
retry_cpuset:
|
2014-04-03 15:47:24 -06:00
|
|
|
cpuset_mems_cookie = read_mems_allowed_begin();
|
2014-04-07 16:37:29 -06:00
|
|
|
zonelist = node_zonelist(mempolicy_slab_node(), flags);
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
|
2006-12-06 21:33:29 -07:00
|
|
|
retry:
|
|
|
|
/*
|
|
|
|
* Look through allowed nodes for objects available
|
|
|
|
* from existing per node queues.
|
|
|
|
*/
|
2008-04-28 03:12:16 -06:00
|
|
|
for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
|
|
|
|
nid = zone_to_nid(zone);
|
2006-10-21 11:24:16 -06:00
|
|
|
|
2008-04-28 03:12:16 -06:00
|
|
|
if (cpuset_zone_allowed_hardwall(zone, flags) &&
|
2014-08-06 17:04:11 -06:00
|
|
|
get_node(cache, nid) &&
|
|
|
|
get_node(cache, nid)->free_objects) {
|
2006-12-06 21:33:29 -07:00
|
|
|
obj = ____cache_alloc_node(cache,
|
|
|
|
flags | GFP_THISNODE, nid);
|
2008-06-21 17:46:35 -06:00
|
|
|
if (obj)
|
|
|
|
break;
|
|
|
|
}
|
2006-12-06 21:33:29 -07:00
|
|
|
}
|
|
|
|
|
2007-05-06 15:50:17 -06:00
|
|
|
if (!obj) {
|
2006-12-06 21:33:29 -07:00
|
|
|
/*
|
|
|
|
* This allocation will be performed within the constraints
|
|
|
|
* of the current cpuset / memory policy requirements.
|
|
|
|
* We may trigger various forms of reclaim on the allowed
|
|
|
|
* set and go into memory reserves if necessary.
|
|
|
|
*/
|
2013-10-23 19:07:38 -06:00
|
|
|
struct page *page;
|
|
|
|
|
2006-12-13 01:34:11 -07:00
|
|
|
if (local_flags & __GFP_WAIT)
|
|
|
|
local_irq_enable();
|
|
|
|
kmem_flagcheck(cache, flags);
|
2013-10-23 19:07:38 -06:00
|
|
|
page = kmem_getpages(cache, local_flags, numa_mem_id());
|
2006-12-13 01:34:11 -07:00
|
|
|
if (local_flags & __GFP_WAIT)
|
|
|
|
local_irq_disable();
|
2013-10-23 19:07:38 -06:00
|
|
|
if (page) {
|
2006-12-06 21:33:29 -07:00
|
|
|
/*
|
|
|
|
* Insert into the appropriate per node queues
|
|
|
|
*/
|
2013-10-23 19:07:38 -06:00
|
|
|
nid = page_to_nid(page);
|
|
|
|
if (cache_grow(cache, flags, nid, page)) {
|
2006-12-06 21:33:29 -07:00
|
|
|
obj = ____cache_alloc_node(cache,
|
|
|
|
flags | GFP_THISNODE, nid);
|
|
|
|
if (!obj)
|
|
|
|
/*
|
|
|
|
* Another processor may allocate the
|
|
|
|
* objects in the slab since we are
|
|
|
|
* not holding any locks.
|
|
|
|
*/
|
|
|
|
goto retry;
|
|
|
|
} else {
|
2007-01-05 17:36:36 -07:00
|
|
|
/* cache_grow already freed obj */
|
2006-12-06 21:33:29 -07:00
|
|
|
obj = NULL;
|
|
|
|
}
|
|
|
|
}
|
2006-10-21 11:24:16 -06:00
|
|
|
}
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
|
2014-04-03 15:47:24 -06:00
|
|
|
if (unlikely(!obj && read_mems_allowed_retry(cpuset_mems_cookie)))
|
cpuset: mm: reduce large amounts of memory barrier related damage v3
Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when
changing cpuset's mems") wins a super prize for the largest number of
memory barriers entered into fast paths for one commit.
[get|put]_mems_allowed is incredibly heavy with pairs of full memory
barriers inserted into a number of hot paths. This was detected while
investigating at large page allocator slowdown introduced some time
after 2.6.32. The largest portion of this overhead was shown by
oprofile to be at an mfence introduced by this commit into the page
allocator hot path.
For extra style points, the commit introduced the use of yield() in an
implementation of what looks like a spinning mutex.
This patch replaces the full memory barriers on both read and write
sides with a sequence counter with just read barriers on the fast path
side. This is much cheaper on some architectures, including x86. The
main bulk of the patch is the retry logic if the nodemask changes in a
manner that can cause a false failure.
While updating the nodemask, a check is made to see if a false failure
is a risk. If it is, the sequence number gets bumped and parallel
allocators will briefly stall while the nodemask update takes place.
In a page fault test microbenchmark, oprofile samples from
__alloc_pages_nodemask went from 4.53% of all samples to 1.15%. The
actual results were
3.3.0-rc3 3.3.0-rc3
rc3-vanilla nobarrier-v2r1
Clients 1 UserTime 0.07 ( 0.00%) 0.08 (-14.19%)
Clients 2 UserTime 0.07 ( 0.00%) 0.07 ( 2.72%)
Clients 4 UserTime 0.08 ( 0.00%) 0.07 ( 3.29%)
Clients 1 SysTime 0.70 ( 0.00%) 0.65 ( 6.65%)
Clients 2 SysTime 0.85 ( 0.00%) 0.82 ( 3.65%)
Clients 4 SysTime 1.41 ( 0.00%) 1.41 ( 0.32%)
Clients 1 WallTime 0.77 ( 0.00%) 0.74 ( 4.19%)
Clients 2 WallTime 0.47 ( 0.00%) 0.45 ( 3.73%)
Clients 4 WallTime 0.38 ( 0.00%) 0.37 ( 1.58%)
Clients 1 Flt/sec/cpu 497620.28 ( 0.00%) 520294.53 ( 4.56%)
Clients 2 Flt/sec/cpu 414639.05 ( 0.00%) 429882.01 ( 3.68%)
Clients 4 Flt/sec/cpu 257959.16 ( 0.00%) 258761.48 ( 0.31%)
Clients 1 Flt/sec 495161.39 ( 0.00%) 517292.87 ( 4.47%)
Clients 2 Flt/sec 820325.95 ( 0.00%) 850289.77 ( 3.65%)
Clients 4 Flt/sec 1020068.93 ( 0.00%) 1022674.06 ( 0.26%)
MMTests Statistics: duration
Sys Time Running Test (seconds) 135.68 132.17
User+Sys Time Running Test (seconds) 164.2 160.13
Total Elapsed Time (seconds) 123.46 120.87
The overall improvement is small but the System CPU time is much
improved and roughly in correlation to what oprofile reported (these
performance figures are without profiling so skew is expected). The
actual number of page faults is noticeably improved.
For benchmarks like kernel builds, the overall benefit is marginal but
the system CPU time is slightly reduced.
To test the actual bug the commit fixed I opened two terminals. The
first ran within a cpuset and continually ran a small program that
faulted 100M of anonymous data. In a second window, the nodemask of the
cpuset was continually randomised in a loop.
Without the commit, the program would fail every so often (usually
within 10 seconds) and obviously with the commit everything worked fine.
With this patch applied, it also worked fine so the fix should be
functionally equivalent.
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Miao Xie <miaox@cn.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:11 -06:00
|
|
|
goto retry_cpuset;
|
2006-09-27 02:50:08 -06:00
|
|
|
return obj;
|
|
|
|
}
|
|
|
|
|
2005-09-09 14:03:32 -06:00
|
|
|
/*
|
|
|
|
* A interface to enable slab creation on nodeid
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2006-12-06 21:32:30 -07:00
|
|
|
static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
|
2006-03-22 01:08:11 -07:00
|
|
|
int nodeid)
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
|
|
|
struct list_head *entry;
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2006-01-08 02:00:37 -07:00
|
|
|
void *obj;
|
|
|
|
int x;
|
|
|
|
|
slab: fix nodeid bounds check for non-contiguous node IDs
The bounds check for nodeid in ____cache_alloc_node gives false
positives on machines where the node IDs are not contiguous, leading to
a panic at boot time. For example, on a POWER8 machine the node IDs are
typically 0, 1, 16 and 17. This means that num_online_nodes() returns
4, so when ____cache_alloc_node is called with nodeid = 16 the VM_BUG_ON
triggers, like this:
kernel BUG at /home/paulus/kernel/kvm/mm/slab.c:3079!
Call Trace:
.____cache_alloc_node+0x5c/0x270 (unreliable)
.kmem_cache_alloc_node_trace+0xdc/0x360
.init_list+0x3c/0x128
.kmem_cache_init+0x1dc/0x258
.start_kernel+0x2a0/0x568
start_here_common+0x20/0xa8
To fix this, we instead compare the nodeid with MAX_NUMNODES, and
additionally make sure it isn't negative (since nodeid is an int). The
check is there mainly to protect the array dereference in the get_node()
call in the next line, and the array being dereferenced is of size
MAX_NUMNODES. If the nodeid is in range but invalid (for example if the
node is off-line), the BUG_ON in the next line will catch that.
Fixes: 14e50c6a9bc2 ("mm: slab: Verify the nodeid passed to ____cache_alloc_node")
Signed-off-by: Paul Mackerras <paulus@samba.org>
Reviewed-by: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Reviewed-by: Pekka Enberg <penberg@kernel.org>
Acked-by: David Rientjes <rientjes@google.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-12-02 16:59:48 -07:00
|
|
|
VM_BUG_ON(nodeid < 0 || nodeid >= MAX_NUMNODES);
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, nodeid);
|
2013-01-10 12:14:19 -07:00
|
|
|
BUG_ON(!n);
|
2006-01-08 02:00:37 -07:00
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
retry:
|
2006-02-05 00:27:58 -07:00
|
|
|
check_irq_off();
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock(&n->list_lock);
|
|
|
|
entry = n->slabs_partial.next;
|
|
|
|
if (entry == &n->slabs_partial) {
|
|
|
|
n->free_touched = 1;
|
|
|
|
entry = n->slabs_free.next;
|
|
|
|
if (entry == &n->slabs_free)
|
2006-01-08 02:00:37 -07:00
|
|
|
goto must_grow;
|
|
|
|
}
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
page = list_entry(entry, struct page, lru);
|
2006-01-08 02:00:37 -07:00
|
|
|
check_spinlock_acquired_node(cachep, nodeid);
|
|
|
|
|
|
|
|
STATS_INC_NODEALLOCS(cachep);
|
|
|
|
STATS_INC_ACTIVE(cachep);
|
|
|
|
STATS_SET_HIGH(cachep);
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
BUG_ON(page->active == cachep->num);
|
2006-01-08 02:00:37 -07:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
obj = slab_get_obj(cachep, page, nodeid);
|
2013-01-10 12:14:19 -07:00
|
|
|
n->free_objects--;
|
2006-01-08 02:00:37 -07:00
|
|
|
/* move slabp to correct slabp list: */
|
2013-10-23 19:07:49 -06:00
|
|
|
list_del(&page->lru);
|
2006-01-08 02:00:37 -07:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
if (page->active == cachep->num)
|
|
|
|
list_add(&page->lru, &n->slabs_full);
|
2006-03-22 01:08:11 -07:00
|
|
|
else
|
2013-10-23 19:07:49 -06:00
|
|
|
list_add(&page->lru, &n->slabs_partial);
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock(&n->list_lock);
|
2006-01-08 02:00:37 -07:00
|
|
|
goto done;
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
must_grow:
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock(&n->list_lock);
|
2006-12-06 21:33:29 -07:00
|
|
|
x = cache_grow(cachep, flags | GFP_THISNODE, nodeid, NULL);
|
2006-09-27 02:50:08 -06:00
|
|
|
if (x)
|
|
|
|
goto retry;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-02-10 02:42:53 -07:00
|
|
|
return fallback_alloc(cachep, flags);
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
done:
|
2006-01-08 02:00:37 -07:00
|
|
|
return obj;
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
2007-02-10 02:42:53 -07:00
|
|
|
|
|
|
|
static __always_inline void *
|
2012-09-08 14:47:57 -06:00
|
|
|
slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
|
2012-09-08 14:47:55 -06:00
|
|
|
unsigned long caller)
|
2007-02-10 02:42:53 -07:00
|
|
|
{
|
|
|
|
unsigned long save_flags;
|
|
|
|
void *ptr;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
int slab_node = numa_mem_id();
|
2007-02-10 02:42:53 -07:00
|
|
|
|
2009-06-17 21:24:12 -06:00
|
|
|
flags &= gfp_allowed_mask;
|
2009-06-12 05:03:06 -06:00
|
|
|
|
lockdep: annotate reclaim context (__GFP_NOFS)
Here is another version, with the incremental patch rolled up, and
added reclaim context annotation to kswapd, and allocation tracing
to slab allocators (which may only ever reach the page allocator
in rare cases, so it is good to put annotations here too).
Haven't tested this version as such, but it should be getting closer
to merge worthy ;)
--
After noticing some code in mm/filemap.c accidentally perform a __GFP_FS
allocation when it should not have been, I thought it might be a good idea to
try to catch this kind of thing with lockdep.
I coded up a little idea that seems to work. Unfortunately the system has to
actually be in __GFP_FS page reclaim, then take the lock, before it will mark
it. But at least that might still be some orders of magnitude more common
(and more debuggable) than an actual deadlock condition, so we have some
improvement I hope (the concept is no less complete than discovery of a lock's
interrupt contexts).
I guess we could even do the same thing with __GFP_IO (normal reclaim), and
even GFP_NOIO locks too... but filesystems will have the most locks and fiddly
code paths, so let's start there and see how it goes.
It *seems* to work. I did a quick test.
=================================
[ INFO: inconsistent lock state ]
2.6.28-rc6-00007-ged31348-dirty #26
---------------------------------
inconsistent {in-reclaim-W} -> {ov-reclaim-W} usage.
modprobe/8526 [HC0[0]:SC0[0]:HE1:SE1] takes:
(testlock){--..}, at: [<ffffffffa0020055>] brd_init+0x55/0x216 [brd]
{in-reclaim-W} state was registered at:
[<ffffffff80267bdb>] __lock_acquire+0x75b/0x1a60
[<ffffffff80268f71>] lock_acquire+0x91/0xc0
[<ffffffff8070f0e1>] mutex_lock_nested+0xb1/0x310
[<ffffffffa002002b>] brd_init+0x2b/0x216 [brd]
[<ffffffff8020903b>] _stext+0x3b/0x170
[<ffffffff80272ebf>] sys_init_module+0xaf/0x1e0
[<ffffffff8020c3fb>] system_call_fastpath+0x16/0x1b
[<ffffffffffffffff>] 0xffffffffffffffff
irq event stamp: 3929
hardirqs last enabled at (3929): [<ffffffff8070f2b5>] mutex_lock_nested+0x285/0x310
hardirqs last disabled at (3928): [<ffffffff8070f089>] mutex_lock_nested+0x59/0x310
softirqs last enabled at (3732): [<ffffffff8061f623>] sk_filter+0x83/0xe0
softirqs last disabled at (3730): [<ffffffff8061f5b6>] sk_filter+0x16/0xe0
other info that might help us debug this:
1 lock held by modprobe/8526:
#0: (testlock){--..}, at: [<ffffffffa0020055>] brd_init+0x55/0x216 [brd]
stack backtrace:
Pid: 8526, comm: modprobe Not tainted 2.6.28-rc6-00007-ged31348-dirty #26
Call Trace:
[<ffffffff80265483>] print_usage_bug+0x193/0x1d0
[<ffffffff80266530>] mark_lock+0xaf0/0xca0
[<ffffffff80266735>] mark_held_locks+0x55/0xc0
[<ffffffffa0020000>] ? brd_init+0x0/0x216 [brd]
[<ffffffff802667ca>] trace_reclaim_fs+0x2a/0x60
[<ffffffff80285005>] __alloc_pages_internal+0x475/0x580
[<ffffffff8070f29e>] ? mutex_lock_nested+0x26e/0x310
[<ffffffffa0020000>] ? brd_init+0x0/0x216 [brd]
[<ffffffffa002006a>] brd_init+0x6a/0x216 [brd]
[<ffffffffa0020000>] ? brd_init+0x0/0x216 [brd]
[<ffffffff8020903b>] _stext+0x3b/0x170
[<ffffffff8070f8b9>] ? mutex_unlock+0x9/0x10
[<ffffffff8070f83d>] ? __mutex_unlock_slowpath+0x10d/0x180
[<ffffffff802669ec>] ? trace_hardirqs_on_caller+0x12c/0x190
[<ffffffff80272ebf>] sys_init_module+0xaf/0x1e0
[<ffffffff8020c3fb>] system_call_fastpath+0x16/0x1b
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-01-21 00:12:39 -07:00
|
|
|
lockdep_trace_alloc(flags);
|
|
|
|
|
2008-12-23 03:37:01 -07:00
|
|
|
if (slab_should_failslab(cachep, flags))
|
2007-05-06 15:49:58 -06:00
|
|
|
return NULL;
|
|
|
|
|
2012-12-18 15:22:48 -07:00
|
|
|
cachep = memcg_kmem_get_cache(cachep, flags);
|
|
|
|
|
2007-02-10 02:42:53 -07:00
|
|
|
cache_alloc_debugcheck_before(cachep, flags);
|
|
|
|
local_irq_save(save_flags);
|
|
|
|
|
2011-07-28 14:59:49 -06:00
|
|
|
if (nodeid == NUMA_NO_NODE)
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
nodeid = slab_node;
|
2007-02-10 02:42:53 -07:00
|
|
|
|
2014-08-06 17:04:11 -06:00
|
|
|
if (unlikely(!get_node(cachep, nodeid))) {
|
2007-02-10 02:42:53 -07:00
|
|
|
/* Node not bootstrapped yet */
|
|
|
|
ptr = fallback_alloc(cachep, flags);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
if (nodeid == slab_node) {
|
2007-02-10 02:42:53 -07:00
|
|
|
/*
|
|
|
|
* Use the locally cached objects if possible.
|
|
|
|
* However ____cache_alloc does not allow fallback
|
|
|
|
* to other nodes. It may fail while we still have
|
|
|
|
* objects on other nodes available.
|
|
|
|
*/
|
|
|
|
ptr = ____cache_alloc(cachep, flags);
|
|
|
|
if (ptr)
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
/* ___cache_alloc_node can fall back to other nodes */
|
|
|
|
ptr = ____cache_alloc_node(cachep, flags, nodeid);
|
|
|
|
out:
|
|
|
|
local_irq_restore(save_flags);
|
|
|
|
ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
|
2012-06-13 09:24:58 -06:00
|
|
|
kmemleak_alloc_recursive(ptr, cachep->object_size, 1, cachep->flags,
|
2009-06-11 06:22:40 -06:00
|
|
|
flags);
|
2007-02-10 02:42:53 -07:00
|
|
|
|
2013-09-10 18:02:51 -06:00
|
|
|
if (likely(ptr)) {
|
2012-06-13 09:24:58 -06:00
|
|
|
kmemcheck_slab_alloc(cachep, flags, ptr, cachep->object_size);
|
2013-09-10 18:02:51 -06:00
|
|
|
if (unlikely(flags & __GFP_ZERO))
|
|
|
|
memset(ptr, 0, cachep->object_size);
|
|
|
|
}
|
2007-07-17 05:03:23 -06:00
|
|
|
|
2007-02-10 02:42:53 -07:00
|
|
|
return ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
static __always_inline void *
|
|
|
|
__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
|
|
|
|
{
|
|
|
|
void *objp;
|
|
|
|
|
2014-09-24 19:41:02 -06:00
|
|
|
if (current->mempolicy || cpuset_do_slab_mem_spread()) {
|
2007-02-10 02:42:53 -07:00
|
|
|
objp = alternate_node_alloc(cache, flags);
|
|
|
|
if (objp)
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
objp = ____cache_alloc(cache, flags);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We may just have run out of memory on the local node.
|
|
|
|
* ____cache_alloc_node() knows how to locate memory on other nodes
|
|
|
|
*/
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
if (!objp)
|
|
|
|
objp = ____cache_alloc_node(cache, flags, numa_mem_id());
|
2007-02-10 02:42:53 -07:00
|
|
|
|
|
|
|
out:
|
|
|
|
return objp;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
|
|
|
|
static __always_inline void *
|
|
|
|
__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
|
|
|
|
{
|
|
|
|
return ____cache_alloc(cachep, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif /* CONFIG_NUMA */
|
|
|
|
|
|
|
|
static __always_inline void *
|
2012-09-08 14:47:57 -06:00
|
|
|
slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
|
2007-02-10 02:42:53 -07:00
|
|
|
{
|
|
|
|
unsigned long save_flags;
|
|
|
|
void *objp;
|
|
|
|
|
2009-06-17 21:24:12 -06:00
|
|
|
flags &= gfp_allowed_mask;
|
2009-06-12 05:03:06 -06:00
|
|
|
|
lockdep: annotate reclaim context (__GFP_NOFS)
Here is another version, with the incremental patch rolled up, and
added reclaim context annotation to kswapd, and allocation tracing
to slab allocators (which may only ever reach the page allocator
in rare cases, so it is good to put annotations here too).
Haven't tested this version as such, but it should be getting closer
to merge worthy ;)
--
After noticing some code in mm/filemap.c accidentally perform a __GFP_FS
allocation when it should not have been, I thought it might be a good idea to
try to catch this kind of thing with lockdep.
I coded up a little idea that seems to work. Unfortunately the system has to
actually be in __GFP_FS page reclaim, then take the lock, before it will mark
it. But at least that might still be some orders of magnitude more common
(and more debuggable) than an actual deadlock condition, so we have some
improvement I hope (the concept is no less complete than discovery of a lock's
interrupt contexts).
I guess we could even do the same thing with __GFP_IO (normal reclaim), and
even GFP_NOIO locks too... but filesystems will have the most locks and fiddly
code paths, so let's start there and see how it goes.
It *seems* to work. I did a quick test.
=================================
[ INFO: inconsistent lock state ]
2.6.28-rc6-00007-ged31348-dirty #26
---------------------------------
inconsistent {in-reclaim-W} -> {ov-reclaim-W} usage.
modprobe/8526 [HC0[0]:SC0[0]:HE1:SE1] takes:
(testlock){--..}, at: [<ffffffffa0020055>] brd_init+0x55/0x216 [brd]
{in-reclaim-W} state was registered at:
[<ffffffff80267bdb>] __lock_acquire+0x75b/0x1a60
[<ffffffff80268f71>] lock_acquire+0x91/0xc0
[<ffffffff8070f0e1>] mutex_lock_nested+0xb1/0x310
[<ffffffffa002002b>] brd_init+0x2b/0x216 [brd]
[<ffffffff8020903b>] _stext+0x3b/0x170
[<ffffffff80272ebf>] sys_init_module+0xaf/0x1e0
[<ffffffff8020c3fb>] system_call_fastpath+0x16/0x1b
[<ffffffffffffffff>] 0xffffffffffffffff
irq event stamp: 3929
hardirqs last enabled at (3929): [<ffffffff8070f2b5>] mutex_lock_nested+0x285/0x310
hardirqs last disabled at (3928): [<ffffffff8070f089>] mutex_lock_nested+0x59/0x310
softirqs last enabled at (3732): [<ffffffff8061f623>] sk_filter+0x83/0xe0
softirqs last disabled at (3730): [<ffffffff8061f5b6>] sk_filter+0x16/0xe0
other info that might help us debug this:
1 lock held by modprobe/8526:
#0: (testlock){--..}, at: [<ffffffffa0020055>] brd_init+0x55/0x216 [brd]
stack backtrace:
Pid: 8526, comm: modprobe Not tainted 2.6.28-rc6-00007-ged31348-dirty #26
Call Trace:
[<ffffffff80265483>] print_usage_bug+0x193/0x1d0
[<ffffffff80266530>] mark_lock+0xaf0/0xca0
[<ffffffff80266735>] mark_held_locks+0x55/0xc0
[<ffffffffa0020000>] ? brd_init+0x0/0x216 [brd]
[<ffffffff802667ca>] trace_reclaim_fs+0x2a/0x60
[<ffffffff80285005>] __alloc_pages_internal+0x475/0x580
[<ffffffff8070f29e>] ? mutex_lock_nested+0x26e/0x310
[<ffffffffa0020000>] ? brd_init+0x0/0x216 [brd]
[<ffffffffa002006a>] brd_init+0x6a/0x216 [brd]
[<ffffffffa0020000>] ? brd_init+0x0/0x216 [brd]
[<ffffffff8020903b>] _stext+0x3b/0x170
[<ffffffff8070f8b9>] ? mutex_unlock+0x9/0x10
[<ffffffff8070f83d>] ? __mutex_unlock_slowpath+0x10d/0x180
[<ffffffff802669ec>] ? trace_hardirqs_on_caller+0x12c/0x190
[<ffffffff80272ebf>] sys_init_module+0xaf/0x1e0
[<ffffffff8020c3fb>] system_call_fastpath+0x16/0x1b
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-01-21 00:12:39 -07:00
|
|
|
lockdep_trace_alloc(flags);
|
|
|
|
|
2008-12-23 03:37:01 -07:00
|
|
|
if (slab_should_failslab(cachep, flags))
|
2007-05-06 15:49:58 -06:00
|
|
|
return NULL;
|
|
|
|
|
2012-12-18 15:22:48 -07:00
|
|
|
cachep = memcg_kmem_get_cache(cachep, flags);
|
|
|
|
|
2007-02-10 02:42:53 -07:00
|
|
|
cache_alloc_debugcheck_before(cachep, flags);
|
|
|
|
local_irq_save(save_flags);
|
|
|
|
objp = __do_cache_alloc(cachep, flags);
|
|
|
|
local_irq_restore(save_flags);
|
|
|
|
objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
|
2012-06-13 09:24:58 -06:00
|
|
|
kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags,
|
2009-06-11 06:22:40 -06:00
|
|
|
flags);
|
2007-02-10 02:42:53 -07:00
|
|
|
prefetchw(objp);
|
|
|
|
|
2013-09-10 18:02:51 -06:00
|
|
|
if (likely(objp)) {
|
2012-06-13 09:24:58 -06:00
|
|
|
kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size);
|
2013-09-10 18:02:51 -06:00
|
|
|
if (unlikely(flags & __GFP_ZERO))
|
|
|
|
memset(objp, 0, cachep->object_size);
|
|
|
|
}
|
2007-07-17 05:03:23 -06:00
|
|
|
|
2007-02-10 02:42:53 -07:00
|
|
|
return objp;
|
|
|
|
}
|
2005-09-09 14:03:32 -06:00
|
|
|
|
|
|
|
/*
|
2014-03-30 03:02:20 -06:00
|
|
|
* Caller needs to acquire correct kmem_cache_node's list_lock
|
2014-08-06 17:04:25 -06:00
|
|
|
* @list: List of detached free slabs should be freed by caller
|
2005-09-09 14:03:32 -06:00
|
|
|
*/
|
2014-08-06 17:04:25 -06:00
|
|
|
static void free_block(struct kmem_cache *cachep, void **objpp,
|
|
|
|
int nr_objects, int node, struct list_head *list)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int i;
|
2014-08-06 17:04:22 -06:00
|
|
|
struct kmem_cache_node *n = get_node(cachep, node);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
for (i = 0; i < nr_objects; i++) {
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
void *objp;
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
clear_obj_pfmemalloc(&objpp[i]);
|
|
|
|
objp = objpp[i];
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
page = virt_to_head_page(objp);
|
|
|
|
list_del(&page->lru);
|
2005-09-22 22:44:02 -06:00
|
|
|
check_spinlock_acquired_node(cachep, node);
|
2013-10-23 19:07:49 -06:00
|
|
|
slab_put_obj(cachep, page, objp, node);
|
2005-04-16 16:20:36 -06:00
|
|
|
STATS_DEC_ACTIVE(cachep);
|
2013-01-10 12:14:19 -07:00
|
|
|
n->free_objects++;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* fixup slab chains */
|
2013-10-23 19:07:49 -06:00
|
|
|
if (page->active == 0) {
|
2013-01-10 12:14:19 -07:00
|
|
|
if (n->free_objects > n->free_limit) {
|
|
|
|
n->free_objects -= cachep->num;
|
2014-08-06 17:04:25 -06:00
|
|
|
list_add_tail(&page->lru, list);
|
2005-04-16 16:20:36 -06:00
|
|
|
} else {
|
2013-10-23 19:07:49 -06:00
|
|
|
list_add(&page->lru, &n->slabs_free);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* Unconditionally move a slab to the end of the
|
|
|
|
* partial list on free - maximum time for the
|
|
|
|
* other objects to be freed, too.
|
|
|
|
*/
|
2013-10-23 19:07:49 -06:00
|
|
|
list_add_tail(&page->lru, &n->slabs_partial);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-02-01 04:05:50 -07:00
|
|
|
static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int batchcount;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
int node = numa_mem_id();
|
2014-08-06 17:04:25 -06:00
|
|
|
LIST_HEAD(list);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
batchcount = ac->batchcount;
|
|
|
|
#if DEBUG
|
|
|
|
BUG_ON(!batchcount || batchcount > ac->avail);
|
|
|
|
#endif
|
|
|
|
check_irq_off();
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock(&n->list_lock);
|
|
|
|
if (n->shared) {
|
|
|
|
struct array_cache *shared_array = n->shared;
|
2006-01-08 02:00:37 -07:00
|
|
|
int max = shared_array->limit - shared_array->avail;
|
2005-04-16 16:20:36 -06:00
|
|
|
if (max) {
|
|
|
|
if (batchcount > max)
|
|
|
|
batchcount = max;
|
2005-09-09 14:03:32 -06:00
|
|
|
memcpy(&(shared_array->entry[shared_array->avail]),
|
2006-01-08 02:00:37 -07:00
|
|
|
ac->entry, sizeof(void *) * batchcount);
|
2005-04-16 16:20:36 -06:00
|
|
|
shared_array->avail += batchcount;
|
|
|
|
goto free_done;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-08-06 17:04:25 -06:00
|
|
|
free_block(cachep, ac->entry, batchcount, node, &list);
|
2006-03-22 01:08:11 -07:00
|
|
|
free_done:
|
2005-04-16 16:20:36 -06:00
|
|
|
#if STATS
|
|
|
|
{
|
|
|
|
int i = 0;
|
|
|
|
struct list_head *p;
|
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
p = n->slabs_free.next;
|
|
|
|
while (p != &(n->slabs_free)) {
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
page = list_entry(p, struct page, lru);
|
|
|
|
BUG_ON(page->active);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
i++;
|
|
|
|
p = p->next;
|
|
|
|
}
|
|
|
|
STATS_SET_FREEABLE(cachep, i);
|
|
|
|
}
|
|
|
|
#endif
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock(&n->list_lock);
|
2014-08-06 17:04:25 -06:00
|
|
|
slabs_destroy(cachep, &list);
|
2005-04-16 16:20:36 -06:00
|
|
|
ac->avail -= batchcount;
|
2006-03-22 01:08:11 -07:00
|
|
|
memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2006-03-22 01:08:11 -07:00
|
|
|
* Release an obj back to its cache. If the obj has a constructed state, it must
|
|
|
|
* be in this state _before_ it is released. Called with disabled ints.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2011-06-02 01:16:42 -06:00
|
|
|
static inline void __cache_free(struct kmem_cache *cachep, void *objp,
|
2012-09-08 14:47:55 -06:00
|
|
|
unsigned long caller)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-02-01 04:05:49 -07:00
|
|
|
struct array_cache *ac = cpu_cache_get(cachep);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
check_irq_off();
|
2009-06-11 06:22:40 -06:00
|
|
|
kmemleak_free_recursive(objp, cachep->flags);
|
2011-06-02 01:16:42 -06:00
|
|
|
objp = cache_free_debugcheck(cachep, objp, caller);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-06-13 09:24:58 -06:00
|
|
|
kmemcheck_slab_free(cachep, objp, cachep->object_size);
|
2008-05-09 12:35:53 -06:00
|
|
|
|
2007-08-22 15:01:49 -06:00
|
|
|
/*
|
|
|
|
* Skip calling cache_free_alien() when the platform is not numa.
|
|
|
|
* This will avoid cache misses that happen while accessing slabp (which
|
|
|
|
* is per page memory reference) to get nodeid. Instead use a global
|
|
|
|
* variable to skip the call, which is mostly likely to be present in
|
|
|
|
* the cache.
|
|
|
|
*/
|
2009-06-16 16:32:16 -06:00
|
|
|
if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
|
2006-06-23 03:03:05 -06:00
|
|
|
return;
|
|
|
|
|
2014-10-09 16:26:04 -06:00
|
|
|
if (ac->avail < ac->limit) {
|
2005-04-16 16:20:36 -06:00
|
|
|
STATS_INC_FREEHIT(cachep);
|
|
|
|
} else {
|
|
|
|
STATS_INC_FREEMISS(cachep);
|
|
|
|
cache_flusharray(cachep, ac);
|
|
|
|
}
|
2011-08-26 10:26:17 -06:00
|
|
|
|
mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
When a user or administrator requires swap for their application, they
create a swap partition and file, format it with mkswap and activate it
with swapon. Swap over the network is considered as an option in diskless
systems. The two likely scenarios are when blade servers are used as part
of a cluster where the form factor or maintenance costs do not allow the
use of disks and thin clients.
The Linux Terminal Server Project recommends the use of the Network Block
Device (NBD) for swap according to the manual at
https://sourceforge.net/projects/ltsp/files/Docs-Admin-Guide/LTSPManual.pdf/download
There is also documentation and tutorials on how to setup swap over NBD at
places like https://help.ubuntu.com/community/UbuntuLTSP/EnableNBDSWAP The
nbd-client also documents the use of NBD as swap. Despite this, the fact
is that a machine using NBD for swap can deadlock within minutes if swap
is used intensively. This patch series addresses the problem.
The core issue is that network block devices do not use mempools like
normal block devices do. As the host cannot control where they receive
packets from, they cannot reliably work out in advance how much memory
they might need. Some years ago, Peter Zijlstra developed a series of
patches that supported swap over an NFS that at least one distribution is
carrying within their kernels. This patch series borrows very heavily
from Peter's work to support swapping over NBD as a pre-requisite to
supporting swap-over-NFS. The bulk of the complexity is concerned with
preserving memory that is allocated from the PFMEMALLOC reserves for use
by the network layer which is needed for both NBD and NFS.
Patch 1 adds knowledge of the PFMEMALLOC reserves to SLAB and SLUB to
preserve access to pages allocated under low memory situations
to callers that are freeing memory.
Patch 2 optimises the SLUB fast path to avoid pfmemalloc checks
Patch 3 introduces __GFP_MEMALLOC to allow access to the PFMEMALLOC
reserves without setting PFMEMALLOC.
Patch 4 opens the possibility for softirqs to use PFMEMALLOC reserves
for later use by network packet processing.
Patch 5 only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was required
Patch 6 ignores memory policies when ALLOC_NO_WATERMARKS is set.
Patches 7-12 allows network processing to use PFMEMALLOC reserves when
the socket has been marked as being used by the VM to clean pages. If
packets are received and stored in pages that were allocated under
low-memory situations and are unrelated to the VM, the packets
are dropped.
Patch 11 reintroduces __skb_alloc_page which the networking
folk may object to but is needed in some cases to propogate
pfmemalloc from a newly allocated page to an skb. If there is a
strong objection, this patch can be dropped with the impact being
that swap-over-network will be slower in some cases but it should
not fail.
Patch 13 is a micro-optimisation to avoid a function call in the
common case.
Patch 14 tags NBD sockets as being SOCK_MEMALLOC so they can use
PFMEMALLOC if necessary.
Patch 15 notes that it is still possible for the PFMEMALLOC reserve
to be depleted. To prevent this, direct reclaimers get throttled on
a waitqueue if 50% of the PFMEMALLOC reserves are depleted. It is
expected that kswapd and the direct reclaimers already running
will clean enough pages for the low watermark to be reached and
the throttled processes are woken up.
Patch 16 adds a statistic to track how often processes get throttled
Some basic performance testing was run using kernel builds, netperf on
loopback for UDP and TCP, hackbench (pipes and sockets), iozone and
sysbench. Each of them were expected to use the sl*b allocators
reasonably heavily but there did not appear to be significant performance
variances.
For testing swap-over-NBD, a machine was booted with 2G of RAM with a
swapfile backed by NBD. 8*NUM_CPU processes were started that create
anonymous memory mappings and read them linearly in a loop. The total
size of the mappings were 4*PHYSICAL_MEMORY to use swap heavily under
memory pressure.
Without the patches and using SLUB, the machine locks up within minutes
and runs to completion with them applied. With SLAB, the story is
different as an unpatched kernel run to completion. However, the patched
kernel completed the test 45% faster.
MICRO
3.5.0-rc2 3.5.0-rc2
vanilla swapnbd
Unrecognised test vmscan-anon-mmap-write
MMTests Statistics: duration
Sys Time Running Test (seconds) 197.80 173.07
User+Sys Time Running Test (seconds) 206.96 182.03
Total Elapsed Time (seconds) 3240.70 1762.09
This patch: mm: sl[au]b: add knowledge of PFMEMALLOC reserve pages
Allocations of pages below the min watermark run a risk of the machine
hanging due to a lack of memory. To prevent this, only callers who have
PF_MEMALLOC or TIF_MEMDIE set and are not processing an interrupt are
allowed to allocate with ALLOC_NO_WATERMARKS. Once they are allocated to
a slab though, nothing prevents other callers consuming free objects
within those slabs. This patch limits access to slab pages that were
alloced from the PFMEMALLOC reserves.
When this patch is applied, pages allocated from below the low watermark
are returned with page->pfmemalloc set and it is up to the caller to
determine how the page should be protected. SLAB restricts access to any
page with page->pfmemalloc set to callers which are known to able to
access the PFMEMALLOC reserve. If one is not available, an attempt is
made to allocate a new page rather than use a reserve. SLUB is a bit more
relaxed in that it only records if the current per-CPU page was allocated
from PFMEMALLOC reserve and uses another partial slab if the caller does
not have the necessary GFP or process flags. This was found to be
sufficient in tests to avoid hangs due to SLUB generally maintaining
smaller lists than SLAB.
In low-memory conditions it does mean that !PFMEMALLOC allocators can fail
a slab allocation even though free objects are available because they are
being preserved for callers that are freeing pages.
[a.p.zijlstra@chello.nl: Original implementation]
[sebastian@breakpoint.cc: Correct order of page flag clearing]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: David Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-07-31 17:43:58 -06:00
|
|
|
ac_put_obj(cachep, ac, objp);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* kmem_cache_alloc - Allocate an object
|
|
|
|
* @cachep: The cache to allocate from.
|
|
|
|
* @flags: See kmalloc().
|
|
|
|
*
|
|
|
|
* Allocate an object from this cache. The flags are only relevant
|
|
|
|
* if the cache has no available objects.
|
|
|
|
*/
|
2006-02-01 04:05:50 -07:00
|
|
|
void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2012-09-08 14:47:57 -06:00
|
|
|
void *ret = slab_alloc(cachep, flags, _RET_IP_);
|
2008-08-10 11:14:05 -06:00
|
|
|
|
2009-03-23 07:12:24 -06:00
|
|
|
trace_kmem_cache_alloc(_RET_IP_, ret,
|
2012-06-13 09:24:58 -06:00
|
|
|
cachep->object_size, cachep->size, flags);
|
2008-08-10 11:14:05 -06:00
|
|
|
|
|
|
|
return ret;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kmem_cache_alloc);
|
|
|
|
|
2009-12-11 00:45:30 -07:00
|
|
|
#ifdef CONFIG_TRACING
|
2010-11-24 14:23:34 -07:00
|
|
|
void *
|
2012-09-08 14:47:56 -06:00
|
|
|
kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
|
2008-08-10 11:14:05 -06:00
|
|
|
{
|
2010-11-24 14:23:34 -07:00
|
|
|
void *ret;
|
|
|
|
|
2012-09-08 14:47:57 -06:00
|
|
|
ret = slab_alloc(cachep, flags, _RET_IP_);
|
2010-11-24 14:23:34 -07:00
|
|
|
|
|
|
|
trace_kmalloc(_RET_IP_, ret,
|
2012-09-08 14:47:52 -06:00
|
|
|
size, cachep->size, flags);
|
2010-11-24 14:23:34 -07:00
|
|
|
return ret;
|
2008-08-10 11:14:05 -06:00
|
|
|
}
|
2010-11-24 14:23:34 -07:00
|
|
|
EXPORT_SYMBOL(kmem_cache_alloc_trace);
|
2008-08-10 11:14:05 -06:00
|
|
|
#endif
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#ifdef CONFIG_NUMA
|
2013-05-15 21:36:23 -06:00
|
|
|
/**
|
|
|
|
* kmem_cache_alloc_node - Allocate an object on the specified node
|
|
|
|
* @cachep: The cache to allocate from.
|
|
|
|
* @flags: See kmalloc().
|
|
|
|
* @nodeid: node number of the target node.
|
|
|
|
*
|
|
|
|
* Identical to kmem_cache_alloc but it will allocate memory on the given
|
|
|
|
* node, which can improve the performance for cpu bound structures.
|
|
|
|
*
|
|
|
|
* Fallback to other node is possible if __GFP_THISNODE is not set.
|
|
|
|
*/
|
2006-12-06 21:32:30 -07:00
|
|
|
void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
|
|
|
|
{
|
2012-09-08 14:47:57 -06:00
|
|
|
void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
|
2008-08-10 11:14:05 -06:00
|
|
|
|
2009-03-23 07:12:24 -06:00
|
|
|
trace_kmem_cache_alloc_node(_RET_IP_, ret,
|
2012-06-13 09:24:58 -06:00
|
|
|
cachep->object_size, cachep->size,
|
2009-03-23 07:12:24 -06:00
|
|
|
flags, nodeid);
|
2008-08-10 11:14:05 -06:00
|
|
|
|
|
|
|
return ret;
|
2006-12-06 21:32:30 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
EXPORT_SYMBOL(kmem_cache_alloc_node);
|
|
|
|
|
2009-12-11 00:45:30 -07:00
|
|
|
#ifdef CONFIG_TRACING
|
2012-09-08 14:47:56 -06:00
|
|
|
void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
|
2010-11-24 14:23:34 -07:00
|
|
|
gfp_t flags,
|
2012-09-08 14:47:56 -06:00
|
|
|
int nodeid,
|
|
|
|
size_t size)
|
2008-08-10 11:14:05 -06:00
|
|
|
{
|
2010-11-24 14:23:34 -07:00
|
|
|
void *ret;
|
|
|
|
|
2012-09-25 05:07:08 -06:00
|
|
|
ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
|
2012-09-08 14:47:55 -06:00
|
|
|
|
2010-11-24 14:23:34 -07:00
|
|
|
trace_kmalloc_node(_RET_IP_, ret,
|
2012-09-08 14:47:52 -06:00
|
|
|
size, cachep->size,
|
2010-11-24 14:23:34 -07:00
|
|
|
flags, nodeid);
|
|
|
|
return ret;
|
2008-08-10 11:14:05 -06:00
|
|
|
}
|
2010-11-24 14:23:34 -07:00
|
|
|
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
|
2008-08-10 11:14:05 -06:00
|
|
|
#endif
|
|
|
|
|
2006-12-06 21:32:30 -07:00
|
|
|
static __always_inline void *
|
2012-09-08 14:47:55 -06:00
|
|
|
__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
|
2005-05-01 09:58:38 -06:00
|
|
|
{
|
2006-02-01 04:05:50 -07:00
|
|
|
struct kmem_cache *cachep;
|
2005-05-01 09:58:38 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
cachep = kmalloc_slab(size, flags);
|
2007-07-17 05:03:22 -06:00
|
|
|
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
|
|
|
|
return cachep;
|
2012-09-08 14:47:56 -06:00
|
|
|
return kmem_cache_alloc_node_trace(cachep, flags, node, size);
|
2005-05-01 09:58:38 -06:00
|
|
|
}
|
2006-12-06 21:32:30 -07:00
|
|
|
|
|
|
|
void *__kmalloc_node(size_t size, gfp_t flags, int node)
|
|
|
|
{
|
2012-09-08 14:47:55 -06:00
|
|
|
return __do_kmalloc_node(size, flags, node, _RET_IP_);
|
2006-12-06 21:32:30 -07:00
|
|
|
}
|
2006-09-26 00:31:36 -06:00
|
|
|
EXPORT_SYMBOL(__kmalloc_node);
|
2006-12-06 21:32:30 -07:00
|
|
|
|
|
|
|
void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
|
2008-08-19 11:43:25 -06:00
|
|
|
int node, unsigned long caller)
|
2006-12-06 21:32:30 -07:00
|
|
|
{
|
2012-09-08 14:47:55 -06:00
|
|
|
return __do_kmalloc_node(size, flags, node, caller);
|
2006-12-06 21:32:30 -07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__kmalloc_node_track_caller);
|
|
|
|
#endif /* CONFIG_NUMA */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/**
|
2006-06-23 03:03:48 -06:00
|
|
|
* __do_kmalloc - allocate memory
|
2005-04-16 16:20:36 -06:00
|
|
|
* @size: how many bytes of memory are required.
|
2006-06-23 03:03:48 -06:00
|
|
|
* @flags: the type of memory to allocate (see kmalloc).
|
2006-03-22 01:08:14 -07:00
|
|
|
* @caller: function caller for debug tracking of the caller
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2006-02-01 04:05:52 -07:00
|
|
|
static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
|
2012-09-08 14:47:55 -06:00
|
|
|
unsigned long caller)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-02-01 04:05:50 -07:00
|
|
|
struct kmem_cache *cachep;
|
2008-08-10 11:14:05 -06:00
|
|
|
void *ret;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
cachep = kmalloc_slab(size, flags);
|
2007-07-19 14:17:15 -06:00
|
|
|
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
|
|
|
|
return cachep;
|
2012-09-08 14:47:57 -06:00
|
|
|
ret = slab_alloc(cachep, flags, caller);
|
2008-08-10 11:14:05 -06:00
|
|
|
|
2012-09-08 14:47:55 -06:00
|
|
|
trace_kmalloc(caller, ret,
|
2012-06-13 09:24:57 -06:00
|
|
|
size, cachep->size, flags);
|
2008-08-10 11:14:05 -06:00
|
|
|
|
|
|
|
return ret;
|
2006-02-01 04:05:52 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
void *__kmalloc(size_t size, gfp_t flags)
|
|
|
|
{
|
2012-09-08 14:47:55 -06:00
|
|
|
return __do_kmalloc(size, flags, _RET_IP_);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__kmalloc);
|
|
|
|
|
2008-08-19 11:43:25 -06:00
|
|
|
void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
|
2006-02-01 04:05:52 -07:00
|
|
|
{
|
2012-09-08 14:47:55 -06:00
|
|
|
return __do_kmalloc(size, flags, caller);
|
2006-02-01 04:05:52 -07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__kmalloc_track_caller);
|
2006-10-04 03:15:25 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/**
|
|
|
|
* kmem_cache_free - Deallocate an object
|
|
|
|
* @cachep: The cache the allocation was from.
|
|
|
|
* @objp: The previously allocated object.
|
|
|
|
*
|
|
|
|
* Free an object which was previously allocated from this
|
|
|
|
* cache.
|
|
|
|
*/
|
2006-02-01 04:05:50 -07:00
|
|
|
void kmem_cache_free(struct kmem_cache *cachep, void *objp)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
2012-12-18 15:22:46 -07:00
|
|
|
cachep = cache_from_obj(cachep, objp);
|
|
|
|
if (!cachep)
|
|
|
|
return;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
local_irq_save(flags);
|
2012-07-02 00:29:10 -06:00
|
|
|
debug_check_no_locks_freed(objp, cachep->object_size);
|
2008-04-30 01:55:01 -06:00
|
|
|
if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
|
2012-06-13 09:24:58 -06:00
|
|
|
debug_check_no_obj_freed(objp, cachep->object_size);
|
2012-09-08 14:47:55 -06:00
|
|
|
__cache_free(cachep, objp, _RET_IP_);
|
2005-04-16 16:20:36 -06:00
|
|
|
local_irq_restore(flags);
|
2008-08-10 11:14:05 -06:00
|
|
|
|
2009-03-23 07:12:24 -06:00
|
|
|
trace_kmem_cache_free(_RET_IP_, objp);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kmem_cache_free);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* kfree - free previously allocated memory
|
|
|
|
* @objp: pointer returned by kmalloc.
|
|
|
|
*
|
2005-09-09 14:10:16 -06:00
|
|
|
* If @objp is NULL, no operation is performed.
|
|
|
|
*
|
2005-04-16 16:20:36 -06:00
|
|
|
* Don't free memory not originally allocated by kmalloc()
|
|
|
|
* or you will run into trouble.
|
|
|
|
*/
|
|
|
|
void kfree(const void *objp)
|
|
|
|
{
|
2006-02-01 04:05:50 -07:00
|
|
|
struct kmem_cache *c;
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned long flags;
|
|
|
|
|
2009-03-25 03:05:57 -06:00
|
|
|
trace_kfree(_RET_IP_, objp);
|
|
|
|
|
2007-07-17 05:03:22 -06:00
|
|
|
if (unlikely(ZERO_OR_NULL_PTR(objp)))
|
2005-04-16 16:20:36 -06:00
|
|
|
return;
|
|
|
|
local_irq_save(flags);
|
|
|
|
kfree_debugcheck(objp);
|
2006-02-01 04:05:49 -07:00
|
|
|
c = virt_to_cache(objp);
|
2012-06-13 09:24:58 -06:00
|
|
|
debug_check_no_locks_freed(objp, c->object_size);
|
|
|
|
|
|
|
|
debug_check_no_obj_freed(objp, c->object_size);
|
2012-09-08 14:47:55 -06:00
|
|
|
__cache_free(c, (void *)objp, _RET_IP_);
|
2005-04-16 16:20:36 -06:00
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(kfree);
|
|
|
|
|
2005-09-09 14:03:32 -06:00
|
|
|
/*
|
2013-01-10 12:14:19 -07:00
|
|
|
* This initializes kmem_cache_node or resizes various caches for all nodes.
|
2005-09-09 14:03:32 -06:00
|
|
|
*/
|
2014-03-30 03:02:20 -06:00
|
|
|
static int alloc_kmem_cache_node(struct kmem_cache *cachep, gfp_t gfp)
|
2005-09-09 14:03:32 -06:00
|
|
|
{
|
|
|
|
int node;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2006-03-25 04:06:46 -07:00
|
|
|
struct array_cache *new_shared;
|
2014-08-06 17:04:29 -06:00
|
|
|
struct alien_cache **new_alien = NULL;
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2008-01-24 06:49:54 -07:00
|
|
|
for_each_online_node(node) {
|
2006-03-25 04:06:46 -07:00
|
|
|
|
2006-12-06 21:32:16 -07:00
|
|
|
if (use_alien_caches) {
|
2009-06-10 10:40:04 -06:00
|
|
|
new_alien = alloc_alien_cache(node, cachep->limit, gfp);
|
2006-12-06 21:32:16 -07:00
|
|
|
if (!new_alien)
|
|
|
|
goto fail;
|
|
|
|
}
|
2006-03-25 04:06:46 -07:00
|
|
|
|
2007-05-06 15:49:28 -06:00
|
|
|
new_shared = NULL;
|
|
|
|
if (cachep->shared) {
|
|
|
|
new_shared = alloc_arraycache(node,
|
2006-03-25 04:06:47 -07:00
|
|
|
cachep->shared*cachep->batchcount,
|
2009-06-10 10:40:04 -06:00
|
|
|
0xbaadf00d, gfp);
|
2007-05-06 15:49:28 -06:00
|
|
|
if (!new_shared) {
|
|
|
|
free_alien_cache(new_alien);
|
|
|
|
goto fail;
|
|
|
|
}
|
2006-03-25 04:06:47 -07:00
|
|
|
}
|
2006-03-25 04:06:46 -07:00
|
|
|
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
2013-01-10 12:14:19 -07:00
|
|
|
if (n) {
|
|
|
|
struct array_cache *shared = n->shared;
|
2014-08-06 17:04:25 -06:00
|
|
|
LIST_HEAD(list);
|
2006-03-25 04:06:46 -07:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock_irq(&n->list_lock);
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2006-03-25 04:06:46 -07:00
|
|
|
if (shared)
|
2006-03-25 04:06:47 -07:00
|
|
|
free_block(cachep, shared->entry,
|
2014-08-06 17:04:25 -06:00
|
|
|
shared->avail, node, &list);
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
n->shared = new_shared;
|
|
|
|
if (!n->alien) {
|
|
|
|
n->alien = new_alien;
|
2005-09-09 14:03:32 -06:00
|
|
|
new_alien = NULL;
|
|
|
|
}
|
2013-01-10 12:14:19 -07:00
|
|
|
n->free_limit = (1 + nr_cpus_node(node)) *
|
2006-03-22 01:08:11 -07:00
|
|
|
cachep->batchcount + cachep->num;
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2014-08-06 17:04:25 -06:00
|
|
|
slabs_destroy(cachep, &list);
|
2006-03-25 04:06:46 -07:00
|
|
|
kfree(shared);
|
2005-09-09 14:03:32 -06:00
|
|
|
free_alien_cache(new_alien);
|
|
|
|
continue;
|
|
|
|
}
|
2013-01-10 12:14:19 -07:00
|
|
|
n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
|
|
|
|
if (!n) {
|
2006-03-25 04:06:47 -07:00
|
|
|
free_alien_cache(new_alien);
|
|
|
|
kfree(new_shared);
|
2005-09-09 14:03:32 -06:00
|
|
|
goto fail;
|
2006-03-25 04:06:47 -07:00
|
|
|
}
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
kmem_cache_node_init(n);
|
2014-03-30 03:02:20 -06:00
|
|
|
n->next_reap = jiffies + REAPTIMEOUT_NODE +
|
|
|
|
((unsigned long)cachep) % REAPTIMEOUT_NODE;
|
2013-01-10 12:14:19 -07:00
|
|
|
n->shared = new_shared;
|
|
|
|
n->alien = new_alien;
|
|
|
|
n->free_limit = (1 + nr_cpus_node(node)) *
|
2006-03-22 01:08:11 -07:00
|
|
|
cachep->batchcount + cachep->num;
|
2013-01-10 12:14:19 -07:00
|
|
|
cachep->node[node] = n;
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
2006-03-25 04:06:46 -07:00
|
|
|
return 0;
|
2006-03-25 04:06:47 -07:00
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
fail:
|
2012-06-13 09:24:57 -06:00
|
|
|
if (!cachep->list.next) {
|
2006-03-25 04:06:47 -07:00
|
|
|
/* Cache is not active yet. Roll back what we did */
|
|
|
|
node--;
|
|
|
|
while (node >= 0) {
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
|
|
|
if (n) {
|
2013-01-10 12:14:19 -07:00
|
|
|
kfree(n->shared);
|
|
|
|
free_alien_cache(n->alien);
|
|
|
|
kfree(n);
|
2013-01-10 12:14:19 -07:00
|
|
|
cachep->node[node] = NULL;
|
2006-03-25 04:06:47 -07:00
|
|
|
}
|
|
|
|
node--;
|
|
|
|
}
|
|
|
|
}
|
2006-03-25 04:06:46 -07:00
|
|
|
return -ENOMEM;
|
2005-09-09 14:03:32 -06:00
|
|
|
}
|
|
|
|
|
2012-07-06 14:25:12 -06:00
|
|
|
/* Always called with the slab_mutex held */
|
2012-12-18 15:23:03 -07:00
|
|
|
static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
|
2009-06-10 10:40:04 -06:00
|
|
|
int batchcount, int shared, gfp_t gfp)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2014-10-09 16:26:27 -06:00
|
|
|
struct array_cache __percpu *cpu_cache, *prev;
|
|
|
|
int cpu;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2014-10-09 16:26:27 -06:00
|
|
|
cpu_cache = alloc_kmem_cache_cpus(cachep, limit, batchcount);
|
|
|
|
if (!cpu_cache)
|
2006-09-26 00:31:47 -06:00
|
|
|
return -ENOMEM;
|
|
|
|
|
2014-10-09 16:26:27 -06:00
|
|
|
prev = cachep->cpu_cache;
|
|
|
|
cachep->cpu_cache = cpu_cache;
|
|
|
|
kick_all_cpus_sync();
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
check_irq_on();
|
|
|
|
cachep->batchcount = batchcount;
|
|
|
|
cachep->limit = limit;
|
2005-09-09 14:03:32 -06:00
|
|
|
cachep->shared = shared;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2014-10-09 16:26:27 -06:00
|
|
|
if (!prev)
|
|
|
|
goto alloc_node;
|
|
|
|
|
|
|
|
for_each_online_cpu(cpu) {
|
2014-08-06 17:04:25 -06:00
|
|
|
LIST_HEAD(list);
|
2014-08-06 17:04:11 -06:00
|
|
|
int node;
|
|
|
|
struct kmem_cache_node *n;
|
2014-10-09 16:26:27 -06:00
|
|
|
struct array_cache *ac = per_cpu_ptr(prev, cpu);
|
2014-08-06 17:04:11 -06:00
|
|
|
|
2014-10-09 16:26:27 -06:00
|
|
|
node = cpu_to_mem(cpu);
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(cachep, node);
|
|
|
|
spin_lock_irq(&n->list_lock);
|
2014-10-09 16:26:27 -06:00
|
|
|
free_block(cachep, ac->entry, ac->avail, node, &list);
|
2014-08-06 17:04:11 -06:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2014-08-06 17:04:25 -06:00
|
|
|
slabs_destroy(cachep, &list);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2014-10-09 16:26:27 -06:00
|
|
|
free_percpu(prev);
|
|
|
|
|
|
|
|
alloc_node:
|
2014-03-30 03:02:20 -06:00
|
|
|
return alloc_kmem_cache_node(cachep, gfp);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2012-12-18 15:23:03 -07:00
|
|
|
static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
|
|
|
|
int batchcount, int shared, gfp_t gfp)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
struct kmem_cache *c = NULL;
|
|
|
|
int i = 0;
|
|
|
|
|
|
|
|
ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
|
|
|
|
|
|
|
|
if (slab_state < FULL)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if ((ret < 0) || !is_root_cache(cachep))
|
|
|
|
return ret;
|
|
|
|
|
2012-12-18 15:23:10 -07:00
|
|
|
VM_BUG_ON(!mutex_is_locked(&slab_mutex));
|
2012-12-18 15:23:03 -07:00
|
|
|
for_each_memcg_cache_index(i) {
|
2013-11-12 16:08:23 -07:00
|
|
|
c = cache_from_memcg_idx(cachep, i);
|
2012-12-18 15:23:03 -07:00
|
|
|
if (c)
|
|
|
|
/* return value determined by the parent cache only */
|
|
|
|
__do_tune_cpucache(c, limit, batchcount, shared, gfp);
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2012-07-06 14:25:12 -06:00
|
|
|
/* Called with slab_mutex held always */
|
2009-06-10 10:40:04 -06:00
|
|
|
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int err;
|
2012-12-18 15:23:03 -07:00
|
|
|
int limit = 0;
|
|
|
|
int shared = 0;
|
|
|
|
int batchcount = 0;
|
|
|
|
|
|
|
|
if (!is_root_cache(cachep)) {
|
|
|
|
struct kmem_cache *root = memcg_root_cache(cachep);
|
|
|
|
limit = root->limit;
|
|
|
|
shared = root->shared;
|
|
|
|
batchcount = root->batchcount;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-12-18 15:23:03 -07:00
|
|
|
if (limit && shared && batchcount)
|
|
|
|
goto skip_setup;
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* The head array serves three purposes:
|
2005-04-16 16:20:36 -06:00
|
|
|
* - create a LIFO ordering, i.e. return objects that are cache-warm
|
|
|
|
* - reduce the number of spinlock operations.
|
2006-03-22 01:08:11 -07:00
|
|
|
* - reduce the number of linked list operations on the slab and
|
2005-04-16 16:20:36 -06:00
|
|
|
* bufctl chains: array operations are cheaper.
|
|
|
|
* The numbers are guessed, we should auto-tune as described by
|
|
|
|
* Bonwick.
|
|
|
|
*/
|
2012-06-13 09:24:57 -06:00
|
|
|
if (cachep->size > 131072)
|
2005-04-16 16:20:36 -06:00
|
|
|
limit = 1;
|
2012-06-13 09:24:57 -06:00
|
|
|
else if (cachep->size > PAGE_SIZE)
|
2005-04-16 16:20:36 -06:00
|
|
|
limit = 8;
|
2012-06-13 09:24:57 -06:00
|
|
|
else if (cachep->size > 1024)
|
2005-04-16 16:20:36 -06:00
|
|
|
limit = 24;
|
2012-06-13 09:24:57 -06:00
|
|
|
else if (cachep->size > 256)
|
2005-04-16 16:20:36 -06:00
|
|
|
limit = 54;
|
|
|
|
else
|
|
|
|
limit = 120;
|
|
|
|
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* CPU bound tasks (e.g. network routing) can exhibit cpu bound
|
2005-04-16 16:20:36 -06:00
|
|
|
* allocation behaviour: Most allocs on one cpu, most free operations
|
|
|
|
* on another cpu. For these cases, an efficient object passing between
|
|
|
|
* cpus is necessary. This is provided by a shared array. The array
|
|
|
|
* replaces Bonwick's magazine layer.
|
|
|
|
* On uniprocessor, it's functionally equivalent (but less efficient)
|
|
|
|
* to a larger limit. Thus disabled by default.
|
|
|
|
*/
|
|
|
|
shared = 0;
|
2012-06-13 09:24:57 -06:00
|
|
|
if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
|
2005-04-16 16:20:36 -06:00
|
|
|
shared = 8;
|
|
|
|
|
|
|
|
#if DEBUG
|
2006-03-22 01:08:11 -07:00
|
|
|
/*
|
|
|
|
* With debugging enabled, large batchcount lead to excessively long
|
|
|
|
* periods with disabled local interrupts. Limit the batchcount
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
if (limit > 32)
|
|
|
|
limit = 32;
|
|
|
|
#endif
|
2012-12-18 15:23:03 -07:00
|
|
|
batchcount = (limit + 1) / 2;
|
|
|
|
skip_setup:
|
|
|
|
err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (err)
|
|
|
|
printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
|
2006-01-08 02:00:37 -07:00
|
|
|
cachep->name, -err);
|
2006-09-26 00:31:38 -06:00
|
|
|
return err;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-03-22 01:09:07 -07:00
|
|
|
/*
|
2013-01-10 12:14:19 -07:00
|
|
|
* Drain an array if it contains any elements taking the node lock only if
|
|
|
|
* necessary. Note that the node listlock also protects the array_cache
|
2006-03-22 01:09:07 -07:00
|
|
|
* if drain_array() is used on the shared array.
|
2006-03-22 01:09:07 -07:00
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
|
2006-03-22 01:09:07 -07:00
|
|
|
struct array_cache *ac, int force, int node)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2014-08-06 17:04:25 -06:00
|
|
|
LIST_HEAD(list);
|
2005-04-16 16:20:36 -06:00
|
|
|
int tofree;
|
|
|
|
|
2006-03-22 01:09:07 -07:00
|
|
|
if (!ac || !ac->avail)
|
|
|
|
return;
|
2005-04-16 16:20:36 -06:00
|
|
|
if (ac->touched && !force) {
|
|
|
|
ac->touched = 0;
|
2006-03-22 01:09:07 -07:00
|
|
|
} else {
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock_irq(&n->list_lock);
|
2006-03-22 01:09:07 -07:00
|
|
|
if (ac->avail) {
|
|
|
|
tofree = force ? ac->avail : (ac->limit + 4) / 5;
|
|
|
|
if (tofree > ac->avail)
|
|
|
|
tofree = (ac->avail + 1) / 2;
|
2014-08-06 17:04:25 -06:00
|
|
|
free_block(cachep, ac->entry, tofree, node, &list);
|
2006-03-22 01:09:07 -07:00
|
|
|
ac->avail -= tofree;
|
|
|
|
memmove(ac->entry, &(ac->entry[tofree]),
|
|
|
|
sizeof(void *) * ac->avail);
|
|
|
|
}
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2014-08-06 17:04:25 -06:00
|
|
|
slabs_destroy(cachep, &list);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* cache_reap - Reclaim memory from caches.
|
2007-02-28 21:12:13 -07:00
|
|
|
* @w: work descriptor
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* Called from workqueue/eventd every few seconds.
|
|
|
|
* Purpose:
|
|
|
|
* - clear the per-cpu caches for this CPU.
|
|
|
|
* - return freeable pages to the main free memory pool.
|
|
|
|
*
|
2006-03-22 01:08:11 -07:00
|
|
|
* If we cannot acquire the cache chain mutex then just give up - we'll try
|
|
|
|
* again on the next iteration.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2007-02-10 02:42:55 -07:00
|
|
|
static void cache_reap(struct work_struct *w)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-06-23 03:03:17 -06:00
|
|
|
struct kmem_cache *searchp;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
numa: slab: use numa_mem_id() for slab local memory node
Example usage of generic "numa_mem_id()":
The mainline slab code, since ~ 2.6.19, does not handle memoryless nodes
well. Specifically, the "fast path"--____cache_alloc()--will never
succeed as slab doesn't cache offnode object on the per cpu queues, and
for memoryless nodes, all memory will be "off node" relative to
numa_node_id(). This adds significant overhead to all kmem cache
allocations, incurring a significant regression relative to earlier
kernels [from before slab.c was reorganized].
This patch uses the generic topology function "numa_mem_id()" to return
the "effective local memory node" for the calling context. This is the
first node in the local node's generic fallback zonelist-- the same node
that "local" mempolicy-based allocations would use. This lets slab cache
these "local" allocations and avoid fallback/refill on every allocation.
N.B.: Slab will need to handle node and memory hotplug events that could
change the value returned by numa_mem_id() for any given node if recent
changes to address memory hotplug don't already address this. E.g., flush
all per cpu slab queues before rebuilding the zonelists while the
"machine" is held in the stopped state.
Performance impact on "hackbench 400 process 200"
2.6.34-rc3-mmotm-100405-1609 no-patch this-patch
ia64 no memoryless nodes [avg of 10]: 11.713 11.637 ~0.65 diff
ia64 cpus all on memless nodes [10]: 228.259 26.484 ~8.6x speedup
The slowdown of the patched kernel from ~12 sec to ~28 seconds when
configured with memoryless nodes is the result of all cpus allocating from
a single node's mm pagepool. The cache lines of the single node are
distributed/interleaved over the memory of the real physical nodes, but
the zone lock, list heads, ... of the single node with memory still each
live in a single cache line that is accessed from all processors.
x86_64 [8x6 AMD] [avg of 40]: 2.883 2.845
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Nick Piggin <npiggin@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-05-26 15:45:03 -06:00
|
|
|
int node = numa_mem_id();
|
2009-04-02 17:56:54 -06:00
|
|
|
struct delayed_work *work = to_delayed_work(w);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-07-06 14:25:12 -06:00
|
|
|
if (!mutex_trylock(&slab_mutex))
|
2005-04-16 16:20:36 -06:00
|
|
|
/* Give up. Setup the next iteration. */
|
2007-02-10 02:42:55 -07:00
|
|
|
goto out;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-07-06 14:25:12 -06:00
|
|
|
list_for_each_entry(searchp, &slab_caches, list) {
|
2005-04-16 16:20:36 -06:00
|
|
|
check_irq_on();
|
|
|
|
|
2006-03-22 01:09:05 -07:00
|
|
|
/*
|
2013-01-10 12:14:19 -07:00
|
|
|
* We only take the node lock if absolutely necessary and we
|
2006-03-22 01:09:05 -07:00
|
|
|
* have established with reasonable certainty that
|
|
|
|
* we can do some work if the lock was obtained.
|
|
|
|
*/
|
2014-08-06 17:04:11 -06:00
|
|
|
n = get_node(searchp, node);
|
2006-03-22 01:09:05 -07:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
reap_alien(searchp, n);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
drain_array(searchp, n, cpu_cache_get(searchp), 0, node);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-03-22 01:09:05 -07:00
|
|
|
/*
|
|
|
|
* These are racy checks but it does not matter
|
|
|
|
* if we skip one check or scan twice.
|
|
|
|
*/
|
2013-01-10 12:14:19 -07:00
|
|
|
if (time_after(n->next_reap, jiffies))
|
2006-03-22 01:09:05 -07:00
|
|
|
goto next;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2014-03-30 03:02:20 -06:00
|
|
|
n->next_reap = jiffies + REAPTIMEOUT_NODE;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
drain_array(searchp, n, n->shared, 0, node);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
if (n->free_touched)
|
|
|
|
n->free_touched = 0;
|
2006-06-30 02:55:45 -06:00
|
|
|
else {
|
|
|
|
int freed;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
freed = drain_freelist(searchp, n, (n->free_limit +
|
2006-06-30 02:55:45 -06:00
|
|
|
5 * searchp->num - 1) / (5 * searchp->num));
|
|
|
|
STATS_ADD_REAPED(searchp, freed);
|
|
|
|
}
|
2006-03-22 01:09:05 -07:00
|
|
|
next:
|
2005-04-16 16:20:36 -06:00
|
|
|
cond_resched();
|
|
|
|
}
|
|
|
|
check_irq_on();
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_unlock(&slab_mutex);
|
2006-03-09 18:33:54 -07:00
|
|
|
next_reap_node();
|
2007-02-10 02:42:55 -07:00
|
|
|
out:
|
2006-03-22 01:08:11 -07:00
|
|
|
/* Set up the next iteration */
|
2014-03-30 03:02:20 -06:00
|
|
|
schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_AC));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-01-02 14:04:48 -07:00
|
|
|
#ifdef CONFIG_SLABINFO
|
2012-10-19 08:20:27 -06:00
|
|
|
void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page;
|
2006-01-08 02:00:37 -07:00
|
|
|
unsigned long active_objs;
|
|
|
|
unsigned long num_objs;
|
|
|
|
unsigned long active_slabs = 0;
|
|
|
|
unsigned long num_slabs, free_objects = 0, shared_avail = 0;
|
2005-09-09 14:03:32 -06:00
|
|
|
const char *name;
|
2005-04-16 16:20:36 -06:00
|
|
|
char *error = NULL;
|
2005-09-09 14:03:32 -06:00
|
|
|
int node;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
active_objs = 0;
|
|
|
|
num_slabs = 0;
|
2014-08-06 17:04:11 -06:00
|
|
|
for_each_kmem_cache_node(cachep, node, n) {
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2006-02-05 00:27:58 -07:00
|
|
|
check_irq_on();
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock_irq(&n->list_lock);
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
list_for_each_entry(page, &n->slabs_full, lru) {
|
|
|
|
if (page->active != cachep->num && !error)
|
2005-09-09 14:03:32 -06:00
|
|
|
error = "slabs_full accounting error";
|
|
|
|
active_objs += cachep->num;
|
|
|
|
active_slabs++;
|
|
|
|
}
|
2013-10-23 19:07:49 -06:00
|
|
|
list_for_each_entry(page, &n->slabs_partial, lru) {
|
|
|
|
if (page->active == cachep->num && !error)
|
2013-10-23 19:07:48 -06:00
|
|
|
error = "slabs_partial accounting error";
|
2013-10-23 19:07:49 -06:00
|
|
|
if (!page->active && !error)
|
2013-10-23 19:07:48 -06:00
|
|
|
error = "slabs_partial accounting error";
|
2013-10-23 19:07:49 -06:00
|
|
|
active_objs += page->active;
|
2005-09-09 14:03:32 -06:00
|
|
|
active_slabs++;
|
|
|
|
}
|
2013-10-23 19:07:49 -06:00
|
|
|
list_for_each_entry(page, &n->slabs_free, lru) {
|
|
|
|
if (page->active && !error)
|
2013-10-23 19:07:48 -06:00
|
|
|
error = "slabs_free accounting error";
|
2005-09-09 14:03:32 -06:00
|
|
|
num_slabs++;
|
|
|
|
}
|
2013-01-10 12:14:19 -07:00
|
|
|
free_objects += n->free_objects;
|
|
|
|
if (n->shared)
|
|
|
|
shared_avail += n->shared->avail;
|
2005-09-09 14:03:32 -06:00
|
|
|
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2006-01-08 02:00:37 -07:00
|
|
|
num_slabs += active_slabs;
|
|
|
|
num_objs = num_slabs * cachep->num;
|
2005-09-09 14:03:32 -06:00
|
|
|
if (num_objs - active_objs != free_objects && !error)
|
2005-04-16 16:20:36 -06:00
|
|
|
error = "free_objects accounting error";
|
|
|
|
|
2006-01-08 02:00:37 -07:00
|
|
|
name = cachep->name;
|
2005-04-16 16:20:36 -06:00
|
|
|
if (error)
|
|
|
|
printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
|
|
|
|
|
2012-10-19 08:20:27 -06:00
|
|
|
sinfo->active_objs = active_objs;
|
|
|
|
sinfo->num_objs = num_objs;
|
|
|
|
sinfo->active_slabs = active_slabs;
|
|
|
|
sinfo->num_slabs = num_slabs;
|
|
|
|
sinfo->shared_avail = shared_avail;
|
|
|
|
sinfo->limit = cachep->limit;
|
|
|
|
sinfo->batchcount = cachep->batchcount;
|
|
|
|
sinfo->shared = cachep->shared;
|
|
|
|
sinfo->objects_per_slab = cachep->num;
|
|
|
|
sinfo->cache_order = cachep->gfporder;
|
|
|
|
}
|
|
|
|
|
|
|
|
void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
|
|
|
|
{
|
2005-04-16 16:20:36 -06:00
|
|
|
#if STATS
|
2013-01-10 12:14:19 -07:00
|
|
|
{ /* node stats */
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned long high = cachep->high_mark;
|
|
|
|
unsigned long allocs = cachep->num_allocations;
|
|
|
|
unsigned long grown = cachep->grown;
|
|
|
|
unsigned long reaped = cachep->reaped;
|
|
|
|
unsigned long errors = cachep->errors;
|
|
|
|
unsigned long max_freeable = cachep->max_freeable;
|
|
|
|
unsigned long node_allocs = cachep->node_allocs;
|
2005-09-09 14:03:32 -06:00
|
|
|
unsigned long node_frees = cachep->node_frees;
|
2006-04-10 23:52:54 -06:00
|
|
|
unsigned long overflows = cachep->node_overflow;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-03-26 20:27:58 -06:00
|
|
|
seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
|
|
|
|
"%4lu %4lu %4lu %4lu %4lu",
|
|
|
|
allocs, high, grown,
|
|
|
|
reaped, errors, max_freeable, node_allocs,
|
|
|
|
node_frees, overflows);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
/* cpu stats */
|
|
|
|
{
|
|
|
|
unsigned long allochit = atomic_read(&cachep->allochit);
|
|
|
|
unsigned long allocmiss = atomic_read(&cachep->allocmiss);
|
|
|
|
unsigned long freehit = atomic_read(&cachep->freehit);
|
|
|
|
unsigned long freemiss = atomic_read(&cachep->freemiss);
|
|
|
|
|
|
|
|
seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
|
2006-01-08 02:00:37 -07:00
|
|
|
allochit, allocmiss, freehit, freemiss);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
#define MAX_SLABINFO_WRITE 128
|
|
|
|
/**
|
|
|
|
* slabinfo_write - Tuning for the slab allocator
|
|
|
|
* @file: unused
|
|
|
|
* @buffer: user buffer
|
|
|
|
* @count: data length
|
|
|
|
* @ppos: unused
|
|
|
|
*/
|
2012-10-19 08:20:25 -06:00
|
|
|
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
|
2006-01-08 02:00:37 -07:00
|
|
|
size_t count, loff_t *ppos)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-01-08 02:00:37 -07:00
|
|
|
char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
|
2005-04-16 16:20:36 -06:00
|
|
|
int limit, batchcount, shared, res;
|
2006-06-23 03:03:17 -06:00
|
|
|
struct kmem_cache *cachep;
|
2006-01-08 02:00:37 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (count > MAX_SLABINFO_WRITE)
|
|
|
|
return -EINVAL;
|
|
|
|
if (copy_from_user(&kbuf, buffer, count))
|
|
|
|
return -EFAULT;
|
2006-01-08 02:00:37 -07:00
|
|
|
kbuf[MAX_SLABINFO_WRITE] = '\0';
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
tmp = strchr(kbuf, ' ');
|
|
|
|
if (!tmp)
|
|
|
|
return -EINVAL;
|
|
|
|
*tmp = '\0';
|
|
|
|
tmp++;
|
|
|
|
if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* Find the cache in the chain of caches. */
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_lock(&slab_mutex);
|
2005-04-16 16:20:36 -06:00
|
|
|
res = -EINVAL;
|
2012-07-06 14:25:12 -06:00
|
|
|
list_for_each_entry(cachep, &slab_caches, list) {
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!strcmp(cachep->name, kbuf)) {
|
2006-03-22 01:08:11 -07:00
|
|
|
if (limit < 1 || batchcount < 1 ||
|
|
|
|
batchcount > limit || shared < 0) {
|
2005-09-09 14:03:32 -06:00
|
|
|
res = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
} else {
|
2005-09-09 14:03:32 -06:00
|
|
|
res = do_tune_cpucache(cachep, limit,
|
2009-06-10 10:40:04 -06:00
|
|
|
batchcount, shared,
|
|
|
|
GFP_KERNEL);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_unlock(&slab_mutex);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (res >= 0)
|
|
|
|
res = count;
|
|
|
|
return res;
|
|
|
|
}
|
2006-03-25 04:06:39 -07:00
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_SLAB_LEAK
|
|
|
|
|
|
|
|
static void *leaks_start(struct seq_file *m, loff_t *pos)
|
|
|
|
{
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_lock(&slab_mutex);
|
|
|
|
return seq_list_start(&slab_caches, *pos);
|
2006-03-25 04:06:39 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline int add_caller(unsigned long *n, unsigned long v)
|
|
|
|
{
|
|
|
|
unsigned long *p;
|
|
|
|
int l;
|
|
|
|
if (!v)
|
|
|
|
return 1;
|
|
|
|
l = n[1];
|
|
|
|
p = n + 2;
|
|
|
|
while (l) {
|
|
|
|
int i = l/2;
|
|
|
|
unsigned long *q = p + 2 * i;
|
|
|
|
if (*q == v) {
|
|
|
|
q[1]++;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
if (*q > v) {
|
|
|
|
l = i;
|
|
|
|
} else {
|
|
|
|
p = q + 2;
|
|
|
|
l -= i + 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (++n[1] == n[0])
|
|
|
|
return 0;
|
|
|
|
memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
|
|
|
|
p[0] = v;
|
|
|
|
p[1] = 1;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
static void handle_slab(unsigned long *n, struct kmem_cache *c,
|
|
|
|
struct page *page)
|
2006-03-25 04:06:39 -07:00
|
|
|
{
|
|
|
|
void *p;
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
int i;
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-23 19:07:45 -06:00
|
|
|
|
2006-03-25 04:06:39 -07:00
|
|
|
if (n[0] == n[1])
|
|
|
|
return;
|
2013-10-23 19:07:49 -06:00
|
|
|
for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {
|
slab: fix oops when reading /proc/slab_allocators
Commit b1cb0982bdd6 ("change the management method of free objects of
the slab") introduced a bug on slab leak detector
('/proc/slab_allocators'). This detector works like as following
decription.
1. traverse all objects on all the slabs.
2. determine whether it is active or not.
3. if active, print who allocate this object.
but that commit changed the way how to manage free objects, so the logic
determining whether it is active or not is also changed. In before, we
regard object in cpu caches as inactive one, but, with this commit, we
mistakenly regard object in cpu caches as active one.
This intoduces kernel oops if DEBUG_PAGEALLOC is enabled. If
DEBUG_PAGEALLOC is enabled, kernel_map_pages() is used to detect who
corrupt free memory in the slab. It unmaps page table mapping if object
is free and map it if object is active. When slab leak detector check
object in cpu caches, it mistakenly think this object active so try to
access object memory to retrieve caller of allocation. At this point,
page table mapping to this object doesn't exist, so oops occurs.
Following is oops message reported from Dave.
It blew up when something tried to read /proc/slab_allocators
(Just cat it, and you should see the oops below)
Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
Modules linked in:
[snip...]
CPU: 1 PID: 9386 Comm: trinity-c33 Not tainted 3.14.0-rc5+ #131
task: ffff8801aa46e890 ti: ffff880076924000 task.ti: ffff880076924000
RIP: 0010:[<ffffffffaa1a8f4a>] [<ffffffffaa1a8f4a>] handle_slab+0x8a/0x180
RSP: 0018:ffff880076925de0 EFLAGS: 00010002
RAX: 0000000000001000 RBX: 0000000000000000 RCX: 000000005ce85ce7
RDX: ffffea00079be100 RSI: 0000000000001000 RDI: ffff880107458000
RBP: ffff880076925e18 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000f R12: ffff8801e6f84000
R13: ffffea00079be100 R14: ffff880107458000 R15: ffff88022bb8d2c0
FS: 00007fb769e45740(0000) GS:ffff88024d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff8801e6f84ff8 CR3: 00000000a22db000 CR4: 00000000001407e0
DR0: 0000000002695000 DR1: 0000000002695000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000070602
Call Trace:
leaks_show+0xce/0x240
seq_read+0x28e/0x490
proc_reg_read+0x3d/0x80
vfs_read+0x9b/0x160
SyS_read+0x58/0xb0
tracesys+0xd4/0xd9
Code: f5 00 00 00 0f 1f 44 00 00 48 63 c8 44 3b 0c 8a 0f 84 e3 00 00 00 83 c0 01 44 39 c0 72 eb 41 f6 47 1a 01 0f 84 e9 00 00 00 89 f0 <4d> 8b 4c 04 f8 4d 85 c9 0f 84 88 00 00 00 49 8b 7e 08 4d 8d 46
RIP handle_slab+0x8a/0x180
To fix the problem, I introduce an object status buffer on each slab.
With this, we can track object status precisely, so slab leak detector
would not access active object and no kernel oops would occur. Memory
overhead caused by this fix is only imposed to CONFIG_DEBUG_SLAB_LEAK
which is mainly used for debugging, so memory overhead isn't big
problem.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Reported-by: Dave Jones <davej@redhat.com>
Reported-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:06 -06:00
|
|
|
if (get_obj_status(page, i) != OBJECT_ACTIVE)
|
2006-03-25 04:06:39 -07:00
|
|
|
continue;
|
slab: change the management method of free objects of the slab
Current free objects management method of the slab is weird, because
it touch random position of the array of kmem_bufctl_t when we try to
get free object. See following example.
struct slab's free = 6
kmem_bufctl_t array: 1 END 5 7 0 4 3 2
To get free objects, we access this array with following pattern.
6 -> 3 -> 7 -> 2 -> 5 -> 4 -> 0 -> 1 -> END
If we have many objects, this array would be larger and be not in the same
cache line. It is not good for performance.
We can do same thing through more easy way, like as the stack.
Only thing we have to do is to maintain stack top to free object. I use
free field of struct slab for this purpose. After that, if we need to get
an object, we can get it at stack top and manipulate top pointer.
That's all. This method already used in array_cache management.
Following is an access pattern when we use this method.
struct slab's free = 0
kmem_bufctl_t array: 6 3 7 2 5 4 0 1
To get free objects, we access this array with following pattern.
0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7
This may help cache line footprint if slab has many objects, and,
in addition, this makes code much much simpler.
Acked-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Pekka Enberg <penberg@iki.fi>
2013-10-23 19:07:45 -06:00
|
|
|
|
2006-03-25 04:06:39 -07:00
|
|
|
if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void show_symbol(struct seq_file *m, unsigned long address)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_KALLSYMS
|
|
|
|
unsigned long offset, size;
|
2007-07-17 05:03:51 -06:00
|
|
|
char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
|
2006-03-25 04:06:39 -07:00
|
|
|
|
2007-05-08 01:28:47 -06:00
|
|
|
if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
|
2006-03-25 04:06:39 -07:00
|
|
|
seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
|
2007-05-08 01:28:47 -06:00
|
|
|
if (modname[0])
|
2006-03-25 04:06:39 -07:00
|
|
|
seq_printf(m, " [%s]", modname);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
seq_printf(m, "%p", (void *)address);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int leaks_show(struct seq_file *m, void *p)
|
|
|
|
{
|
2012-06-22 11:42:49 -06:00
|
|
|
struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
|
2013-10-23 19:07:49 -06:00
|
|
|
struct page *page;
|
2013-01-10 12:14:19 -07:00
|
|
|
struct kmem_cache_node *n;
|
2006-03-25 04:06:39 -07:00
|
|
|
const char *name;
|
2013-02-05 11:45:23 -07:00
|
|
|
unsigned long *x = m->private;
|
2006-03-25 04:06:39 -07:00
|
|
|
int node;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!(cachep->flags & SLAB_STORE_USER))
|
|
|
|
return 0;
|
|
|
|
if (!(cachep->flags & SLAB_RED_ZONE))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* OK, we can do it */
|
|
|
|
|
2013-02-05 11:45:23 -07:00
|
|
|
x[1] = 0;
|
2006-03-25 04:06:39 -07:00
|
|
|
|
2014-08-06 17:04:11 -06:00
|
|
|
for_each_kmem_cache_node(cachep, node, n) {
|
2006-03-25 04:06:39 -07:00
|
|
|
|
|
|
|
check_irq_on();
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_lock_irq(&n->list_lock);
|
2006-03-25 04:06:39 -07:00
|
|
|
|
2013-10-23 19:07:49 -06:00
|
|
|
list_for_each_entry(page, &n->slabs_full, lru)
|
|
|
|
handle_slab(x, cachep, page);
|
|
|
|
list_for_each_entry(page, &n->slabs_partial, lru)
|
|
|
|
handle_slab(x, cachep, page);
|
2013-01-10 12:14:19 -07:00
|
|
|
spin_unlock_irq(&n->list_lock);
|
2006-03-25 04:06:39 -07:00
|
|
|
}
|
|
|
|
name = cachep->name;
|
2013-02-05 11:45:23 -07:00
|
|
|
if (x[0] == x[1]) {
|
2006-03-25 04:06:39 -07:00
|
|
|
/* Increase the buffer size */
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_unlock(&slab_mutex);
|
2013-02-05 11:45:23 -07:00
|
|
|
m->private = kzalloc(x[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
|
2006-03-25 04:06:39 -07:00
|
|
|
if (!m->private) {
|
|
|
|
/* Too bad, we are really out */
|
2013-02-05 11:45:23 -07:00
|
|
|
m->private = x;
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_lock(&slab_mutex);
|
2006-03-25 04:06:39 -07:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
2013-02-05 11:45:23 -07:00
|
|
|
*(unsigned long *)m->private = x[0] * 2;
|
|
|
|
kfree(x);
|
2012-07-06 14:25:12 -06:00
|
|
|
mutex_lock(&slab_mutex);
|
2006-03-25 04:06:39 -07:00
|
|
|
/* Now make sure this entry will be retried */
|
|
|
|
m->count = m->size;
|
|
|
|
return 0;
|
|
|
|
}
|
2013-02-05 11:45:23 -07:00
|
|
|
for (i = 0; i < x[1]; i++) {
|
|
|
|
seq_printf(m, "%s: %lu ", name, x[2*i+3]);
|
|
|
|
show_symbol(m, x[2*i+2]);
|
2006-03-25 04:06:39 -07:00
|
|
|
seq_putc(m, '\n');
|
|
|
|
}
|
2006-09-26 00:31:47 -06:00
|
|
|
|
2006-03-25 04:06:39 -07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-10-05 14:59:10 -06:00
|
|
|
static const struct seq_operations slabstats_op = {
|
2006-03-25 04:06:39 -07:00
|
|
|
.start = leaks_start,
|
2013-07-07 18:08:28 -06:00
|
|
|
.next = slab_next,
|
|
|
|
.stop = slab_stop,
|
2006-03-25 04:06:39 -07:00
|
|
|
.show = leaks_show,
|
|
|
|
};
|
2008-10-05 14:59:10 -06:00
|
|
|
|
|
|
|
static int slabstats_open(struct inode *inode, struct file *file)
|
|
|
|
{
|
2014-10-09 16:28:03 -06:00
|
|
|
unsigned long *n;
|
|
|
|
|
|
|
|
n = __seq_open_private(file, &slabstats_op, PAGE_SIZE);
|
|
|
|
if (!n)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
*n = PAGE_SIZE / (2 * sizeof(unsigned long));
|
|
|
|
|
|
|
|
return 0;
|
2008-10-05 14:59:10 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static const struct file_operations proc_slabstats_operations = {
|
|
|
|
.open = slabstats_open,
|
|
|
|
.read = seq_read,
|
|
|
|
.llseek = seq_lseek,
|
|
|
|
.release = seq_release_private,
|
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static int __init slab_proc_init(void)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_DEBUG_SLAB_LEAK
|
|
|
|
proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
|
2006-03-25 04:06:39 -07:00
|
|
|
#endif
|
2008-10-05 14:59:10 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
module_init(slab_proc_init);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
|
2005-09-03 16:55:07 -06:00
|
|
|
/**
|
|
|
|
* ksize - get the actual amount of memory allocated for a given object
|
|
|
|
* @objp: Pointer to the object
|
|
|
|
*
|
|
|
|
* kmalloc may internally round up allocations and return more memory
|
|
|
|
* than requested. ksize() can be used to determine the actual amount of
|
|
|
|
* memory allocated. The caller may use this additional memory, even though
|
|
|
|
* a smaller amount of memory was initially specified with the kmalloc call.
|
|
|
|
* The caller must guarantee that objp points to a valid object previously
|
|
|
|
* allocated with either kmalloc() or kmem_cache_alloc(). The object
|
|
|
|
* must not be freed during the duration of the call.
|
|
|
|
*/
|
2007-05-06 15:48:40 -06:00
|
|
|
size_t ksize(const void *objp)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-10-16 02:24:46 -06:00
|
|
|
BUG_ON(!objp);
|
|
|
|
if (unlikely(objp == ZERO_SIZE_PTR))
|
2005-09-03 16:55:07 -06:00
|
|
|
return 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-06-13 09:24:58 -06:00
|
|
|
return virt_to_cache(objp)->object_size;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-02-10 06:21:44 -07:00
|
|
|
EXPORT_SYMBOL(ksize);
|