From 0d24db337e6d81c0c620ab65cc6947bd6553f742 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 21 Oct 2010 13:01:56 -0500 Subject: [PATCH 1/6] slub: move slabinfo.c to tools/slub/slabinfo.c We now have a tools directory for these things. Reviewed-by: KOSAKI Motohiro Acked-by: David Rientjes Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- {Documentation/vm => tools/slub}/slabinfo.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename {Documentation/vm => tools/slub}/slabinfo.c (99%) diff --git a/Documentation/vm/slabinfo.c b/tools/slub/slabinfo.c similarity index 99% rename from Documentation/vm/slabinfo.c rename to tools/slub/slabinfo.c index 92e729f4b676..516551c9f172 100644 --- a/Documentation/vm/slabinfo.c +++ b/tools/slub/slabinfo.c @@ -607,7 +607,7 @@ static int debug_opt_scan(char *opt) } for ( ; *opt; opt++) - switch (*opt) { + switch (*opt) { case 'F' : case 'f': if (sanity) return 0; @@ -1127,7 +1127,7 @@ static void read_slab_dir(void) continue; switch (de->d_type) { case DT_LNK: - alias->name = strdup(de->d_name); + alias->name = strdup(de->d_name); count = readlink(de->d_name, buffer, sizeof(buffer)); if (count < 0) @@ -1143,7 +1143,7 @@ static void read_slab_dir(void) case DT_DIR: if (chdir(de->d_name)) fatal("Unable to access slab %s\n", slab->name); - slab->name = strdup(de->d_name); + slab->name = strdup(de->d_name); slab->alias = 0; slab->refs = 0; slab->aliases = get_obj("aliases"); From 4a92379bdfb48680a5e6775dd53a586df7b6b0b1 Mon Sep 17 00:00:00 2001 From: Richard Kennedy Date: Thu, 21 Oct 2010 10:29:19 +0100 Subject: [PATCH 2/6] slub tracing: move trace calls out of always inlined functions to reduce kernel code size Having the trace calls defined in the always inlined kmalloc functions in include/linux/slub_def.h causes a lot of code duplication as the trace functions get instantiated for each kamalloc call site. This can simply be removed by pushing the trace calls down into the functions in slub.c. On my x86_64 built this patch shrinks the code size of the kernel by approx 36K and also shrinks the code size of many modules -- too many to list here ;) size vmlinux (2.6.36) reports text data bss dec hex filename 5410611 743172 828928 6982711 6a8c37 vmlinux 5373738 744244 828928 6946910 6a005e vmlinux + patch The resulting kernel has had some testing & kmalloc trace still seems to work. This patch - moves trace_kmalloc out of the inlined kmalloc() and pushes it down into kmem_cache_alloc_trace() so this it only get instantiated once. - rename kmem_cache_alloc_notrace() to kmem_cache_alloc_trace() to indicate that now is does have tracing. (maybe this would better being called something like kmalloc_kmem_cache ?) - adds a new function kmalloc_order() to handle allocation and tracing of large allocations of page order. - removes tracing from the inlined kmalloc_large() replacing them with a call to kmalloc_order(); - move tracing out of inlined kmalloc_node() and pushing it down into kmem_cache_alloc_node_trace - rename kmem_cache_alloc_node_notrace() to kmem_cache_alloc_node_trace() - removes the include of trace/events/kmem.h from slub_def.h. v2 - keep kmalloc_order_trace inline when !CONFIG_TRACE Signed-off-by: Richard Kennedy Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 55 +++++++++++++++++++--------------------- mm/slub.c | 30 +++++++++++++++++----- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index e4f5ed180b9b..8b6e8ae5d5ca 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -10,9 +10,8 @@ #include #include #include -#include -#include +#include enum stat_item { ALLOC_FASTPATH, /* Allocation from cpu slab */ @@ -216,31 +215,40 @@ static __always_inline struct kmem_cache *kmalloc_slab(size_t size) void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); +static __always_inline void * +kmalloc_order(size_t size, gfp_t flags, unsigned int order) +{ + void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order); + kmemleak_alloc(ret, size, 1, flags); + return ret; +} + #ifdef CONFIG_TRACING -extern void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags); +extern void * +kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size); +extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order); #else static __always_inline void * -kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags) +kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) { return kmem_cache_alloc(s, gfpflags); } + +static __always_inline void * +kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) +{ + return kmalloc_order(size, flags, order); +} #endif static __always_inline void *kmalloc_large(size_t size, gfp_t flags) { unsigned int order = get_order(size); - void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order); - - kmemleak_alloc(ret, size, 1, flags); - trace_kmalloc(_THIS_IP_, ret, size, PAGE_SIZE << order, flags); - - return ret; + return kmalloc_order_trace(size, flags, order); } static __always_inline void *kmalloc(size_t size, gfp_t flags) { - void *ret; - if (__builtin_constant_p(size)) { if (size > SLUB_MAX_SIZE) return kmalloc_large(size, flags); @@ -251,11 +259,7 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags) if (!s) return ZERO_SIZE_PTR; - ret = kmem_cache_alloc_notrace(s, flags); - - trace_kmalloc(_THIS_IP_, ret, size, s->size, flags); - - return ret; + return kmem_cache_alloc_trace(s, flags, size); } } return __kmalloc(size, flags); @@ -266,14 +270,14 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node); void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); #ifdef CONFIG_TRACING -extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *s, +extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, - int node); + int node, size_t size); #else static __always_inline void * -kmem_cache_alloc_node_notrace(struct kmem_cache *s, +kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, - int node) + int node, size_t size) { return kmem_cache_alloc_node(s, gfpflags, node); } @@ -281,8 +285,6 @@ kmem_cache_alloc_node_notrace(struct kmem_cache *s, static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) { - void *ret; - if (__builtin_constant_p(size) && size <= SLUB_MAX_SIZE && !(flags & SLUB_DMA)) { struct kmem_cache *s = kmalloc_slab(size); @@ -290,12 +292,7 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) if (!s) return ZERO_SIZE_PTR; - ret = kmem_cache_alloc_node_notrace(s, flags, node); - - trace_kmalloc_node(_THIS_IP_, ret, - size, s->size, flags, node); - - return ret; + return kmem_cache_alloc_node_trace(s, flags, node, size); } return __kmalloc_node(size, flags, node); } diff --git a/mm/slub.c b/mm/slub.c index 8fd5401bb071..7e657aa19475 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -28,6 +28,8 @@ #include #include +#include + /* * Lock order: * 1. slab_lock(page) @@ -1774,11 +1776,21 @@ void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) EXPORT_SYMBOL(kmem_cache_alloc); #ifdef CONFIG_TRACING -void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags) +void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) { - return slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_); + void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_); + trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); + return ret; } -EXPORT_SYMBOL(kmem_cache_alloc_notrace); +EXPORT_SYMBOL(kmem_cache_alloc_trace); + +void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) +{ + void *ret = kmalloc_order(size, flags, order); + trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags); + return ret; +} +EXPORT_SYMBOL(kmalloc_order_trace); #endif #ifdef CONFIG_NUMA @@ -1794,13 +1806,17 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) EXPORT_SYMBOL(kmem_cache_alloc_node); #ifdef CONFIG_TRACING -void *kmem_cache_alloc_node_notrace(struct kmem_cache *s, +void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, - int node) + int node, size_t size) { - return slab_alloc(s, gfpflags, node, _RET_IP_); + void *ret = slab_alloc(s, gfpflags, node, _RET_IP_); + + trace_kmalloc_node(_RET_IP_, ret, + size, s->size, gfpflags, node); + return ret; } -EXPORT_SYMBOL(kmem_cache_alloc_node_notrace); +EXPORT_SYMBOL(kmem_cache_alloc_node_trace); #endif #endif From 716ce5d4a6f81653507c21b152eb7629d150ce7a Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Wed, 27 Oct 2010 18:17:38 +0300 Subject: [PATCH 3/6] slub: Fix build breakage in Documentation/vm This patch fixes a build breakage introduced by commit f5ac4916e9840292edd33c7a52b10364526547f3 ("slub: move slabinfo.c to tools/slub/slabinfo.c") that was repoted by Stephen: After merging the slab tree, today's linux-next build (x86_64 allmodconfig) failed like this: gcc: /scratch/sfr/next/Documentation/vm/slabinfo.c: No such file or directory gcc: no input files Caused by commit f5ac4916e9840292edd33c7a52b10364526547f3 ("slub: move slabinfo.c to tools/slub/slabinfo.c"). Missing update to Documentation/vm/Makefile? Reported-by: Stephen Rothwell Signed-off-by: Pekka Enberg --- Documentation/vm/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/vm/Makefile b/Documentation/vm/Makefile index 9dcff328b964..3fa4d0668864 100644 --- a/Documentation/vm/Makefile +++ b/Documentation/vm/Makefile @@ -2,7 +2,7 @@ obj- := dummy.o # List of programs to build -hostprogs-y := slabinfo page-types hugepage-mmap hugepage-shm map_hugetlb +hostprogs-y := page-types hugepage-mmap hugepage-shm map_hugetlb # Tell kbuild to always build the programs always := $(hostprogs-y) From 98072e4d977aabe6a39abb95951cd8bf2c2202d5 Mon Sep 17 00:00:00 2001 From: Pavel Emelyanov Date: Thu, 28 Oct 2010 13:50:37 +0400 Subject: [PATCH 4/6] slub: Fix slub_lock down/up imbalance There are two places, that do not release the slub_lock. Respective bugs were introduced by sysfs changes ab4d5ed5 (slub: Enable sysfs support for !CONFIG_SLUB_DEBUG) and 2bce6485 ( slub: Allow removal of slab caches during boot). Acked-by: Christoph Lameter Signed-off-by: Pavel Emelyanov Signed-off-by: Pekka Enberg --- mm/slub.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index 7e657aa19475..7796a0446b3f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3289,9 +3289,9 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, kfree(n); kfree(s); } +err: up_write(&slub_lock); -err: if (flags & SLAB_PANIC) panic("Cannot create slabcache %s\n", name); else @@ -3878,6 +3878,7 @@ static ssize_t show_slab_objects(struct kmem_cache *s, x += sprintf(buf + x, " N%d=%lu", node, nodes[node]); #endif + up_read(&slub_lock); kfree(nodes); return x + sprintf(buf + x, "\n"); } From 85beb5869a4f6abb52a7cf8e01de6fa57e9ee47d Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Wed, 24 Nov 2010 16:23:34 -0500 Subject: [PATCH 5/6] tracing/slab: Move kmalloc tracepoint out of inline code The tracepoint for kmalloc is in the slab inlined code which causes every instance of kmalloc to have the tracepoint. This patch moves the tracepoint out of the inline code to the slab C file, which removes a large number of inlined trace points. objdump -dr vmlinux.slab| grep 'jmpq.* Signed-off-by: Pekka Enberg --- include/linux/slab_def.h | 33 +++++++++++++-------------------- mm/slab.c | 38 +++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 791a502f6906..83203ae9390b 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -138,11 +138,12 @@ void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); #ifdef CONFIG_TRACING -extern void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags); +extern void *kmem_cache_alloc_trace(size_t size, + struct kmem_cache *cachep, gfp_t flags); extern size_t slab_buffer_size(struct kmem_cache *cachep); #else static __always_inline void * -kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags) +kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags) { return kmem_cache_alloc(cachep, flags); } @@ -179,10 +180,7 @@ found: #endif cachep = malloc_sizes[i].cs_cachep; - ret = kmem_cache_alloc_notrace(cachep, flags); - - trace_kmalloc(_THIS_IP_, ret, - size, slab_buffer_size(cachep), flags); + ret = kmem_cache_alloc_trace(size, cachep, flags); return ret; } @@ -194,14 +192,16 @@ extern void *__kmalloc_node(size_t size, gfp_t flags, int node); extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); #ifdef CONFIG_TRACING -extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep, - gfp_t flags, - int nodeid); +extern void *kmem_cache_alloc_node_trace(size_t size, + struct kmem_cache *cachep, + gfp_t flags, + int nodeid); #else static __always_inline void * -kmem_cache_alloc_node_notrace(struct kmem_cache *cachep, - gfp_t flags, - int nodeid) +kmem_cache_alloc_node_trace(size_t size, + struct kmem_cache *cachep, + gfp_t flags, + int nodeid) { return kmem_cache_alloc_node(cachep, flags, nodeid); } @@ -210,7 +210,6 @@ kmem_cache_alloc_node_notrace(struct kmem_cache *cachep, static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) { struct kmem_cache *cachep; - void *ret; if (__builtin_constant_p(size)) { int i = 0; @@ -234,13 +233,7 @@ found: #endif cachep = malloc_sizes[i].cs_cachep; - ret = kmem_cache_alloc_node_notrace(cachep, flags, node); - - trace_kmalloc_node(_THIS_IP_, ret, - size, slab_buffer_size(cachep), - flags, node); - - return ret; + return kmem_cache_alloc_node_trace(size, cachep, flags, node); } return __kmalloc_node(size, flags, node); } diff --git a/mm/slab.c b/mm/slab.c index b1e40dafbab3..dfcc8885d7d5 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3653,11 +3653,18 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) EXPORT_SYMBOL(kmem_cache_alloc); #ifdef CONFIG_TRACING -void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags) +void * +kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags) { - return __cache_alloc(cachep, flags, __builtin_return_address(0)); + void *ret; + + ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); + + trace_kmalloc(_RET_IP_, ret, + size, slab_buffer_size(cachep), flags); + return ret; } -EXPORT_SYMBOL(kmem_cache_alloc_notrace); +EXPORT_SYMBOL(kmem_cache_alloc_trace); #endif /** @@ -3705,31 +3712,32 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) EXPORT_SYMBOL(kmem_cache_alloc_node); #ifdef CONFIG_TRACING -void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep, - gfp_t flags, - int nodeid) +void *kmem_cache_alloc_node_trace(size_t size, + struct kmem_cache *cachep, + gfp_t flags, + int nodeid) { - return __cache_alloc_node(cachep, flags, nodeid, + void *ret; + + ret = __cache_alloc_node(cachep, flags, nodeid, __builtin_return_address(0)); + trace_kmalloc_node(_RET_IP_, ret, + size, slab_buffer_size(cachep), + flags, nodeid); + return ret; } -EXPORT_SYMBOL(kmem_cache_alloc_node_notrace); +EXPORT_SYMBOL(kmem_cache_alloc_node_trace); #endif static __always_inline void * __do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller) { struct kmem_cache *cachep; - void *ret; cachep = kmem_find_general_cachep(size, flags); if (unlikely(ZERO_OR_NULL_PTR(cachep))) return cachep; - ret = kmem_cache_alloc_node_notrace(cachep, flags, node); - - trace_kmalloc_node((unsigned long) caller, ret, - size, cachep->buffer_size, flags, node); - - return ret; + return kmem_cache_alloc_node_trace(size, cachep, flags, node); } #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING) From 8165984acf825917437debae519209073c32a5a7 Mon Sep 17 00:00:00 2001 From: Tero Roponen Date: Wed, 1 Dec 2010 20:04:20 +0200 Subject: [PATCH 6/6] slub: Fix a crash during slabinfo -v Commit f7cb1933621bce66a77f690776a16fe3ebbc4d58 ("SLUB: Pass active and inactive redzone flags instead of boolean to debug functions") missed two instances of check_object(). This caused a lot of warnings during 'slabinfo -v' finally leading to a crash: BUG ext4_xattr: Freepointer corrupt ... BUG buffer_head: Freepointer corrupt ... BUG ext4_alloc_context: Freepointer corrupt ... ... BUG: unable to handle kernel NULL pointer dereference at 0000000000000008 IP: [] file_sb_list_del+0x1c/0x35 PGD 79d78067 PUD 79e67067 PMD 0 Oops: 0002 [#1] SMP last sysfs file: /sys/kernel/slab/:t-0000192/validate This patch fixes the problem by converting the two missed instances. Acked-by: Christoph Lameter Signed-off-by: Tero Roponen Signed-off-by: Pekka Enberg --- mm/slub.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 7796a0446b3f..48d82a55f4b6 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3417,13 +3417,13 @@ static int validate_slab(struct kmem_cache *s, struct page *page, for_each_free_object(p, s, page->freelist) { set_bit(slab_index(p, s, addr), map); - if (!check_object(s, page, p, 0)) + if (!check_object(s, page, p, SLUB_RED_INACTIVE)) return 0; } for_each_object(p, s, addr, page->objects) if (!test_bit(slab_index(p, s, addr), map)) - if (!check_object(s, page, p, 1)) + if (!check_object(s, page, p, SLUB_RED_ACTIVE)) return 0; return 1; }