894b8788d7
Avoid atomic overhead in slab_alloc and slab_free SLUB needs to use the slab_lock for the per cpu slabs to synchronize with potential kfree operations. This patch avoids that need by moving all free objects onto a lockless_freelist. The regular freelist continues to exist and will be used to free objects. So while we consume the lockless_freelist the regular freelist may build up objects. If we are out of objects on the lockless_freelist then we may check the regular freelist. If it has objects then we move those over to the lockless_freelist and do this again. There is a significant savings in terms of atomic operations that have to be performed. We can even free directly to the lockless_freelist if we know that we are running on the same processor. So this speeds up short lived objects. They may be allocated and freed without taking the slab_lock. This is particular good for netperf. In order to maximize the effect of the new faster hotpath we extract the hottest performance pieces into inlined functions. These are then inlined into kmem_cache_alloc and kmem_cache_free. So hotpath allocation and freeing no longer requires a subroutine call within SLUB. [I am not sure that it is worth doing this because it changes the easy to read structure of slub just to reduce atomic ops. However, there is someone out there with a benchmark on 4 way and 8 way processor systems that seems to show a 5% regression vs. Slab. Seems that the regression is due to increased atomic operations use vs. SLAB in SLUB). I wonder if this is applicable or discernable at all in a real workload?] Signed-off-by: Christoph Lameter <clameter@sgi.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
84 lines
2.5 KiB
C
84 lines
2.5 KiB
C
#ifndef _LINUX_MM_TYPES_H
|
|
#define _LINUX_MM_TYPES_H
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/threads.h>
|
|
#include <linux/list.h>
|
|
#include <linux/spinlock.h>
|
|
|
|
struct address_space;
|
|
|
|
/*
|
|
* Each physical page in the system has a struct page associated with
|
|
* it to keep track of whatever it is we are using the page for at the
|
|
* moment. Note that we have no way to track which tasks are using
|
|
* a page, though if it is a pagecache page, rmap structures can tell us
|
|
* who is mapping it.
|
|
*/
|
|
struct page {
|
|
unsigned long flags; /* Atomic flags, some possibly
|
|
* updated asynchronously */
|
|
atomic_t _count; /* Usage count, see below. */
|
|
union {
|
|
atomic_t _mapcount; /* Count of ptes mapped in mms,
|
|
* to show when page is mapped
|
|
* & limit reverse map searches.
|
|
*/
|
|
struct { /* SLUB uses */
|
|
short unsigned int inuse;
|
|
short unsigned int offset;
|
|
};
|
|
};
|
|
union {
|
|
struct {
|
|
unsigned long private; /* Mapping-private opaque data:
|
|
* usually used for buffer_heads
|
|
* if PagePrivate set; used for
|
|
* swp_entry_t if PageSwapCache;
|
|
* indicates order in the buddy
|
|
* system if PG_buddy is set.
|
|
*/
|
|
struct address_space *mapping; /* If low bit clear, points to
|
|
* inode address_space, or NULL.
|
|
* If page mapped as anonymous
|
|
* memory, low bit is set, and
|
|
* it points to anon_vma object:
|
|
* see PAGE_MAPPING_ANON below.
|
|
*/
|
|
};
|
|
#if NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS
|
|
spinlock_t ptl;
|
|
#endif
|
|
struct { /* SLUB uses */
|
|
void **lockless_freelist;
|
|
struct kmem_cache *slab; /* Pointer to slab */
|
|
};
|
|
struct {
|
|
struct page *first_page; /* Compound pages */
|
|
};
|
|
};
|
|
union {
|
|
pgoff_t index; /* Our offset within mapping. */
|
|
void *freelist; /* SLUB: freelist req. slab lock */
|
|
};
|
|
struct list_head lru; /* Pageout list, eg. active_list
|
|
* protected by zone->lru_lock !
|
|
*/
|
|
/*
|
|
* On machines where all RAM is mapped into kernel address space,
|
|
* we can simply calculate the virtual address. On machines with
|
|
* highmem some memory is mapped into kernel virtual memory
|
|
* dynamically, so we need a place to store that address.
|
|
* Note that this field could be 16 bits on x86 ... ;)
|
|
*
|
|
* Architectures with slow multiplication can define
|
|
* WANT_PAGE_VIRTUAL in asm/page.h
|
|
*/
|
|
#if defined(WANT_PAGE_VIRTUAL)
|
|
void *virtual; /* Kernel virtual address (NULL if
|
|
not kmapped, ie. highmem) */
|
|
#endif /* WANT_PAGE_VIRTUAL */
|
|
};
|
|
|
|
#endif /* _LINUX_MM_TYPES_H */
|