7e496299d4
The current implementation of tmpfs is not scalable. We found that stat_lock is contended by multiple threads when we need to get a new page, leading to useless spinning inside this spin lock. This patch makes use of the percpu_counter library to maintain local count of used blocks to speed up getting and returning of pages. So the acquisition of stat_lock is unnecessary for getting and returning blocks, improving the performance of tmpfs on system with large number of cpus. On a 4 socket 32 core NHM-EX system, we saw improvement of 270%. The implementation below has a slight chance of race between threads causing a slight overshoot of the maximum configured blocks. However, any overshoot is small, and is bounded by the number of cpus. This happens when the number of used blocks is slightly below the maximum configured blocks when a thread checks the used block count, and another thread allocates the last block before the current thread does. This should not be a problem for tmpfs, as the overshoot is most likely to be a few blocks and bounded. If a strict limit is really desired, then configured the max blocks to be the limit less the number of cpus in system. Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com> Cc: Hugh Dickins <hughd@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
46 lines
1.5 KiB
C
46 lines
1.5 KiB
C
#ifndef __SHMEM_FS_H
|
|
#define __SHMEM_FS_H
|
|
|
|
#include <linux/swap.h>
|
|
#include <linux/mempolicy.h>
|
|
#include <linux/percpu_counter.h>
|
|
|
|
/* inode in-kernel data */
|
|
|
|
#define SHMEM_NR_DIRECT 16
|
|
|
|
struct shmem_inode_info {
|
|
spinlock_t lock;
|
|
unsigned long flags;
|
|
unsigned long alloced; /* data pages alloced to file */
|
|
unsigned long swapped; /* subtotal assigned to swap */
|
|
unsigned long next_index; /* highest alloced index + 1 */
|
|
struct shared_policy policy; /* NUMA memory alloc policy */
|
|
struct page *i_indirect; /* top indirect blocks page */
|
|
swp_entry_t i_direct[SHMEM_NR_DIRECT]; /* first blocks */
|
|
struct list_head swaplist; /* chain of maybes on swap */
|
|
struct inode vfs_inode;
|
|
};
|
|
|
|
struct shmem_sb_info {
|
|
unsigned long max_blocks; /* How many blocks are allowed */
|
|
struct percpu_counter used_blocks; /* How many are allocated */
|
|
unsigned long max_inodes; /* How many inodes are allowed */
|
|
unsigned long free_inodes; /* How many are left for allocation */
|
|
spinlock_t stat_lock; /* Serialize shmem_sb_info changes */
|
|
uid_t uid; /* Mount uid for root directory */
|
|
gid_t gid; /* Mount gid for root directory */
|
|
mode_t mode; /* Mount mode for root directory */
|
|
struct mempolicy *mpol; /* default memory policy for mappings */
|
|
};
|
|
|
|
static inline struct shmem_inode_info *SHMEM_I(struct inode *inode)
|
|
{
|
|
return container_of(inode, struct shmem_inode_info, vfs_inode);
|
|
}
|
|
|
|
extern int init_tmpfs(void);
|
|
extern int shmem_fill_super(struct super_block *sb, void *data, int silent);
|
|
|
|
#endif
|