2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* hugetlbpage-backed filesystem. Based on ramfs.
|
|
|
|
*
|
|
|
|
* William Irwin, 2002
|
|
|
|
*
|
|
|
|
* Copyright (C) 2002 Linus Torvalds.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/thread_info.h>
|
|
|
|
#include <asm/current.h>
|
|
|
|
#include <linux/sched.h> /* remove ASAP */
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/mount.h>
|
|
|
|
#include <linux/file.h>
|
|
|
|
#include <linux/writeback.h>
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
#include <linux/highmem.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/string.h>
|
2006-01-11 13:17:46 -07:00
|
|
|
#include <linux/capability.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/backing-dev.h>
|
|
|
|
#include <linux/hugetlb.h>
|
|
|
|
#include <linux/pagevec.h>
|
|
|
|
#include <linux/quotaops.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/dnotify.h>
|
|
|
|
#include <linux/statfs.h>
|
|
|
|
#include <linux/security.h>
|
|
|
|
|
|
|
|
#include <asm/uaccess.h>
|
|
|
|
|
|
|
|
/* some random number */
|
|
|
|
#define HUGETLBFS_MAGIC 0x958458f6
|
|
|
|
|
|
|
|
static struct super_operations hugetlbfs_ops;
|
|
|
|
static struct address_space_operations hugetlbfs_aops;
|
2006-03-28 02:56:42 -07:00
|
|
|
const struct file_operations hugetlbfs_file_operations;
|
2005-04-16 16:20:36 -06:00
|
|
|
static struct inode_operations hugetlbfs_dir_inode_operations;
|
|
|
|
static struct inode_operations hugetlbfs_inode_operations;
|
|
|
|
|
|
|
|
static struct backing_dev_info hugetlbfs_backing_dev_info = {
|
|
|
|
.ra_pages = 0, /* No readahead */
|
|
|
|
.capabilities = BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK,
|
|
|
|
};
|
|
|
|
|
|
|
|
int sysctl_hugetlb_shm_group;
|
|
|
|
|
2005-10-29 19:16:47 -06:00
|
|
|
static void huge_pagevec_release(struct pagevec *pvec)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < pagevec_count(pvec); ++i)
|
|
|
|
put_page(pvec->pages[i]);
|
|
|
|
|
|
|
|
pagevec_reinit(pvec);
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
struct inode *inode = file->f_dentry->d_inode;
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
|
2005-04-16 16:20:36 -06:00
|
|
|
loff_t len, vma_len;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (vma->vm_pgoff & (HPAGE_SIZE / PAGE_SIZE - 1))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (vma->vm_start & ~HPAGE_MASK)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (vma->vm_end & ~HPAGE_MASK)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (vma->vm_end - vma->vm_start < HPAGE_SIZE)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
vma_len = (loff_t)(vma->vm_end - vma->vm_start);
|
|
|
|
|
2006-01-09 16:59:24 -07:00
|
|
|
mutex_lock(&inode->i_mutex);
|
2005-04-16 16:20:36 -06:00
|
|
|
file_accessed(file);
|
|
|
|
vma->vm_flags |= VM_HUGETLB | VM_RESERVED;
|
|
|
|
vma->vm_ops = &hugetlb_vm_ops;
|
|
|
|
|
|
|
|
ret = -ENOMEM;
|
|
|
|
len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
|
|
|
|
if (!(vma->vm_flags & VM_WRITE) && len > inode->i_size)
|
|
|
|
goto out;
|
|
|
|
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
if (vma->vm_flags & VM_MAYSHARE)
|
|
|
|
if (hugetlb_extend_reservation(info, len >> HPAGE_SHIFT) != 0)
|
|
|
|
goto out;
|
|
|
|
|
2005-10-29 19:16:46 -06:00
|
|
|
ret = 0;
|
|
|
|
hugetlb_prefault_arch_hook(vma->vm_mm);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (inode->i_size < len)
|
|
|
|
inode->i_size = len;
|
|
|
|
out:
|
2006-01-09 16:59:24 -07:00
|
|
|
mutex_unlock(&inode->i_mutex);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2005-10-29 19:16:30 -06:00
|
|
|
* Called under down_write(mmap_sem).
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
|
|
|
|
#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
|
|
|
|
unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
|
|
|
|
unsigned long len, unsigned long pgoff, unsigned long flags);
|
|
|
|
#else
|
|
|
|
static unsigned long
|
|
|
|
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
|
|
|
|
unsigned long len, unsigned long pgoff, unsigned long flags)
|
|
|
|
{
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
unsigned long start_addr;
|
|
|
|
|
|
|
|
if (len & ~HPAGE_MASK)
|
|
|
|
return -EINVAL;
|
|
|
|
if (len > TASK_SIZE)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
if (addr) {
|
|
|
|
addr = ALIGN(addr, HPAGE_SIZE);
|
|
|
|
vma = find_vma(mm, addr);
|
|
|
|
if (TASK_SIZE - len >= addr &&
|
|
|
|
(!vma || addr + len <= vma->vm_start))
|
|
|
|
return addr;
|
|
|
|
}
|
|
|
|
|
|
|
|
start_addr = mm->free_area_cache;
|
|
|
|
|
2005-06-21 18:14:49 -06:00
|
|
|
if (len <= mm->cached_hole_size)
|
|
|
|
start_addr = TASK_UNMAPPED_BASE;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
full_search:
|
|
|
|
addr = ALIGN(start_addr, HPAGE_SIZE);
|
|
|
|
|
|
|
|
for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
|
|
|
|
/* At this point: (!vma || addr < vma->vm_end). */
|
|
|
|
if (TASK_SIZE - len < addr) {
|
|
|
|
/*
|
|
|
|
* Start a new search - just in case we missed
|
|
|
|
* some holes.
|
|
|
|
*/
|
|
|
|
if (start_addr != TASK_UNMAPPED_BASE) {
|
|
|
|
start_addr = TASK_UNMAPPED_BASE;
|
|
|
|
goto full_search;
|
|
|
|
}
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!vma || addr + len <= vma->vm_start)
|
|
|
|
return addr;
|
|
|
|
addr = ALIGN(vma->vm_end, HPAGE_SIZE);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Read a page. Again trivial. If it didn't already exist
|
|
|
|
* in the page cache, it is zero-filled.
|
|
|
|
*/
|
|
|
|
static int hugetlbfs_readpage(struct file *file, struct page * page)
|
|
|
|
{
|
|
|
|
unlock_page(page);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int hugetlbfs_prepare_write(struct file *file,
|
|
|
|
struct page *page, unsigned offset, unsigned to)
|
|
|
|
{
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int hugetlbfs_commit_write(struct file *file,
|
|
|
|
struct page *page, unsigned offset, unsigned to)
|
|
|
|
{
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void truncate_huge_page(struct page *page)
|
|
|
|
{
|
|
|
|
clear_page_dirty(page);
|
|
|
|
ClearPageUptodate(page);
|
|
|
|
remove_from_page_cache(page);
|
|
|
|
put_page(page);
|
|
|
|
}
|
|
|
|
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
static void truncate_hugepages(struct inode *inode, loff_t lstart)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
struct address_space *mapping = &inode->i_data;
|
2005-04-16 16:20:36 -06:00
|
|
|
const pgoff_t start = lstart >> HPAGE_SHIFT;
|
|
|
|
struct pagevec pvec;
|
|
|
|
pgoff_t next;
|
|
|
|
int i;
|
|
|
|
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
hugetlb_truncate_reservation(HUGETLBFS_I(inode),
|
|
|
|
lstart >> HPAGE_SHIFT);
|
|
|
|
if (!mapping->nrpages)
|
|
|
|
return;
|
2005-04-16 16:20:36 -06:00
|
|
|
pagevec_init(&pvec, 0);
|
|
|
|
next = start;
|
|
|
|
while (1) {
|
|
|
|
if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
|
|
|
|
if (next == start)
|
|
|
|
break;
|
|
|
|
next = start;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < pagevec_count(&pvec); ++i) {
|
|
|
|
struct page *page = pvec.pages[i];
|
|
|
|
|
|
|
|
lock_page(page);
|
|
|
|
if (page->index > next)
|
|
|
|
next = page->index;
|
|
|
|
++next;
|
|
|
|
truncate_huge_page(page);
|
|
|
|
unlock_page(page);
|
|
|
|
hugetlb_put_quota(mapping);
|
|
|
|
}
|
|
|
|
huge_pagevec_release(&pvec);
|
|
|
|
}
|
|
|
|
BUG_ON(!lstart && mapping->nrpages);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hugetlbfs_delete_inode(struct inode *inode)
|
|
|
|
{
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
truncate_hugepages(inode, 0);
|
2005-10-29 19:16:43 -06:00
|
|
|
clear_inode(inode);
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static void hugetlbfs_forget_inode(struct inode *inode)
|
|
|
|
{
|
2005-10-29 19:16:45 -06:00
|
|
|
struct super_block *sb = inode->i_sb;
|
|
|
|
|
|
|
|
if (!hlist_unhashed(&inode->i_hash)) {
|
|
|
|
if (!(inode->i_state & (I_DIRTY|I_LOCK)))
|
|
|
|
list_move(&inode->i_list, &inode_unused);
|
|
|
|
inodes_stat.nr_unused++;
|
|
|
|
if (!sb || (sb->s_flags & MS_ACTIVE)) {
|
|
|
|
spin_unlock(&inode_lock);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
inode->i_state |= I_WILL_FREE;
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_unlock(&inode_lock);
|
2005-10-29 19:16:45 -06:00
|
|
|
/*
|
|
|
|
* write_inode_now is a noop as we set BDI_CAP_NO_WRITEBACK
|
|
|
|
* in our backing_dev_info.
|
|
|
|
*/
|
|
|
|
write_inode_now(inode, 1);
|
|
|
|
spin_lock(&inode_lock);
|
|
|
|
inode->i_state &= ~I_WILL_FREE;
|
|
|
|
inodes_stat.nr_unused--;
|
|
|
|
hlist_del_init(&inode->i_hash);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
list_del_init(&inode->i_list);
|
|
|
|
list_del_init(&inode->i_sb_list);
|
|
|
|
inode->i_state |= I_FREEING;
|
|
|
|
inodes_stat.nr_inodes--;
|
|
|
|
spin_unlock(&inode_lock);
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
truncate_hugepages(inode, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
clear_inode(inode);
|
|
|
|
destroy_inode(inode);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hugetlbfs_drop_inode(struct inode *inode)
|
|
|
|
{
|
|
|
|
if (!inode->i_nlink)
|
2005-10-29 19:16:44 -06:00
|
|
|
generic_delete_inode(inode);
|
2005-04-16 16:20:36 -06:00
|
|
|
else
|
|
|
|
hugetlbfs_forget_inode(inode);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* h_pgoff is in HPAGE_SIZE units.
|
|
|
|
* vma->vm_pgoff is in PAGE_SIZE units.
|
|
|
|
*/
|
|
|
|
static inline void
|
|
|
|
hugetlb_vmtruncate_list(struct prio_tree_root *root, unsigned long h_pgoff)
|
|
|
|
{
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
struct prio_tree_iter iter;
|
|
|
|
|
|
|
|
vma_prio_tree_foreach(vma, &iter, root, h_pgoff, ULONG_MAX) {
|
|
|
|
unsigned long h_vm_pgoff;
|
|
|
|
unsigned long v_offset;
|
|
|
|
|
|
|
|
h_vm_pgoff = vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT);
|
|
|
|
v_offset = (h_pgoff - h_vm_pgoff) << HPAGE_SHIFT;
|
|
|
|
/*
|
|
|
|
* Is this VMA fully outside the truncation point?
|
|
|
|
*/
|
|
|
|
if (h_vm_pgoff >= h_pgoff)
|
|
|
|
v_offset = 0;
|
|
|
|
|
2005-10-29 19:16:30 -06:00
|
|
|
unmap_hugepage_range(vma,
|
|
|
|
vma->vm_start + v_offset, vma->vm_end);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Expanding truncates are not allowed.
|
|
|
|
*/
|
|
|
|
static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)
|
|
|
|
{
|
|
|
|
unsigned long pgoff;
|
|
|
|
struct address_space *mapping = inode->i_mapping;
|
|
|
|
|
|
|
|
if (offset > inode->i_size)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
BUG_ON(offset & ~HPAGE_MASK);
|
|
|
|
pgoff = offset >> HPAGE_SHIFT;
|
|
|
|
|
|
|
|
inode->i_size = offset;
|
|
|
|
spin_lock(&mapping->i_mmap_lock);
|
|
|
|
if (!prio_tree_empty(&mapping->i_mmap))
|
|
|
|
hugetlb_vmtruncate_list(&mapping->i_mmap, pgoff);
|
|
|
|
spin_unlock(&mapping->i_mmap_lock);
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
truncate_hugepages(inode, offset);
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr)
|
|
|
|
{
|
|
|
|
struct inode *inode = dentry->d_inode;
|
|
|
|
int error;
|
|
|
|
unsigned int ia_valid = attr->ia_valid;
|
|
|
|
|
|
|
|
BUG_ON(!inode);
|
|
|
|
|
|
|
|
error = inode_change_ok(inode, attr);
|
|
|
|
if (error)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
if (ia_valid & ATTR_SIZE) {
|
|
|
|
error = -EINVAL;
|
|
|
|
if (!(attr->ia_size & ~HPAGE_MASK))
|
|
|
|
error = hugetlb_vmtruncate(inode, attr->ia_size);
|
|
|
|
if (error)
|
|
|
|
goto out;
|
|
|
|
attr->ia_valid &= ~ATTR_SIZE;
|
|
|
|
}
|
|
|
|
error = inode_setattr(inode, attr);
|
|
|
|
out:
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct inode *hugetlbfs_get_inode(struct super_block *sb, uid_t uid,
|
|
|
|
gid_t gid, int mode, dev_t dev)
|
|
|
|
{
|
|
|
|
struct inode *inode;
|
|
|
|
|
|
|
|
inode = new_inode(sb);
|
|
|
|
if (inode) {
|
|
|
|
struct hugetlbfs_inode_info *info;
|
|
|
|
inode->i_mode = mode;
|
|
|
|
inode->i_uid = uid;
|
|
|
|
inode->i_gid = gid;
|
|
|
|
inode->i_blksize = HPAGE_SIZE;
|
|
|
|
inode->i_blocks = 0;
|
|
|
|
inode->i_mapping->a_ops = &hugetlbfs_aops;
|
|
|
|
inode->i_mapping->backing_dev_info =&hugetlbfs_backing_dev_info;
|
|
|
|
inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
|
|
|
|
info = HUGETLBFS_I(inode);
|
2006-01-14 14:20:48 -07:00
|
|
|
mpol_shared_policy_init(&info->policy, MPOL_DEFAULT, NULL);
|
2005-04-16 16:20:36 -06:00
|
|
|
switch (mode & S_IFMT) {
|
|
|
|
default:
|
|
|
|
init_special_inode(inode, mode, dev);
|
|
|
|
break;
|
|
|
|
case S_IFREG:
|
|
|
|
inode->i_op = &hugetlbfs_inode_operations;
|
|
|
|
inode->i_fop = &hugetlbfs_file_operations;
|
|
|
|
break;
|
|
|
|
case S_IFDIR:
|
|
|
|
inode->i_op = &hugetlbfs_dir_inode_operations;
|
|
|
|
inode->i_fop = &simple_dir_operations;
|
|
|
|
|
|
|
|
/* directory inodes start off with i_nlink == 2 (for "." entry) */
|
|
|
|
inode->i_nlink++;
|
|
|
|
break;
|
|
|
|
case S_IFLNK:
|
|
|
|
inode->i_op = &page_symlink_inode_operations;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return inode;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* File creation. Allocate an inode, and we're done..
|
|
|
|
*/
|
|
|
|
static int hugetlbfs_mknod(struct inode *dir,
|
|
|
|
struct dentry *dentry, int mode, dev_t dev)
|
|
|
|
{
|
|
|
|
struct inode *inode;
|
|
|
|
int error = -ENOSPC;
|
|
|
|
gid_t gid;
|
|
|
|
|
|
|
|
if (dir->i_mode & S_ISGID) {
|
|
|
|
gid = dir->i_gid;
|
|
|
|
if (S_ISDIR(mode))
|
|
|
|
mode |= S_ISGID;
|
|
|
|
} else {
|
|
|
|
gid = current->fsgid;
|
|
|
|
}
|
|
|
|
inode = hugetlbfs_get_inode(dir->i_sb, current->fsuid, gid, mode, dev);
|
|
|
|
if (inode) {
|
|
|
|
dir->i_ctime = dir->i_mtime = CURRENT_TIME;
|
|
|
|
d_instantiate(dentry, inode);
|
|
|
|
dget(dentry); /* Extra count - pin the dentry in core */
|
|
|
|
error = 0;
|
|
|
|
}
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, int mode)
|
|
|
|
{
|
|
|
|
int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
|
|
|
|
if (!retval)
|
|
|
|
dir->i_nlink++;
|
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, int mode, struct nameidata *nd)
|
|
|
|
{
|
|
|
|
return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int hugetlbfs_symlink(struct inode *dir,
|
|
|
|
struct dentry *dentry, const char *symname)
|
|
|
|
{
|
|
|
|
struct inode *inode;
|
|
|
|
int error = -ENOSPC;
|
|
|
|
gid_t gid;
|
|
|
|
|
|
|
|
if (dir->i_mode & S_ISGID)
|
|
|
|
gid = dir->i_gid;
|
|
|
|
else
|
|
|
|
gid = current->fsgid;
|
|
|
|
|
|
|
|
inode = hugetlbfs_get_inode(dir->i_sb, current->fsuid,
|
|
|
|
gid, S_IFLNK|S_IRWXUGO, 0);
|
|
|
|
if (inode) {
|
|
|
|
int l = strlen(symname)+1;
|
|
|
|
error = page_symlink(inode, symname, l);
|
|
|
|
if (!error) {
|
|
|
|
d_instantiate(dentry, inode);
|
|
|
|
dget(dentry);
|
|
|
|
} else
|
|
|
|
iput(inode);
|
|
|
|
}
|
|
|
|
dir->i_ctime = dir->i_mtime = CURRENT_TIME;
|
|
|
|
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For direct-IO reads into hugetlb pages
|
|
|
|
*/
|
|
|
|
static int hugetlbfs_set_page_dirty(struct page *page)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-06-23 03:02:58 -06:00
|
|
|
static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-06-23 03:02:58 -06:00
|
|
|
struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
buf->f_type = HUGETLBFS_MAGIC;
|
|
|
|
buf->f_bsize = HPAGE_SIZE;
|
|
|
|
if (sbinfo) {
|
|
|
|
spin_lock(&sbinfo->stat_lock);
|
2005-11-21 22:32:24 -07:00
|
|
|
/* If no limits set, just report 0 for max/free/used
|
|
|
|
* blocks, like simple_statfs() */
|
|
|
|
if (sbinfo->max_blocks >= 0) {
|
|
|
|
buf->f_blocks = sbinfo->max_blocks;
|
|
|
|
buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
|
|
|
|
buf->f_files = sbinfo->max_inodes;
|
|
|
|
buf->f_ffree = sbinfo->free_inodes;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
buf->f_namelen = NAME_MAX;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hugetlbfs_put_super(struct super_block *sb)
|
|
|
|
{
|
|
|
|
struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
|
|
|
|
|
|
|
|
if (sbi) {
|
|
|
|
sb->s_fs_info = NULL;
|
|
|
|
kfree(sbi);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-10-29 19:16:42 -06:00
|
|
|
static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
|
|
|
|
{
|
|
|
|
if (sbinfo->free_inodes >= 0) {
|
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
if (unlikely(!sbinfo->free_inodes)) {
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
sbinfo->free_inodes--;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
|
|
|
|
{
|
|
|
|
if (sbinfo->free_inodes >= 0) {
|
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
sbinfo->free_inodes++;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static kmem_cache_t *hugetlbfs_inode_cachep;
|
|
|
|
|
|
|
|
static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
|
|
|
|
{
|
2005-10-29 19:16:42 -06:00
|
|
|
struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
|
2005-04-16 16:20:36 -06:00
|
|
|
struct hugetlbfs_inode_info *p;
|
|
|
|
|
2005-10-29 19:16:42 -06:00
|
|
|
if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
|
|
|
|
return NULL;
|
2005-04-16 16:20:36 -06:00
|
|
|
p = kmem_cache_alloc(hugetlbfs_inode_cachep, SLAB_KERNEL);
|
2005-10-29 19:16:42 -06:00
|
|
|
if (unlikely(!p)) {
|
|
|
|
hugetlbfs_inc_free_inodes(sbinfo);
|
2005-04-16 16:20:36 -06:00
|
|
|
return NULL;
|
2005-10-29 19:16:42 -06:00
|
|
|
}
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
p->prereserved_hpages = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
return &p->vfs_inode;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hugetlbfs_destroy_inode(struct inode *inode)
|
|
|
|
{
|
2005-10-29 19:16:42 -06:00
|
|
|
hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
|
2005-04-16 16:20:36 -06:00
|
|
|
mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
|
|
|
|
kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct address_space_operations hugetlbfs_aops = {
|
|
|
|
.readpage = hugetlbfs_readpage,
|
|
|
|
.prepare_write = hugetlbfs_prepare_write,
|
|
|
|
.commit_write = hugetlbfs_commit_write,
|
|
|
|
.set_page_dirty = hugetlbfs_set_page_dirty,
|
|
|
|
};
|
|
|
|
|
2005-10-29 19:16:42 -06:00
|
|
|
|
|
|
|
static void init_once(void *foo, kmem_cache_t *cachep, unsigned long flags)
|
|
|
|
{
|
|
|
|
struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;
|
|
|
|
|
|
|
|
if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
|
|
|
|
SLAB_CTOR_CONSTRUCTOR)
|
|
|
|
inode_init_once(&ei->vfs_inode);
|
|
|
|
}
|
|
|
|
|
2006-03-28 02:56:42 -07:00
|
|
|
const struct file_operations hugetlbfs_file_operations = {
|
2005-04-16 16:20:36 -06:00
|
|
|
.mmap = hugetlbfs_file_mmap,
|
|
|
|
.fsync = simple_sync_file,
|
|
|
|
.get_unmapped_area = hugetlb_get_unmapped_area,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct inode_operations hugetlbfs_dir_inode_operations = {
|
|
|
|
.create = hugetlbfs_create,
|
|
|
|
.lookup = simple_lookup,
|
|
|
|
.link = simple_link,
|
|
|
|
.unlink = simple_unlink,
|
|
|
|
.symlink = hugetlbfs_symlink,
|
|
|
|
.mkdir = hugetlbfs_mkdir,
|
|
|
|
.rmdir = simple_rmdir,
|
|
|
|
.mknod = hugetlbfs_mknod,
|
|
|
|
.rename = simple_rename,
|
|
|
|
.setattr = hugetlbfs_setattr,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct inode_operations hugetlbfs_inode_operations = {
|
|
|
|
.setattr = hugetlbfs_setattr,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct super_operations hugetlbfs_ops = {
|
|
|
|
.alloc_inode = hugetlbfs_alloc_inode,
|
|
|
|
.destroy_inode = hugetlbfs_destroy_inode,
|
|
|
|
.statfs = hugetlbfs_statfs,
|
2005-10-29 19:16:43 -06:00
|
|
|
.delete_inode = hugetlbfs_delete_inode,
|
2005-04-16 16:20:36 -06:00
|
|
|
.drop_inode = hugetlbfs_drop_inode,
|
|
|
|
.put_super = hugetlbfs_put_super,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int
|
|
|
|
hugetlbfs_parse_options(char *options, struct hugetlbfs_config *pconfig)
|
|
|
|
{
|
|
|
|
char *opt, *value, *rest;
|
|
|
|
|
|
|
|
if (!options)
|
|
|
|
return 0;
|
|
|
|
while ((opt = strsep(&options, ",")) != NULL) {
|
|
|
|
if (!*opt)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
value = strchr(opt, '=');
|
|
|
|
if (!value || !*value)
|
|
|
|
return -EINVAL;
|
|
|
|
else
|
|
|
|
*value++ = '\0';
|
|
|
|
|
|
|
|
if (!strcmp(opt, "uid"))
|
|
|
|
pconfig->uid = simple_strtoul(value, &value, 0);
|
|
|
|
else if (!strcmp(opt, "gid"))
|
|
|
|
pconfig->gid = simple_strtoul(value, &value, 0);
|
|
|
|
else if (!strcmp(opt, "mode"))
|
|
|
|
pconfig->mode = simple_strtoul(value,&value,0) & 0777U;
|
|
|
|
else if (!strcmp(opt, "size")) {
|
|
|
|
unsigned long long size = memparse(value, &rest);
|
|
|
|
if (*rest == '%') {
|
|
|
|
size <<= HPAGE_SHIFT;
|
|
|
|
size *= max_huge_pages;
|
|
|
|
do_div(size, 100);
|
|
|
|
rest++;
|
|
|
|
}
|
|
|
|
size &= HPAGE_MASK;
|
|
|
|
pconfig->nr_blocks = (size >> HPAGE_SHIFT);
|
|
|
|
value = rest;
|
|
|
|
} else if (!strcmp(opt,"nr_inodes")) {
|
|
|
|
pconfig->nr_inodes = memparse(value, &rest);
|
|
|
|
value = rest;
|
|
|
|
} else
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (*value)
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
hugetlbfs_fill_super(struct super_block *sb, void *data, int silent)
|
|
|
|
{
|
|
|
|
struct inode * inode;
|
|
|
|
struct dentry * root;
|
|
|
|
int ret;
|
|
|
|
struct hugetlbfs_config config;
|
|
|
|
struct hugetlbfs_sb_info *sbinfo;
|
|
|
|
|
|
|
|
config.nr_blocks = -1; /* No limit on size by default */
|
|
|
|
config.nr_inodes = -1; /* No limit on number of inodes by default */
|
|
|
|
config.uid = current->fsuid;
|
|
|
|
config.gid = current->fsgid;
|
|
|
|
config.mode = 0755;
|
|
|
|
ret = hugetlbfs_parse_options(data, &config);
|
|
|
|
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
|
|
|
|
if (!sbinfo)
|
|
|
|
return -ENOMEM;
|
|
|
|
sb->s_fs_info = sbinfo;
|
|
|
|
spin_lock_init(&sbinfo->stat_lock);
|
|
|
|
sbinfo->max_blocks = config.nr_blocks;
|
|
|
|
sbinfo->free_blocks = config.nr_blocks;
|
|
|
|
sbinfo->max_inodes = config.nr_inodes;
|
|
|
|
sbinfo->free_inodes = config.nr_inodes;
|
|
|
|
sb->s_maxbytes = MAX_LFS_FILESIZE;
|
|
|
|
sb->s_blocksize = HPAGE_SIZE;
|
|
|
|
sb->s_blocksize_bits = HPAGE_SHIFT;
|
|
|
|
sb->s_magic = HUGETLBFS_MAGIC;
|
|
|
|
sb->s_op = &hugetlbfs_ops;
|
|
|
|
sb->s_time_gran = 1;
|
|
|
|
inode = hugetlbfs_get_inode(sb, config.uid, config.gid,
|
|
|
|
S_IFDIR | config.mode, 0);
|
|
|
|
if (!inode)
|
|
|
|
goto out_free;
|
|
|
|
|
|
|
|
root = d_alloc_root(inode);
|
|
|
|
if (!root) {
|
|
|
|
iput(inode);
|
|
|
|
goto out_free;
|
|
|
|
}
|
|
|
|
sb->s_root = root;
|
|
|
|
return 0;
|
|
|
|
out_free:
|
|
|
|
kfree(sbinfo);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
int hugetlb_get_quota(struct address_space *mapping)
|
|
|
|
{
|
|
|
|
int ret = 0;
|
|
|
|
struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(mapping->host->i_sb);
|
|
|
|
|
|
|
|
if (sbinfo->free_blocks > -1) {
|
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
if (sbinfo->free_blocks > 0)
|
|
|
|
sbinfo->free_blocks--;
|
|
|
|
else
|
|
|
|
ret = -ENOMEM;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
void hugetlb_put_quota(struct address_space *mapping)
|
|
|
|
{
|
|
|
|
struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(mapping->host->i_sb);
|
|
|
|
|
|
|
|
if (sbinfo->free_blocks > -1) {
|
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
sbinfo->free_blocks++;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[PATCH] VFS: Permit filesystem to override root dentry on mount
Extend the get_sb() filesystem operation to take an extra argument that
permits the VFS to pass in the target vfsmount that defines the mountpoint.
The filesystem is then required to manually set the superblock and root dentry
pointers. For most filesystems, this should be done with simple_set_mnt()
which will set the superblock pointer and then set the root dentry to the
superblock's s_root (as per the old default behaviour).
The get_sb() op now returns an integer as there's now no need to return the
superblock pointer.
This patch permits a superblock to be implicitly shared amongst several mount
points, such as can be done with NFS to avoid potential inode aliasing. In
such a case, simple_set_mnt() would not be called, and instead the mnt_root
and mnt_sb would be set directly.
The patch also makes the following changes:
(*) the get_sb_*() convenience functions in the core kernel now take a vfsmount
pointer argument and return an integer, so most filesystems have to change
very little.
(*) If one of the convenience function is not used, then get_sb() should
normally call simple_set_mnt() to instantiate the vfsmount. This will
always return 0, and so can be tail-called from get_sb().
(*) generic_shutdown_super() now calls shrink_dcache_sb() to clean up the
dcache upon superblock destruction rather than shrink_dcache_anon().
This is required because the superblock may now have multiple trees that
aren't actually bound to s_root, but that still need to be cleaned up. The
currently called functions assume that the whole tree is rooted at s_root,
and that anonymous dentries are not the roots of trees which results in
dentries being left unculled.
However, with the way NFS superblock sharing are currently set to be
implemented, these assumptions are violated: the root of the filesystem is
simply a dummy dentry and inode (the real inode for '/' may well be
inaccessible), and all the vfsmounts are rooted on anonymous[*] dentries
with child trees.
[*] Anonymous until discovered from another tree.
(*) The documentation has been adjusted, including the additional bit of
changing ext2_* into foo_* in the documentation.
[akpm@osdl.org: convert ipath_fs, do other stuff]
Signed-off-by: David Howells <dhowells@redhat.com>
Acked-by: Al Viro <viro@zeniv.linux.org.uk>
Cc: Nathan Scott <nathans@sgi.com>
Cc: Roland Dreier <rolandd@cisco.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-23 03:02:57 -06:00
|
|
|
static int hugetlbfs_get_sb(struct file_system_type *fs_type,
|
|
|
|
int flags, const char *dev_name, void *data, struct vfsmount *mnt)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
[PATCH] VFS: Permit filesystem to override root dentry on mount
Extend the get_sb() filesystem operation to take an extra argument that
permits the VFS to pass in the target vfsmount that defines the mountpoint.
The filesystem is then required to manually set the superblock and root dentry
pointers. For most filesystems, this should be done with simple_set_mnt()
which will set the superblock pointer and then set the root dentry to the
superblock's s_root (as per the old default behaviour).
The get_sb() op now returns an integer as there's now no need to return the
superblock pointer.
This patch permits a superblock to be implicitly shared amongst several mount
points, such as can be done with NFS to avoid potential inode aliasing. In
such a case, simple_set_mnt() would not be called, and instead the mnt_root
and mnt_sb would be set directly.
The patch also makes the following changes:
(*) the get_sb_*() convenience functions in the core kernel now take a vfsmount
pointer argument and return an integer, so most filesystems have to change
very little.
(*) If one of the convenience function is not used, then get_sb() should
normally call simple_set_mnt() to instantiate the vfsmount. This will
always return 0, and so can be tail-called from get_sb().
(*) generic_shutdown_super() now calls shrink_dcache_sb() to clean up the
dcache upon superblock destruction rather than shrink_dcache_anon().
This is required because the superblock may now have multiple trees that
aren't actually bound to s_root, but that still need to be cleaned up. The
currently called functions assume that the whole tree is rooted at s_root,
and that anonymous dentries are not the roots of trees which results in
dentries being left unculled.
However, with the way NFS superblock sharing are currently set to be
implemented, these assumptions are violated: the root of the filesystem is
simply a dummy dentry and inode (the real inode for '/' may well be
inaccessible), and all the vfsmounts are rooted on anonymous[*] dentries
with child trees.
[*] Anonymous until discovered from another tree.
(*) The documentation has been adjusted, including the additional bit of
changing ext2_* into foo_* in the documentation.
[akpm@osdl.org: convert ipath_fs, do other stuff]
Signed-off-by: David Howells <dhowells@redhat.com>
Acked-by: Al Viro <viro@zeniv.linux.org.uk>
Cc: Nathan Scott <nathans@sgi.com>
Cc: Roland Dreier <rolandd@cisco.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-23 03:02:57 -06:00
|
|
|
return get_sb_nodev(fs_type, flags, data, hugetlbfs_fill_super, mnt);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct file_system_type hugetlbfs_fs_type = {
|
|
|
|
.name = "hugetlbfs",
|
|
|
|
.get_sb = hugetlbfs_get_sb,
|
|
|
|
.kill_sb = kill_litter_super,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct vfsmount *hugetlbfs_vfsmount;
|
|
|
|
|
|
|
|
static int can_do_hugetlb_shm(void)
|
|
|
|
{
|
|
|
|
return likely(capable(CAP_IPC_LOCK) ||
|
|
|
|
in_group_p(sysctl_hugetlb_shm_group) ||
|
|
|
|
can_do_mlock());
|
|
|
|
}
|
|
|
|
|
|
|
|
struct file *hugetlb_zero_setup(size_t size)
|
|
|
|
{
|
|
|
|
int error = -ENOMEM;
|
|
|
|
struct file *file;
|
|
|
|
struct inode *inode;
|
|
|
|
struct dentry *dentry, *root;
|
|
|
|
struct qstr quick_string;
|
|
|
|
char buf[16];
|
2006-03-22 01:09:02 -07:00
|
|
|
static atomic_t counter;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (!can_do_hugetlb_shm())
|
|
|
|
return ERR_PTR(-EPERM);
|
|
|
|
|
|
|
|
if (!user_shm_lock(size, current->user))
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
|
|
|
|
root = hugetlbfs_vfsmount->mnt_root;
|
2006-03-22 01:09:02 -07:00
|
|
|
snprintf(buf, 16, "%u", atomic_inc_return(&counter));
|
2005-04-16 16:20:36 -06:00
|
|
|
quick_string.name = buf;
|
|
|
|
quick_string.len = strlen(quick_string.name);
|
|
|
|
quick_string.hash = 0;
|
|
|
|
dentry = d_alloc(root, &quick_string);
|
|
|
|
if (!dentry)
|
|
|
|
goto out_shm_unlock;
|
|
|
|
|
|
|
|
error = -ENFILE;
|
|
|
|
file = get_empty_filp();
|
|
|
|
if (!file)
|
|
|
|
goto out_dentry;
|
|
|
|
|
|
|
|
error = -ENOSPC;
|
|
|
|
inode = hugetlbfs_get_inode(root->d_sb, current->fsuid,
|
|
|
|
current->fsgid, S_IFREG | S_IRWXUGO, 0);
|
|
|
|
if (!inode)
|
|
|
|
goto out_file;
|
|
|
|
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
error = -ENOMEM;
|
|
|
|
if (hugetlb_extend_reservation(HUGETLBFS_I(inode),
|
|
|
|
size >> HPAGE_SHIFT) != 0)
|
|
|
|
goto out_inode;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
d_instantiate(dentry, inode);
|
|
|
|
inode->i_size = size;
|
|
|
|
inode->i_nlink = 0;
|
|
|
|
file->f_vfsmnt = mntget(hugetlbfs_vfsmount);
|
|
|
|
file->f_dentry = dentry;
|
|
|
|
file->f_mapping = inode->i_mapping;
|
|
|
|
file->f_op = &hugetlbfs_file_operations;
|
|
|
|
file->f_mode = FMODE_WRITE | FMODE_READ;
|
|
|
|
return file;
|
|
|
|
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
out_inode:
|
|
|
|
iput(inode);
|
2005-04-16 16:20:36 -06:00
|
|
|
out_file:
|
|
|
|
put_filp(file);
|
|
|
|
out_dentry:
|
|
|
|
dput(dentry);
|
|
|
|
out_shm_unlock:
|
|
|
|
user_shm_unlock(size, current->user);
|
|
|
|
return ERR_PTR(error);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init init_hugetlbfs_fs(void)
|
|
|
|
{
|
|
|
|
int error;
|
|
|
|
struct vfsmount *vfsmount;
|
|
|
|
|
|
|
|
hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
|
|
|
|
sizeof(struct hugetlbfs_inode_info),
|
|
|
|
0, 0, init_once, NULL);
|
|
|
|
if (hugetlbfs_inode_cachep == NULL)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
error = register_filesystem(&hugetlbfs_fs_type);
|
|
|
|
if (error)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
vfsmount = kern_mount(&hugetlbfs_fs_type);
|
|
|
|
|
|
|
|
if (!IS_ERR(vfsmount)) {
|
|
|
|
hugetlbfs_vfsmount = vfsmount;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
error = PTR_ERR(vfsmount);
|
|
|
|
|
|
|
|
out:
|
|
|
|
if (error)
|
|
|
|
kmem_cache_destroy(hugetlbfs_inode_cachep);
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __exit exit_hugetlbfs_fs(void)
|
|
|
|
{
|
|
|
|
kmem_cache_destroy(hugetlbfs_inode_cachep);
|
|
|
|
unregister_filesystem(&hugetlbfs_fs_type);
|
|
|
|
}
|
|
|
|
|
|
|
|
module_init(init_hugetlbfs_fs)
|
|
|
|
module_exit(exit_hugetlbfs_fs)
|
|
|
|
|
|
|
|
MODULE_LICENSE("GPL");
|