2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* hugetlbpage-backed filesystem. Based on ramfs.
|
|
|
|
*
|
2012-12-06 02:39:54 -07:00
|
|
|
* Nadia Yvette Chambers, 2002
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* Copyright (C) 2002 Linus Torvalds.
|
|
|
|
*/
|
|
|
|
|
2014-06-04 17:07:21 -06:00
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/thread_info.h>
|
|
|
|
#include <asm/current.h>
|
|
|
|
#include <linux/sched.h> /* remove ASAP */
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/mount.h>
|
|
|
|
#include <linux/file.h>
|
2007-07-16 00:40:52 -06:00
|
|
|
#include <linux/kernel.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/writeback.h>
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
#include <linux/highmem.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/string.h>
|
2006-01-11 13:17:46 -07:00
|
|
|
#include <linux/capability.h>
|
2007-07-16 00:40:52 -06:00
|
|
|
#include <linux/ctype.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/backing-dev.h>
|
|
|
|
#include <linux/hugetlb.h>
|
|
|
|
#include <linux/pagevec.h>
|
2007-07-16 00:40:52 -06:00
|
|
|
#include <linux/parser.h>
|
2007-05-06 15:50:12 -06:00
|
|
|
#include <linux/mman.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/dnotify.h>
|
|
|
|
#include <linux/statfs.h>
|
|
|
|
#include <linux/security.h>
|
2009-09-22 17:43:33 -06:00
|
|
|
#include <linux/magic.h>
|
2010-09-07 19:19:35 -06:00
|
|
|
#include <linux/migrate.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
#include <asm/uaccess.h>
|
|
|
|
|
2007-02-12 01:55:41 -07:00
|
|
|
static const struct super_operations hugetlbfs_ops;
|
2006-06-28 05:26:44 -06:00
|
|
|
static const struct address_space_operations hugetlbfs_aops;
|
2006-03-28 02:56:42 -07:00
|
|
|
const struct file_operations hugetlbfs_file_operations;
|
2007-02-12 01:55:39 -07:00
|
|
|
static const struct inode_operations hugetlbfs_dir_inode_operations;
|
|
|
|
static const struct inode_operations hugetlbfs_inode_operations;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-03-21 17:34:12 -06:00
|
|
|
struct hugetlbfs_config {
|
2012-02-07 17:19:25 -07:00
|
|
|
kuid_t uid;
|
|
|
|
kgid_t gid;
|
2012-03-21 17:34:12 -06:00
|
|
|
umode_t mode;
|
|
|
|
long nr_blocks;
|
|
|
|
long nr_inodes;
|
|
|
|
struct hstate *hstate;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct hugetlbfs_inode_info {
|
|
|
|
struct shared_policy policy;
|
|
|
|
struct inode vfs_inode;
|
|
|
|
};
|
|
|
|
|
|
|
|
static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
|
|
|
|
{
|
|
|
|
return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static struct backing_dev_info hugetlbfs_backing_dev_info = {
|
2009-06-12 06:45:52 -06:00
|
|
|
.name = "hugetlbfs",
|
2005-04-16 16:20:36 -06:00
|
|
|
.ra_pages = 0, /* No readahead */
|
2008-04-30 01:54:37 -06:00
|
|
|
.capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK,
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
int sysctl_hugetlb_shm_group;
|
|
|
|
|
2007-07-16 00:40:52 -06:00
|
|
|
enum {
|
|
|
|
Opt_size, Opt_nr_inodes,
|
|
|
|
Opt_mode, Opt_uid, Opt_gid,
|
2008-07-23 22:27:43 -06:00
|
|
|
Opt_pagesize,
|
2007-07-16 00:40:52 -06:00
|
|
|
Opt_err,
|
|
|
|
};
|
|
|
|
|
2008-10-13 03:46:57 -06:00
|
|
|
static const match_table_t tokens = {
|
2007-07-16 00:40:52 -06:00
|
|
|
{Opt_size, "size=%s"},
|
|
|
|
{Opt_nr_inodes, "nr_inodes=%s"},
|
|
|
|
{Opt_mode, "mode=%o"},
|
|
|
|
{Opt_uid, "uid=%u"},
|
|
|
|
{Opt_gid, "gid=%u"},
|
2008-07-23 22:27:43 -06:00
|
|
|
{Opt_pagesize, "pagesize=%s"},
|
2007-07-16 00:40:52 -06:00
|
|
|
{Opt_err, NULL},
|
|
|
|
};
|
|
|
|
|
2005-10-29 19:16:47 -06:00
|
|
|
static void huge_pagevec_release(struct pagevec *pvec)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < pagevec_count(pvec); ++i)
|
|
|
|
put_page(pvec->pages[i]);
|
|
|
|
|
|
|
|
pagevec_reinit(pvec);
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
|
|
|
|
{
|
2013-01-23 15:07:38 -07:00
|
|
|
struct inode *inode = file_inode(file);
|
2005-04-16 16:20:36 -06:00
|
|
|
loff_t len, vma_len;
|
|
|
|
int ret;
|
2008-07-23 22:27:41 -06:00
|
|
|
struct hstate *h = hstate_file(file);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
[PATCH] hugetlb: prepare_hugepage_range check offset too
(David:)
If hugetlbfs_file_mmap() returns a failure to do_mmap_pgoff() - for example,
because the given file offset is not hugepage aligned - then do_mmap_pgoff
will go to the unmap_and_free_vma backout path.
But at this stage the vma hasn't been marked as hugepage, and the backout path
will call unmap_region() on it. That will eventually call down to the
non-hugepage version of unmap_page_range(). On ppc64, at least, that will
cause serious problems if there are any existing hugepage pagetable entries in
the vicinity - for example if there are any other hugepage mappings under the
same PUD. unmap_page_range() will trigger a bad_pud() on the hugepage pud
entries. I suspect this will also cause bad problems on ia64, though I don't
have a machine to test it on.
(Hugh:)
prepare_hugepage_range() should check file offset alignment when it checks
virtual address and length, to stop MAP_FIXED with a bad huge offset from
unmapping before it fails further down. PowerPC should apply the same
prepare_hugepage_range alignment checks as ia64 and all the others do.
Then none of the alignment checks in hugetlbfs_file_mmap are required (nor
is the check for too small a mapping); but even so, move up setting of
VM_HUGETLB and add a comment to warn of what David Gibson discovered - if
hugetlbfs_file_mmap fails before setting it, do_mmap_pgoff's unmap_region
when unwinding from error will go the non-huge way, which may cause bad
behaviour on architectures (powerpc and ia64) which segregate their huge
mappings into a separate region of the address space.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: "David S. Miller" <davem@davemloft.net>
Acked-by: Adam Litke <agl@us.ibm.com>
Acked-by: David Gibson <david@gibson.dropbear.id.au>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-11-14 03:03:32 -07:00
|
|
|
/*
|
2007-08-31 00:56:40 -06:00
|
|
|
* vma address alignment (but not the pgoff alignment) has
|
|
|
|
* already been checked by prepare_hugepage_range. If you add
|
|
|
|
* any error returns here, do so after setting VM_HUGETLB, so
|
|
|
|
* is_vm_hugetlb_page tests below unmap_region go the right
|
|
|
|
* way when do_mmap_pgoff unwinds (may be important on powerpc
|
|
|
|
* and ia64).
|
[PATCH] hugetlb: prepare_hugepage_range check offset too
(David:)
If hugetlbfs_file_mmap() returns a failure to do_mmap_pgoff() - for example,
because the given file offset is not hugepage aligned - then do_mmap_pgoff
will go to the unmap_and_free_vma backout path.
But at this stage the vma hasn't been marked as hugepage, and the backout path
will call unmap_region() on it. That will eventually call down to the
non-hugepage version of unmap_page_range(). On ppc64, at least, that will
cause serious problems if there are any existing hugepage pagetable entries in
the vicinity - for example if there are any other hugepage mappings under the
same PUD. unmap_page_range() will trigger a bad_pud() on the hugepage pud
entries. I suspect this will also cause bad problems on ia64, though I don't
have a machine to test it on.
(Hugh:)
prepare_hugepage_range() should check file offset alignment when it checks
virtual address and length, to stop MAP_FIXED with a bad huge offset from
unmapping before it fails further down. PowerPC should apply the same
prepare_hugepage_range alignment checks as ia64 and all the others do.
Then none of the alignment checks in hugetlbfs_file_mmap are required (nor
is the check for too small a mapping); but even so, move up setting of
VM_HUGETLB and add a comment to warn of what David Gibson discovered - if
hugetlbfs_file_mmap fails before setting it, do_mmap_pgoff's unmap_region
when unwinding from error will go the non-huge way, which may cause bad
behaviour on architectures (powerpc and ia64) which segregate their huge
mappings into a separate region of the address space.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: "David S. Miller" <davem@davemloft.net>
Acked-by: Adam Litke <agl@us.ibm.com>
Acked-by: David Gibson <david@gibson.dropbear.id.au>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-11-14 03:03:32 -07:00
|
|
|
*/
|
2013-04-17 16:58:27 -06:00
|
|
|
vma->vm_flags |= VM_HUGETLB | VM_DONTEXPAND;
|
[PATCH] hugetlb: prepare_hugepage_range check offset too
(David:)
If hugetlbfs_file_mmap() returns a failure to do_mmap_pgoff() - for example,
because the given file offset is not hugepage aligned - then do_mmap_pgoff
will go to the unmap_and_free_vma backout path.
But at this stage the vma hasn't been marked as hugepage, and the backout path
will call unmap_region() on it. That will eventually call down to the
non-hugepage version of unmap_page_range(). On ppc64, at least, that will
cause serious problems if there are any existing hugepage pagetable entries in
the vicinity - for example if there are any other hugepage mappings under the
same PUD. unmap_page_range() will trigger a bad_pud() on the hugepage pud
entries. I suspect this will also cause bad problems on ia64, though I don't
have a machine to test it on.
(Hugh:)
prepare_hugepage_range() should check file offset alignment when it checks
virtual address and length, to stop MAP_FIXED with a bad huge offset from
unmapping before it fails further down. PowerPC should apply the same
prepare_hugepage_range alignment checks as ia64 and all the others do.
Then none of the alignment checks in hugetlbfs_file_mmap are required (nor
is the check for too small a mapping); but even so, move up setting of
VM_HUGETLB and add a comment to warn of what David Gibson discovered - if
hugetlbfs_file_mmap fails before setting it, do_mmap_pgoff's unmap_region
when unwinding from error will go the non-huge way, which may cause bad
behaviour on architectures (powerpc and ia64) which segregate their huge
mappings into a separate region of the address space.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: "David S. Miller" <davem@davemloft.net>
Acked-by: Adam Litke <agl@us.ibm.com>
Acked-by: David Gibson <david@gibson.dropbear.id.au>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-11-14 03:03:32 -07:00
|
|
|
vma->vm_ops = &hugetlb_vm_ops;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2011-07-25 18:11:49 -06:00
|
|
|
if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
|
2007-08-31 00:56:40 -06:00
|
|
|
return -EINVAL;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
vma_len = (loff_t)(vma->vm_end - vma->vm_start);
|
|
|
|
|
2006-01-09 16:59:24 -07:00
|
|
|
mutex_lock(&inode->i_mutex);
|
2005-04-16 16:20:36 -06:00
|
|
|
file_accessed(file);
|
|
|
|
|
|
|
|
ret = -ENOMEM;
|
|
|
|
len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
|
|
|
|
|
2008-07-23 22:27:23 -06:00
|
|
|
if (hugetlb_reserve_pages(inode,
|
2008-07-23 22:27:41 -06:00
|
|
|
vma->vm_pgoff >> huge_page_order(h),
|
2009-02-10 07:02:27 -07:00
|
|
|
len >> huge_page_shift(h), vma,
|
|
|
|
vma->vm_flags))
|
2006-06-23 03:03:15 -06:00
|
|
|
goto out;
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
|
2005-10-29 19:16:46 -06:00
|
|
|
ret = 0;
|
|
|
|
hugetlb_prefault_arch_hook(vma->vm_mm);
|
[PATCH] mmap zero-length hugetlb file with PROT_NONE to protect a hugetlb virtual area
Sometimes, applications need below call to be successful although
"/mnt/hugepages/file1" doesn't exist.
fd = open("/mnt/hugepages/file1", O_CREAT|O_RDWR, 0755);
*addr = mmap(NULL, 0x1024*1024*256, PROT_NONE, 0, fd, 0);
As for regular pages (or files), above call does work, but as for huge
pages, above call would fail because hugetlbfs_file_mmap would fail if
(!(vma->vm_flags & VM_WRITE) && len > inode->i_size).
This capability on huge page is useful on ia64 when the process wants to
protect one area on region 4, so other threads couldn't read/write this
area. A famous JVM (Java Virtual Machine) implementation on IA64 needs the
capability.
Signed-off-by: Zhang Yanmin <yanmin.zhang@intel.com>
Cc: David Gibson <david@gibson.dropbear.id.au>
Cc: Hugh Dickins <hugh@veritas.com>
[ Expand-on-mmap semantics again... this time matching normal fs's. wli ]
Acked-by: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-07-10 05:44:49 -06:00
|
|
|
if (vma->vm_flags & VM_WRITE && inode->i_size < len)
|
2005-04-16 16:20:36 -06:00
|
|
|
inode->i_size = len;
|
|
|
|
out:
|
2006-01-09 16:59:24 -07:00
|
|
|
mutex_unlock(&inode->i_mutex);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2005-10-29 19:16:30 -06:00
|
|
|
* Called under down_write(mmap_sem).
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
|
2007-05-06 15:49:00 -06:00
|
|
|
#ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
|
2005-04-16 16:20:36 -06:00
|
|
|
static unsigned long
|
|
|
|
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
|
|
|
|
unsigned long len, unsigned long pgoff, unsigned long flags)
|
|
|
|
{
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
struct vm_area_struct *vma;
|
2008-07-23 22:27:41 -06:00
|
|
|
struct hstate *h = hstate_file(file);
|
2012-12-11 17:02:00 -07:00
|
|
|
struct vm_unmapped_area_info info;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-07-23 22:27:41 -06:00
|
|
|
if (len & ~huge_page_mask(h))
|
2005-04-16 16:20:36 -06:00
|
|
|
return -EINVAL;
|
|
|
|
if (len > TASK_SIZE)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2007-05-06 15:50:12 -06:00
|
|
|
if (flags & MAP_FIXED) {
|
2008-07-23 22:27:41 -06:00
|
|
|
if (prepare_hugepage_range(file, addr, len))
|
2007-05-06 15:50:12 -06:00
|
|
|
return -EINVAL;
|
|
|
|
return addr;
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (addr) {
|
2008-07-23 22:27:41 -06:00
|
|
|
addr = ALIGN(addr, huge_page_size(h));
|
2005-04-16 16:20:36 -06:00
|
|
|
vma = find_vma(mm, addr);
|
|
|
|
if (TASK_SIZE - len >= addr &&
|
|
|
|
(!vma || addr + len <= vma->vm_start))
|
|
|
|
return addr;
|
|
|
|
}
|
|
|
|
|
2012-12-11 17:02:00 -07:00
|
|
|
info.flags = 0;
|
|
|
|
info.length = len;
|
|
|
|
info.low_limit = TASK_UNMAPPED_BASE;
|
|
|
|
info.high_limit = TASK_SIZE;
|
|
|
|
info.align_mask = PAGE_MASK & ~huge_page_mask(h);
|
|
|
|
info.align_offset = 0;
|
|
|
|
return vm_unmapped_area(&info);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2007-10-16 02:26:22 -06:00
|
|
|
static int
|
|
|
|
hugetlbfs_read_actor(struct page *page, unsigned long offset,
|
|
|
|
char __user *buf, unsigned long count,
|
|
|
|
unsigned long size)
|
|
|
|
{
|
|
|
|
char *kaddr;
|
|
|
|
unsigned long left, copied = 0;
|
|
|
|
int i, chunksize;
|
|
|
|
|
|
|
|
if (size > count)
|
|
|
|
size = count;
|
|
|
|
|
|
|
|
/* Find which 4k chunk and offset with in that chunk */
|
|
|
|
i = offset >> PAGE_CACHE_SHIFT;
|
|
|
|
offset = offset & ~PAGE_CACHE_MASK;
|
|
|
|
|
|
|
|
while (size) {
|
|
|
|
chunksize = PAGE_CACHE_SIZE;
|
|
|
|
if (offset)
|
|
|
|
chunksize -= offset;
|
|
|
|
if (chunksize > size)
|
|
|
|
chunksize = size;
|
|
|
|
kaddr = kmap(&page[i]);
|
|
|
|
left = __copy_to_user(buf, kaddr + offset, chunksize);
|
|
|
|
kunmap(&page[i]);
|
|
|
|
if (left) {
|
|
|
|
copied += (chunksize - left);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
offset = 0;
|
|
|
|
size -= chunksize;
|
|
|
|
buf += chunksize;
|
|
|
|
copied += chunksize;
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
return copied ? copied : -EFAULT;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Support for read() - Find the page attached to f_mapping and copy out the
|
|
|
|
* data. Its *very* similar to do_generic_mapping_read(), we can't use that
|
|
|
|
* since it has PAGE_CACHE_SIZE assumptions.
|
|
|
|
*/
|
|
|
|
static ssize_t hugetlbfs_read(struct file *filp, char __user *buf,
|
|
|
|
size_t len, loff_t *ppos)
|
|
|
|
{
|
2008-07-23 22:27:41 -06:00
|
|
|
struct hstate *h = hstate_file(filp);
|
2007-10-16 02:26:22 -06:00
|
|
|
struct address_space *mapping = filp->f_mapping;
|
|
|
|
struct inode *inode = mapping->host;
|
2008-07-23 22:27:41 -06:00
|
|
|
unsigned long index = *ppos >> huge_page_shift(h);
|
|
|
|
unsigned long offset = *ppos & ~huge_page_mask(h);
|
2007-10-16 02:26:22 -06:00
|
|
|
unsigned long end_index;
|
|
|
|
loff_t isize;
|
|
|
|
ssize_t retval = 0;
|
|
|
|
|
|
|
|
/* validate length */
|
|
|
|
if (len == 0)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
struct page *page;
|
2008-07-23 22:27:41 -06:00
|
|
|
unsigned long nr, ret;
|
2009-01-06 15:40:14 -07:00
|
|
|
int ra;
|
2007-10-16 02:26:22 -06:00
|
|
|
|
|
|
|
/* nr is the maximum number of bytes to copy from this page */
|
2008-07-23 22:27:41 -06:00
|
|
|
nr = huge_page_size(h);
|
2012-03-21 17:34:08 -06:00
|
|
|
isize = i_size_read(inode);
|
|
|
|
if (!isize)
|
|
|
|
goto out;
|
|
|
|
end_index = (isize - 1) >> huge_page_shift(h);
|
2007-10-16 02:26:22 -06:00
|
|
|
if (index >= end_index) {
|
|
|
|
if (index > end_index)
|
|
|
|
goto out;
|
2008-07-23 22:27:41 -06:00
|
|
|
nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
|
2012-03-21 17:34:08 -06:00
|
|
|
if (nr <= offset)
|
2007-10-16 02:26:22 -06:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
nr = nr - offset;
|
|
|
|
|
|
|
|
/* Find the page */
|
2012-03-21 17:34:08 -06:00
|
|
|
page = find_lock_page(mapping, index);
|
2007-10-16 02:26:22 -06:00
|
|
|
if (unlikely(page == NULL)) {
|
|
|
|
/*
|
|
|
|
* We have a HOLE, zero out the user-buffer for the
|
|
|
|
* length of the hole or request.
|
|
|
|
*/
|
|
|
|
ret = len < nr ? len : nr;
|
|
|
|
if (clear_user(buf, ret))
|
2009-01-06 15:40:14 -07:00
|
|
|
ra = -EFAULT;
|
|
|
|
else
|
|
|
|
ra = 0;
|
2007-10-16 02:26:22 -06:00
|
|
|
} else {
|
2012-03-21 17:34:08 -06:00
|
|
|
unlock_page(page);
|
|
|
|
|
2007-10-16 02:26:22 -06:00
|
|
|
/*
|
|
|
|
* We have the page, copy it to user space buffer.
|
|
|
|
*/
|
2009-01-06 15:40:14 -07:00
|
|
|
ra = hugetlbfs_read_actor(page, offset, buf, len, nr);
|
|
|
|
ret = ra;
|
2012-03-21 17:34:08 -06:00
|
|
|
page_cache_release(page);
|
2007-10-16 02:26:22 -06:00
|
|
|
}
|
2009-01-06 15:40:14 -07:00
|
|
|
if (ra < 0) {
|
2007-10-16 02:26:22 -06:00
|
|
|
if (retval == 0)
|
2009-01-06 15:40:14 -07:00
|
|
|
retval = ra;
|
2007-10-16 02:26:22 -06:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
offset += ret;
|
|
|
|
retval += ret;
|
|
|
|
len -= ret;
|
2008-07-23 22:27:41 -06:00
|
|
|
index += offset >> huge_page_shift(h);
|
|
|
|
offset &= ~huge_page_mask(h);
|
2007-10-16 02:26:22 -06:00
|
|
|
|
|
|
|
/* short read or no more work */
|
|
|
|
if ((ret != nr) || (len == 0))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
out:
|
2008-07-23 22:27:41 -06:00
|
|
|
*ppos = ((loff_t)index << huge_page_shift(h)) + offset;
|
2007-10-16 02:26:22 -06:00
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
|
2007-10-16 02:25:03 -06:00
|
|
|
static int hugetlbfs_write_begin(struct file *file,
|
|
|
|
struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned flags,
|
|
|
|
struct page **pagep, void **fsdata)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2007-10-16 02:25:03 -06:00
|
|
|
static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
|
|
struct page *page, void *fsdata)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-10-16 02:25:03 -06:00
|
|
|
BUG();
|
2005-04-16 16:20:36 -06:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void truncate_huge_page(struct page *page)
|
|
|
|
{
|
VM: Remove "clear_page_dirty()" and "test_clear_page_dirty()" functions
They were horribly easy to mis-use because of their tempting naming, and
they also did way more than any users of them generally wanted them to
do.
A dirty page can become clean under two circumstances:
(a) when we write it out. We have "clear_page_dirty_for_io()" for
this, and that function remains unchanged.
In the "for IO" case it is not sufficient to just clear the dirty
bit, you also have to mark the page as being under writeback etc.
(b) when we actually remove a page due to it becoming inaccessible to
users, notably because it was truncate()'d away or the file (or
metadata) no longer exists, and we thus want to cancel any
outstanding dirty state.
For the (b) case, we now introduce "cancel_dirty_page()", which only
touches the page state itself, and verifies that the page is not mapped
(since cancelling writes on a mapped page would be actively wrong as it
is still accessible to users).
Some filesystems need to be fixed up for this: CIFS, FUSE, JFS,
ReiserFS, XFS all use the old confusing functions, and will be fixed
separately in subsequent commits (with some of them just removing the
offending logic, and others using clear_page_dirty_for_io()).
This was confirmed by Martin Michlmayr to fix the apt database
corruption on ARM.
Cc: Martin Michlmayr <tbm@cyrius.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Arjan van de Ven <arjan@infradead.org>
Cc: Andrei Popa <andrei.popa@i-neo.ro>
Cc: Andrew Morton <akpm@osdl.org>
Cc: Dave Kleikamp <shaggy@linux.vnet.ibm.com>
Cc: Gordon Farquharson <gordonfarquharson@gmail.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Trond Myklebust <trond.myklebust@fys.uio.no>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-20 14:46:42 -07:00
|
|
|
cancel_dirty_page(page, /* No IO accounting for huge pages? */0);
|
2005-04-16 16:20:36 -06:00
|
|
|
ClearPageUptodate(page);
|
2011-03-22 17:30:54 -06:00
|
|
|
delete_from_page_cache(page);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
static void truncate_hugepages(struct inode *inode, loff_t lstart)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-07-23 22:27:41 -06:00
|
|
|
struct hstate *h = hstate_inode(inode);
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
struct address_space *mapping = &inode->i_data;
|
2008-07-23 22:27:41 -06:00
|
|
|
const pgoff_t start = lstart >> huge_page_shift(h);
|
2005-04-16 16:20:36 -06:00
|
|
|
struct pagevec pvec;
|
|
|
|
pgoff_t next;
|
2006-06-23 03:03:15 -06:00
|
|
|
int i, freed = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
pagevec_init(&pvec, 0);
|
|
|
|
next = start;
|
|
|
|
while (1) {
|
|
|
|
if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
|
|
|
|
if (next == start)
|
|
|
|
break;
|
|
|
|
next = start;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < pagevec_count(&pvec); ++i) {
|
|
|
|
struct page *page = pvec.pages[i];
|
|
|
|
|
|
|
|
lock_page(page);
|
|
|
|
if (page->index > next)
|
|
|
|
next = page->index;
|
|
|
|
++next;
|
|
|
|
truncate_huge_page(page);
|
|
|
|
unlock_page(page);
|
2006-06-23 03:03:15 -06:00
|
|
|
freed++;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
huge_pagevec_release(&pvec);
|
|
|
|
}
|
|
|
|
BUG_ON(!lstart && mapping->nrpages);
|
2006-06-23 03:03:15 -06:00
|
|
|
hugetlb_unreserve_pages(inode, start, freed);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2010-06-04 17:52:12 -06:00
|
|
|
static void hugetlbfs_evict_inode(struct inode *inode)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
mm, hugetlb: unify region structure handling
Currently, to track reserved and allocated regions, we use two different
ways, depending on the mapping. For MAP_SHARED, we use
address_mapping's private_list and, while for MAP_PRIVATE, we use a
resv_map.
Now, we are preparing to change a coarse grained lock which protect a
region structure to fine grained lock, and this difference hinder it.
So, before changing it, unify region structure handling, consistently
using a resv_map regardless of the kind of mapping.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-03 15:47:25 -06:00
|
|
|
struct resv_map *resv_map;
|
|
|
|
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
truncate_hugepages(inode, 0);
|
mm, hugetlb: unify region structure handling
Currently, to track reserved and allocated regions, we use two different
ways, depending on the mapping. For MAP_SHARED, we use
address_mapping's private_list and, while for MAP_PRIVATE, we use a
resv_map.
Now, we are preparing to change a coarse grained lock which protect a
region structure to fine grained lock, and this difference hinder it.
So, before changing it, unify region structure handling, consistently
using a resv_map regardless of the kind of mapping.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-03 15:47:25 -06:00
|
|
|
resv_map = (struct resv_map *)inode->i_mapping->private_data;
|
|
|
|
/* root inode doesn't have the resv_map, so we should check it */
|
|
|
|
if (resv_map)
|
|
|
|
resv_map_release(&resv_map->refs);
|
2012-05-03 06:48:02 -06:00
|
|
|
clear_inode(inode);
|
2005-10-29 19:16:43 -06:00
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static inline void
|
2012-10-08 17:31:25 -06:00
|
|
|
hugetlb_vmtruncate_list(struct rb_root *root, pgoff_t pgoff)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
2012-10-08 17:31:25 -06:00
|
|
|
vma_interval_tree_foreach(vma, root, pgoff, ULONG_MAX) {
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned long v_offset;
|
|
|
|
|
|
|
|
/*
|
2006-10-28 11:38:43 -06:00
|
|
|
* Can the expression below overflow on 32-bit arches?
|
2012-10-08 17:31:25 -06:00
|
|
|
* No, because the interval tree returns us only those vmas
|
2006-10-28 11:38:43 -06:00
|
|
|
* which overlap the truncated area starting at pgoff,
|
|
|
|
* and no vma on a 32-bit arch can span beyond the 4GB.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2006-10-28 11:38:43 -06:00
|
|
|
if (vma->vm_pgoff < pgoff)
|
|
|
|
v_offset = (pgoff - vma->vm_pgoff) << PAGE_SHIFT;
|
|
|
|
else
|
2005-04-16 16:20:36 -06:00
|
|
|
v_offset = 0;
|
|
|
|
|
2012-07-31 17:42:03 -06:00
|
|
|
unmap_hugepage_range(vma, vma->vm_start + v_offset,
|
|
|
|
vma->vm_end, NULL);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)
|
|
|
|
{
|
2006-10-28 11:38:43 -06:00
|
|
|
pgoff_t pgoff;
|
2005-04-16 16:20:36 -06:00
|
|
|
struct address_space *mapping = inode->i_mapping;
|
2008-07-23 22:27:41 -06:00
|
|
|
struct hstate *h = hstate_inode(inode);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-07-23 22:27:41 -06:00
|
|
|
BUG_ON(offset & ~huge_page_mask(h));
|
2006-10-28 11:38:43 -06:00
|
|
|
pgoff = offset >> PAGE_SHIFT;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-16 02:26:21 -06:00
|
|
|
i_size_write(inode, offset);
|
2014-12-12 17:54:21 -07:00
|
|
|
i_mmap_lock_write(mapping);
|
2012-10-08 17:31:25 -06:00
|
|
|
if (!RB_EMPTY_ROOT(&mapping->i_mmap))
|
2005-04-16 16:20:36 -06:00
|
|
|
hugetlb_vmtruncate_list(&mapping->i_mmap, pgoff);
|
2014-12-12 17:54:21 -07:00
|
|
|
i_mmap_unlock_write(mapping);
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
truncate_hugepages(inode, offset);
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr)
|
|
|
|
{
|
|
|
|
struct inode *inode = dentry->d_inode;
|
2008-07-23 22:27:41 -06:00
|
|
|
struct hstate *h = hstate_inode(inode);
|
2005-04-16 16:20:36 -06:00
|
|
|
int error;
|
|
|
|
unsigned int ia_valid = attr->ia_valid;
|
|
|
|
|
|
|
|
BUG_ON(!inode);
|
|
|
|
|
|
|
|
error = inode_change_ok(inode, attr);
|
|
|
|
if (error)
|
2010-06-04 03:30:02 -06:00
|
|
|
return error;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (ia_valid & ATTR_SIZE) {
|
|
|
|
error = -EINVAL;
|
2010-06-04 03:30:02 -06:00
|
|
|
if (attr->ia_size & ~huge_page_mask(h))
|
|
|
|
return -EINVAL;
|
|
|
|
error = hugetlb_vmtruncate(inode, attr->ia_size);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (error)
|
2010-06-04 03:30:02 -06:00
|
|
|
return error;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2010-06-04 03:30:02 -06:00
|
|
|
|
|
|
|
setattr_copy(inode, attr);
|
|
|
|
mark_inode_dirty(inode);
|
|
|
|
return 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2011-07-24 18:20:48 -06:00
|
|
|
static struct inode *hugetlbfs_get_root(struct super_block *sb,
|
|
|
|
struct hugetlbfs_config *config)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct inode *inode;
|
|
|
|
|
|
|
|
inode = new_inode(sb);
|
|
|
|
if (inode) {
|
|
|
|
struct hugetlbfs_inode_info *info;
|
2010-10-23 09:19:54 -06:00
|
|
|
inode->i_ino = get_next_ino();
|
2011-07-24 18:20:48 -06:00
|
|
|
inode->i_mode = S_IFDIR | config->mode;
|
|
|
|
inode->i_uid = config->uid;
|
|
|
|
inode->i_gid = config->gid;
|
|
|
|
inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
|
|
|
|
info = HUGETLBFS_I(inode);
|
|
|
|
mpol_shared_policy_init(&info->policy, NULL);
|
|
|
|
inode->i_op = &hugetlbfs_dir_inode_operations;
|
|
|
|
inode->i_fop = &simple_dir_operations;
|
|
|
|
/* directory inodes start off with i_nlink == 2 (for "." entry) */
|
|
|
|
inc_nlink(inode);
|
2012-04-25 17:01:50 -06:00
|
|
|
lockdep_annotate_inode_mutex_key(inode);
|
2011-07-24 18:20:48 -06:00
|
|
|
}
|
|
|
|
return inode;
|
|
|
|
}
|
|
|
|
|
2013-08-13 17:00:55 -06:00
|
|
|
/*
|
2014-12-12 17:54:24 -07:00
|
|
|
* Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never
|
2013-08-13 17:00:55 -06:00
|
|
|
* be taken from reclaim -- unlike regular filesystems. This needs an
|
|
|
|
* annotation because huge_pmd_share() does an allocation under
|
2014-12-12 17:54:24 -07:00
|
|
|
* i_mmap_rwsem.
|
2013-08-13 17:00:55 -06:00
|
|
|
*/
|
2014-12-12 17:54:24 -07:00
|
|
|
static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
|
2013-08-13 17:00:55 -06:00
|
|
|
|
2011-07-24 18:20:48 -06:00
|
|
|
static struct inode *hugetlbfs_get_inode(struct super_block *sb,
|
|
|
|
struct inode *dir,
|
2011-07-24 21:17:40 -06:00
|
|
|
umode_t mode, dev_t dev)
|
2011-07-24 18:20:48 -06:00
|
|
|
{
|
|
|
|
struct inode *inode;
|
mm, hugetlb: unify region structure handling
Currently, to track reserved and allocated regions, we use two different
ways, depending on the mapping. For MAP_SHARED, we use
address_mapping's private_list and, while for MAP_PRIVATE, we use a
resv_map.
Now, we are preparing to change a coarse grained lock which protect a
region structure to fine grained lock, and this difference hinder it.
So, before changing it, unify region structure handling, consistently
using a resv_map regardless of the kind of mapping.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-03 15:47:25 -06:00
|
|
|
struct resv_map *resv_map;
|
|
|
|
|
|
|
|
resv_map = resv_map_alloc();
|
|
|
|
if (!resv_map)
|
|
|
|
return NULL;
|
2011-07-24 18:20:48 -06:00
|
|
|
|
|
|
|
inode = new_inode(sb);
|
|
|
|
if (inode) {
|
|
|
|
struct hugetlbfs_inode_info *info;
|
|
|
|
inode->i_ino = get_next_ino();
|
|
|
|
inode_init_owner(inode, dir, mode);
|
2014-12-12 17:54:24 -07:00
|
|
|
lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
|
|
|
|
&hugetlbfs_i_mmap_rwsem_key);
|
2005-04-16 16:20:36 -06:00
|
|
|
inode->i_mapping->a_ops = &hugetlbfs_aops;
|
|
|
|
inode->i_mapping->backing_dev_info =&hugetlbfs_backing_dev_info;
|
|
|
|
inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
|
mm, hugetlb: unify region structure handling
Currently, to track reserved and allocated regions, we use two different
ways, depending on the mapping. For MAP_SHARED, we use
address_mapping's private_list and, while for MAP_PRIVATE, we use a
resv_map.
Now, we are preparing to change a coarse grained lock which protect a
region structure to fine grained lock, and this difference hinder it.
So, before changing it, unify region structure handling, consistently
using a resv_map regardless of the kind of mapping.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-03 15:47:25 -06:00
|
|
|
inode->i_mapping->private_data = resv_map;
|
2005-04-16 16:20:36 -06:00
|
|
|
info = HUGETLBFS_I(inode);
|
2009-09-21 18:03:43 -06:00
|
|
|
/*
|
|
|
|
* The policy is initialized here even if we are creating a
|
|
|
|
* private inode because initialization simply creates an
|
|
|
|
* an empty rb tree and calls spin_lock_init(), later when we
|
|
|
|
* call mpol_free_shared_policy() it will just return because
|
|
|
|
* the rb tree will still be empty.
|
|
|
|
*/
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 03:13:26 -06:00
|
|
|
mpol_shared_policy_init(&info->policy, NULL);
|
2005-04-16 16:20:36 -06:00
|
|
|
switch (mode & S_IFMT) {
|
|
|
|
default:
|
|
|
|
init_special_inode(inode, mode, dev);
|
|
|
|
break;
|
|
|
|
case S_IFREG:
|
|
|
|
inode->i_op = &hugetlbfs_inode_operations;
|
|
|
|
inode->i_fop = &hugetlbfs_file_operations;
|
|
|
|
break;
|
|
|
|
case S_IFDIR:
|
|
|
|
inode->i_op = &hugetlbfs_dir_inode_operations;
|
|
|
|
inode->i_fop = &simple_dir_operations;
|
|
|
|
|
|
|
|
/* directory inodes start off with i_nlink == 2 (for "." entry) */
|
2006-10-01 00:29:04 -06:00
|
|
|
inc_nlink(inode);
|
2005-04-16 16:20:36 -06:00
|
|
|
break;
|
|
|
|
case S_IFLNK:
|
|
|
|
inode->i_op = &page_symlink_inode_operations;
|
|
|
|
break;
|
|
|
|
}
|
lockdep: Add helper function for dir vs file i_mutex annotation
Purely in-memory filesystems do not use the inode hash as the dcache
tells us if an entry already exists. As a result, they do not call
unlock_new_inode, and thus directory inodes do not get put into a
different lockdep class for i_sem.
We need the different lockdep classes, because the locking order for
i_mutex is different for directory inodes and regular inodes. Directory
inodes can do "readdir()", which takes i_mutex *before* possibly taking
mm->mmap_sem (due to a page fault while copying the directory entry to
user space).
In contrast, regular inodes can be mmap'ed, which takes mm->mmap_sem
before accessing i_mutex.
The two cases can never happen for the same inode, so no real deadlock
can occur, but without the different lockdep classes, lockdep cannot
understand that. As a result, if CONFIG_DEBUG_LOCK_ALLOC is set, this
can lead to false positives from lockdep like below:
find/645 is trying to acquire lock:
(&mm->mmap_sem){++++++}, at: [<ffffffff81109514>] might_fault+0x5c/0xac
but task is already holding lock:
(&sb->s_type->i_mutex_key#15){+.+.+.}, at: [<ffffffff81149f34>]
vfs_readdir+0x5b/0xb4
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (&sb->s_type->i_mutex_key#15){+.+.+.}:
[<ffffffff8108ac26>] lock_acquire+0xbf/0x103
[<ffffffff814db822>] __mutex_lock_common+0x4c/0x361
[<ffffffff814dbc46>] mutex_lock_nested+0x40/0x45
[<ffffffff811daa87>] hugetlbfs_file_mmap+0x82/0x110
[<ffffffff81111557>] mmap_region+0x258/0x432
[<ffffffff811119dd>] do_mmap_pgoff+0x2ac/0x306
[<ffffffff81111b4f>] sys_mmap_pgoff+0x118/0x16a
[<ffffffff8100c858>] sys_mmap+0x22/0x24
[<ffffffff814e3ec2>] system_call_fastpath+0x16/0x1b
-> #0 (&mm->mmap_sem){++++++}:
[<ffffffff8108a4bc>] __lock_acquire+0xa1a/0xcf7
[<ffffffff8108ac26>] lock_acquire+0xbf/0x103
[<ffffffff81109541>] might_fault+0x89/0xac
[<ffffffff81149cff>] filldir+0x6f/0xc7
[<ffffffff811586ea>] dcache_readdir+0x67/0x205
[<ffffffff81149f54>] vfs_readdir+0x7b/0xb4
[<ffffffff8114a073>] sys_getdents+0x7e/0xd1
[<ffffffff814e3ec2>] system_call_fastpath+0x16/0x1b
This patch moves the directory vs file lockdep annotation into a helper
function that can be called by in-memory filesystems and has hugetlbfs
call it.
Signed-off-by: Josh Boyer <jwboyer@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-08-25 05:48:12 -06:00
|
|
|
lockdep_annotate_inode_mutex_key(inode);
|
mm, hugetlb: unify region structure handling
Currently, to track reserved and allocated regions, we use two different
ways, depending on the mapping. For MAP_SHARED, we use
address_mapping's private_list and, while for MAP_PRIVATE, we use a
resv_map.
Now, we are preparing to change a coarse grained lock which protect a
region structure to fine grained lock, and this difference hinder it.
So, before changing it, unify region structure handling, consistently
using a resv_map regardless of the kind of mapping.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-03 15:47:25 -06:00
|
|
|
} else
|
|
|
|
kref_put(&resv_map->refs, resv_map_release);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
return inode;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* File creation. Allocate an inode, and we're done..
|
|
|
|
*/
|
|
|
|
static int hugetlbfs_mknod(struct inode *dir,
|
2011-07-25 23:52:52 -06:00
|
|
|
struct dentry *dentry, umode_t mode, dev_t dev)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct inode *inode;
|
|
|
|
int error = -ENOSPC;
|
2011-07-24 18:20:48 -06:00
|
|
|
|
|
|
|
inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (inode) {
|
|
|
|
dir->i_ctime = dir->i_mtime = CURRENT_TIME;
|
|
|
|
d_instantiate(dentry, inode);
|
|
|
|
dget(dentry); /* Extra count - pin the dentry in core */
|
|
|
|
error = 0;
|
|
|
|
}
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
2011-07-25 23:41:39 -06:00
|
|
|
static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
|
|
|
|
if (!retval)
|
2006-10-01 00:29:04 -06:00
|
|
|
inc_nlink(dir);
|
2005-04-16 16:20:36 -06:00
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
|
2012-06-10 16:05:36 -06:00
|
|
|
static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int hugetlbfs_symlink(struct inode *dir,
|
|
|
|
struct dentry *dentry, const char *symname)
|
|
|
|
{
|
|
|
|
struct inode *inode;
|
|
|
|
int error = -ENOSPC;
|
|
|
|
|
2011-07-24 18:20:48 -06:00
|
|
|
inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (inode) {
|
|
|
|
int l = strlen(symname)+1;
|
|
|
|
error = page_symlink(inode, symname, l);
|
|
|
|
if (!error) {
|
|
|
|
d_instantiate(dentry, inode);
|
|
|
|
dget(dentry);
|
|
|
|
} else
|
|
|
|
iput(inode);
|
|
|
|
}
|
|
|
|
dir->i_ctime = dir->i_mtime = CURRENT_TIME;
|
|
|
|
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2007-02-08 15:20:27 -07:00
|
|
|
* mark the head page dirty
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
static int hugetlbfs_set_page_dirty(struct page *page)
|
|
|
|
{
|
2007-05-06 15:49:39 -06:00
|
|
|
struct page *head = compound_head(page);
|
2007-02-08 15:20:27 -07:00
|
|
|
|
|
|
|
SetPageDirty(head);
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-09-07 19:19:35 -06:00
|
|
|
static int hugetlbfs_migrate_page(struct address_space *mapping,
|
2012-01-12 18:19:34 -07:00
|
|
|
struct page *newpage, struct page *page,
|
2012-01-12 18:19:43 -07:00
|
|
|
enum migrate_mode mode)
|
2010-09-07 19:19:35 -06:00
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
rc = migrate_huge_page_move_mapping(mapping, newpage, page);
|
2012-12-11 17:02:31 -07:00
|
|
|
if (rc != MIGRATEPAGE_SUCCESS)
|
2010-09-07 19:19:35 -06:00
|
|
|
return rc;
|
|
|
|
migrate_page_copy(newpage, page);
|
|
|
|
|
2012-12-11 17:02:31 -07:00
|
|
|
return MIGRATEPAGE_SUCCESS;
|
2010-09-07 19:19:35 -06:00
|
|
|
}
|
|
|
|
|
2006-06-23 03:02:58 -06:00
|
|
|
static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-06-23 03:02:58 -06:00
|
|
|
struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
|
2008-07-23 22:27:41 -06:00
|
|
|
struct hstate *h = hstate_inode(dentry->d_inode);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
buf->f_type = HUGETLBFS_MAGIC;
|
2008-07-23 22:27:41 -06:00
|
|
|
buf->f_bsize = huge_page_size(h);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (sbinfo) {
|
|
|
|
spin_lock(&sbinfo->stat_lock);
|
2005-11-21 22:32:24 -07:00
|
|
|
/* If no limits set, just report 0 for max/free/used
|
|
|
|
* blocks, like simple_statfs() */
|
hugepages: fix use after free bug in "quota" handling
hugetlbfs_{get,put}_quota() are badly named. They don't interact with the
general quota handling code, and they don't much resemble its behaviour.
Rather than being about maintaining limits on on-disk block usage by
particular users, they are instead about maintaining limits on in-memory
page usage (including anonymous MAP_PRIVATE copied-on-write pages)
associated with a particular hugetlbfs filesystem instance.
Worse, they work by having callbacks to the hugetlbfs filesystem code from
the low-level page handling code, in particular from free_huge_page().
This is a layering violation of itself, but more importantly, if the
kernel does a get_user_pages() on hugepages (which can happen from KVM
amongst others), then the free_huge_page() can be delayed until after the
associated inode has already been freed. If an unmount occurs at the
wrong time, even the hugetlbfs superblock where the "quota" limits are
stored may have been freed.
Andrew Barry proposed a patch to fix this by having hugepages, instead of
storing a pointer to their address_space and reaching the superblock from
there, had the hugepages store pointers directly to the superblock,
bumping the reference count as appropriate to avoid it being freed.
Andrew Morton rejected that version, however, on the grounds that it made
the existing layering violation worse.
This is a reworked version of Andrew's patch, which removes the extra, and
some of the existing, layering violation. It works by introducing the
concept of a hugepage "subpool" at the lower hugepage mm layer - that is a
finite logical pool of hugepages to allocate from. hugetlbfs now creates
a subpool for each filesystem instance with a page limit set, and a
pointer to the subpool gets added to each allocated hugepage, instead of
the address_space pointer used now. The subpool has its own lifetime and
is only freed once all pages in it _and_ all other references to it (i.e.
superblocks) are gone.
subpools are optional - a NULL subpool pointer is taken by the code to
mean that no subpool limits are in effect.
Previous discussion of this bug found in: "Fix refcounting in hugetlbfs
quota handling.". See: https://lkml.org/lkml/2011/8/11/28 or
http://marc.info/?l=linux-mm&m=126928970510627&w=1
v2: Fixed a bug spotted by Hillf Danton, and removed the extra parameter to
alloc_huge_page() - since it already takes the vma, it is not necessary.
Signed-off-by: Andrew Barry <abarry@cray.com>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Paul Mackerras <paulus@samba.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:12 -06:00
|
|
|
if (sbinfo->spool) {
|
|
|
|
long free_pages;
|
|
|
|
|
|
|
|
spin_lock(&sbinfo->spool->lock);
|
|
|
|
buf->f_blocks = sbinfo->spool->max_hpages;
|
|
|
|
free_pages = sbinfo->spool->max_hpages
|
|
|
|
- sbinfo->spool->used_hpages;
|
|
|
|
buf->f_bavail = buf->f_bfree = free_pages;
|
|
|
|
spin_unlock(&sbinfo->spool->lock);
|
2005-11-21 22:32:24 -07:00
|
|
|
buf->f_files = sbinfo->max_inodes;
|
|
|
|
buf->f_ffree = sbinfo->free_inodes;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
buf->f_namelen = NAME_MAX;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hugetlbfs_put_super(struct super_block *sb)
|
|
|
|
{
|
|
|
|
struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
|
|
|
|
|
|
|
|
if (sbi) {
|
|
|
|
sb->s_fs_info = NULL;
|
hugepages: fix use after free bug in "quota" handling
hugetlbfs_{get,put}_quota() are badly named. They don't interact with the
general quota handling code, and they don't much resemble its behaviour.
Rather than being about maintaining limits on on-disk block usage by
particular users, they are instead about maintaining limits on in-memory
page usage (including anonymous MAP_PRIVATE copied-on-write pages)
associated with a particular hugetlbfs filesystem instance.
Worse, they work by having callbacks to the hugetlbfs filesystem code from
the low-level page handling code, in particular from free_huge_page().
This is a layering violation of itself, but more importantly, if the
kernel does a get_user_pages() on hugepages (which can happen from KVM
amongst others), then the free_huge_page() can be delayed until after the
associated inode has already been freed. If an unmount occurs at the
wrong time, even the hugetlbfs superblock where the "quota" limits are
stored may have been freed.
Andrew Barry proposed a patch to fix this by having hugepages, instead of
storing a pointer to their address_space and reaching the superblock from
there, had the hugepages store pointers directly to the superblock,
bumping the reference count as appropriate to avoid it being freed.
Andrew Morton rejected that version, however, on the grounds that it made
the existing layering violation worse.
This is a reworked version of Andrew's patch, which removes the extra, and
some of the existing, layering violation. It works by introducing the
concept of a hugepage "subpool" at the lower hugepage mm layer - that is a
finite logical pool of hugepages to allocate from. hugetlbfs now creates
a subpool for each filesystem instance with a page limit set, and a
pointer to the subpool gets added to each allocated hugepage, instead of
the address_space pointer used now. The subpool has its own lifetime and
is only freed once all pages in it _and_ all other references to it (i.e.
superblocks) are gone.
subpools are optional - a NULL subpool pointer is taken by the code to
mean that no subpool limits are in effect.
Previous discussion of this bug found in: "Fix refcounting in hugetlbfs
quota handling.". See: https://lkml.org/lkml/2011/8/11/28 or
http://marc.info/?l=linux-mm&m=126928970510627&w=1
v2: Fixed a bug spotted by Hillf Danton, and removed the extra parameter to
alloc_huge_page() - since it already takes the vma, it is not necessary.
Signed-off-by: Andrew Barry <abarry@cray.com>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Paul Mackerras <paulus@samba.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:12 -06:00
|
|
|
|
|
|
|
if (sbi->spool)
|
|
|
|
hugepage_put_subpool(sbi->spool);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
kfree(sbi);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-10-29 19:16:42 -06:00
|
|
|
static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
|
|
|
|
{
|
|
|
|
if (sbinfo->free_inodes >= 0) {
|
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
if (unlikely(!sbinfo->free_inodes)) {
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
sbinfo->free_inodes--;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
|
|
|
|
{
|
|
|
|
if (sbinfo->free_inodes >= 0) {
|
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
sbinfo->free_inodes++;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2006-12-06 21:33:20 -07:00
|
|
|
static struct kmem_cache *hugetlbfs_inode_cachep;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
|
|
|
|
{
|
2005-10-29 19:16:42 -06:00
|
|
|
struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
|
2005-04-16 16:20:36 -06:00
|
|
|
struct hugetlbfs_inode_info *p;
|
|
|
|
|
2005-10-29 19:16:42 -06:00
|
|
|
if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
|
|
|
|
return NULL;
|
2006-12-06 21:33:17 -07:00
|
|
|
p = kmem_cache_alloc(hugetlbfs_inode_cachep, GFP_KERNEL);
|
2005-10-29 19:16:42 -06:00
|
|
|
if (unlikely(!p)) {
|
|
|
|
hugetlbfs_inc_free_inodes(sbinfo);
|
2005-04-16 16:20:36 -06:00
|
|
|
return NULL;
|
2005-10-29 19:16:42 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
return &p->vfs_inode;
|
|
|
|
}
|
|
|
|
|
2011-01-06 23:49:49 -07:00
|
|
|
static void hugetlbfs_i_callback(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
struct inode *inode = container_of(head, struct inode, i_rcu);
|
|
|
|
kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static void hugetlbfs_destroy_inode(struct inode *inode)
|
|
|
|
{
|
2005-10-29 19:16:42 -06:00
|
|
|
hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
|
2005-04-16 16:20:36 -06:00
|
|
|
mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
|
2011-01-06 23:49:49 -07:00
|
|
|
call_rcu(&inode->i_rcu, hugetlbfs_i_callback);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-06-28 05:26:44 -06:00
|
|
|
static const struct address_space_operations hugetlbfs_aops = {
|
2007-10-16 02:25:03 -06:00
|
|
|
.write_begin = hugetlbfs_write_begin,
|
|
|
|
.write_end = hugetlbfs_write_end,
|
2005-04-16 16:20:36 -06:00
|
|
|
.set_page_dirty = hugetlbfs_set_page_dirty,
|
2010-09-07 19:19:35 -06:00
|
|
|
.migratepage = hugetlbfs_migrate_page,
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
2005-10-29 19:16:42 -06:00
|
|
|
|
2008-07-25 20:45:34 -06:00
|
|
|
static void init_once(void *foo)
|
2005-10-29 19:16:42 -06:00
|
|
|
{
|
|
|
|
struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;
|
|
|
|
|
2007-05-16 23:10:57 -06:00
|
|
|
inode_init_once(&ei->vfs_inode);
|
2005-10-29 19:16:42 -06:00
|
|
|
}
|
|
|
|
|
2006-03-28 02:56:42 -07:00
|
|
|
const struct file_operations hugetlbfs_file_operations = {
|
2007-10-16 02:26:22 -06:00
|
|
|
.read = hugetlbfs_read,
|
2005-04-16 16:20:36 -06:00
|
|
|
.mmap = hugetlbfs_file_mmap,
|
2010-05-26 09:53:41 -06:00
|
|
|
.fsync = noop_fsync,
|
2005-04-16 16:20:36 -06:00
|
|
|
.get_unmapped_area = hugetlb_get_unmapped_area,
|
llseek: automatically add .llseek fop
All file_operations should get a .llseek operation so we can make
nonseekable_open the default for future file operations without a
.llseek pointer.
The three cases that we can automatically detect are no_llseek, seq_lseek
and default_llseek. For cases where we can we can automatically prove that
the file offset is always ignored, we use noop_llseek, which maintains
the current behavior of not returning an error from a seek.
New drivers should normally not use noop_llseek but instead use no_llseek
and call nonseekable_open at open time. Existing drivers can be converted
to do the same when the maintainer knows for certain that no user code
relies on calling seek on the device file.
The generated code is often incorrectly indented and right now contains
comments that clarify for each added line why a specific variant was
chosen. In the version that gets submitted upstream, the comments will
be gone and I will manually fix the indentation, because there does not
seem to be a way to do that using coccinelle.
Some amount of new code is currently sitting in linux-next that should get
the same modifications, which I will do at the end of the merge window.
Many thanks to Julia Lawall for helping me learn to write a semantic
patch that does all this.
===== begin semantic patch =====
// This adds an llseek= method to all file operations,
// as a preparation for making no_llseek the default.
//
// The rules are
// - use no_llseek explicitly if we do nonseekable_open
// - use seq_lseek for sequential files
// - use default_llseek if we know we access f_pos
// - use noop_llseek if we know we don't access f_pos,
// but we still want to allow users to call lseek
//
@ open1 exists @
identifier nested_open;
@@
nested_open(...)
{
<+...
nonseekable_open(...)
...+>
}
@ open exists@
identifier open_f;
identifier i, f;
identifier open1.nested_open;
@@
int open_f(struct inode *i, struct file *f)
{
<+...
(
nonseekable_open(...)
|
nested_open(...)
)
...+>
}
@ read disable optional_qualifier exists @
identifier read_f;
identifier f, p, s, off;
type ssize_t, size_t, loff_t;
expression E;
identifier func;
@@
ssize_t read_f(struct file *f, char *p, size_t s, loff_t *off)
{
<+...
(
*off = E
|
*off += E
|
func(..., off, ...)
|
E = *off
)
...+>
}
@ read_no_fpos disable optional_qualifier exists @
identifier read_f;
identifier f, p, s, off;
type ssize_t, size_t, loff_t;
@@
ssize_t read_f(struct file *f, char *p, size_t s, loff_t *off)
{
... when != off
}
@ write @
identifier write_f;
identifier f, p, s, off;
type ssize_t, size_t, loff_t;
expression E;
identifier func;
@@
ssize_t write_f(struct file *f, const char *p, size_t s, loff_t *off)
{
<+...
(
*off = E
|
*off += E
|
func(..., off, ...)
|
E = *off
)
...+>
}
@ write_no_fpos @
identifier write_f;
identifier f, p, s, off;
type ssize_t, size_t, loff_t;
@@
ssize_t write_f(struct file *f, const char *p, size_t s, loff_t *off)
{
... when != off
}
@ fops0 @
identifier fops;
@@
struct file_operations fops = {
...
};
@ has_llseek depends on fops0 @
identifier fops0.fops;
identifier llseek_f;
@@
struct file_operations fops = {
...
.llseek = llseek_f,
...
};
@ has_read depends on fops0 @
identifier fops0.fops;
identifier read_f;
@@
struct file_operations fops = {
...
.read = read_f,
...
};
@ has_write depends on fops0 @
identifier fops0.fops;
identifier write_f;
@@
struct file_operations fops = {
...
.write = write_f,
...
};
@ has_open depends on fops0 @
identifier fops0.fops;
identifier open_f;
@@
struct file_operations fops = {
...
.open = open_f,
...
};
// use no_llseek if we call nonseekable_open
////////////////////////////////////////////
@ nonseekable1 depends on !has_llseek && has_open @
identifier fops0.fops;
identifier nso ~= "nonseekable_open";
@@
struct file_operations fops = {
... .open = nso, ...
+.llseek = no_llseek, /* nonseekable */
};
@ nonseekable2 depends on !has_llseek @
identifier fops0.fops;
identifier open.open_f;
@@
struct file_operations fops = {
... .open = open_f, ...
+.llseek = no_llseek, /* open uses nonseekable */
};
// use seq_lseek for sequential files
/////////////////////////////////////
@ seq depends on !has_llseek @
identifier fops0.fops;
identifier sr ~= "seq_read";
@@
struct file_operations fops = {
... .read = sr, ...
+.llseek = seq_lseek, /* we have seq_read */
};
// use default_llseek if there is a readdir
///////////////////////////////////////////
@ fops1 depends on !has_llseek && !nonseekable1 && !nonseekable2 && !seq @
identifier fops0.fops;
identifier readdir_e;
@@
// any other fop is used that changes pos
struct file_operations fops = {
... .readdir = readdir_e, ...
+.llseek = default_llseek, /* readdir is present */
};
// use default_llseek if at least one of read/write touches f_pos
/////////////////////////////////////////////////////////////////
@ fops2 depends on !fops1 && !has_llseek && !nonseekable1 && !nonseekable2 && !seq @
identifier fops0.fops;
identifier read.read_f;
@@
// read fops use offset
struct file_operations fops = {
... .read = read_f, ...
+.llseek = default_llseek, /* read accesses f_pos */
};
@ fops3 depends on !fops1 && !fops2 && !has_llseek && !nonseekable1 && !nonseekable2 && !seq @
identifier fops0.fops;
identifier write.write_f;
@@
// write fops use offset
struct file_operations fops = {
... .write = write_f, ...
+ .llseek = default_llseek, /* write accesses f_pos */
};
// Use noop_llseek if neither read nor write accesses f_pos
///////////////////////////////////////////////////////////
@ fops4 depends on !fops1 && !fops2 && !fops3 && !has_llseek && !nonseekable1 && !nonseekable2 && !seq @
identifier fops0.fops;
identifier read_no_fpos.read_f;
identifier write_no_fpos.write_f;
@@
// write fops use offset
struct file_operations fops = {
...
.write = write_f,
.read = read_f,
...
+.llseek = noop_llseek, /* read and write both use no f_pos */
};
@ depends on has_write && !has_read && !fops1 && !fops2 && !has_llseek && !nonseekable1 && !nonseekable2 && !seq @
identifier fops0.fops;
identifier write_no_fpos.write_f;
@@
struct file_operations fops = {
... .write = write_f, ...
+.llseek = noop_llseek, /* write uses no f_pos */
};
@ depends on has_read && !has_write && !fops1 && !fops2 && !has_llseek && !nonseekable1 && !nonseekable2 && !seq @
identifier fops0.fops;
identifier read_no_fpos.read_f;
@@
struct file_operations fops = {
... .read = read_f, ...
+.llseek = noop_llseek, /* read uses no f_pos */
};
@ depends on !has_read && !has_write && !fops1 && !fops2 && !has_llseek && !nonseekable1 && !nonseekable2 && !seq @
identifier fops0.fops;
@@
struct file_operations fops = {
...
+.llseek = noop_llseek, /* no read or write fn */
};
===== End semantic patch =====
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Cc: Julia Lawall <julia@diku.dk>
Cc: Christoph Hellwig <hch@infradead.org>
2010-08-15 10:52:59 -06:00
|
|
|
.llseek = default_llseek,
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
2007-02-12 01:55:39 -07:00
|
|
|
static const struct inode_operations hugetlbfs_dir_inode_operations = {
|
2005-04-16 16:20:36 -06:00
|
|
|
.create = hugetlbfs_create,
|
|
|
|
.lookup = simple_lookup,
|
|
|
|
.link = simple_link,
|
|
|
|
.unlink = simple_unlink,
|
|
|
|
.symlink = hugetlbfs_symlink,
|
|
|
|
.mkdir = hugetlbfs_mkdir,
|
|
|
|
.rmdir = simple_rmdir,
|
|
|
|
.mknod = hugetlbfs_mknod,
|
|
|
|
.rename = simple_rename,
|
|
|
|
.setattr = hugetlbfs_setattr,
|
|
|
|
};
|
|
|
|
|
2007-02-12 01:55:39 -07:00
|
|
|
static const struct inode_operations hugetlbfs_inode_operations = {
|
2005-04-16 16:20:36 -06:00
|
|
|
.setattr = hugetlbfs_setattr,
|
|
|
|
};
|
|
|
|
|
2007-02-12 01:55:41 -07:00
|
|
|
static const struct super_operations hugetlbfs_ops = {
|
2005-04-16 16:20:36 -06:00
|
|
|
.alloc_inode = hugetlbfs_alloc_inode,
|
|
|
|
.destroy_inode = hugetlbfs_destroy_inode,
|
2010-06-04 17:52:12 -06:00
|
|
|
.evict_inode = hugetlbfs_evict_inode,
|
2005-04-16 16:20:36 -06:00
|
|
|
.statfs = hugetlbfs_statfs,
|
|
|
|
.put_super = hugetlbfs_put_super,
|
2008-02-08 05:21:45 -07:00
|
|
|
.show_options = generic_show_options,
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
static int
|
|
|
|
hugetlbfs_parse_options(char *options, struct hugetlbfs_config *pconfig)
|
|
|
|
{
|
2007-07-16 00:40:52 -06:00
|
|
|
char *p, *rest;
|
|
|
|
substring_t args[MAX_OPT_ARGS];
|
|
|
|
int option;
|
2008-07-23 22:27:43 -06:00
|
|
|
unsigned long long size = 0;
|
|
|
|
enum { NO_SIZE, SIZE_STD, SIZE_PERCENT } setsize = NO_SIZE;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (!options)
|
|
|
|
return 0;
|
|
|
|
|
2007-07-16 00:40:52 -06:00
|
|
|
while ((p = strsep(&options, ",")) != NULL) {
|
|
|
|
int token;
|
2007-07-16 00:40:54 -06:00
|
|
|
if (!*p)
|
|
|
|
continue;
|
2007-07-16 00:40:52 -06:00
|
|
|
|
|
|
|
token = match_token(p, tokens, args);
|
|
|
|
switch (token) {
|
|
|
|
case Opt_uid:
|
|
|
|
if (match_int(&args[0], &option))
|
|
|
|
goto bad_val;
|
2012-02-07 17:19:25 -07:00
|
|
|
pconfig->uid = make_kuid(current_user_ns(), option);
|
|
|
|
if (!uid_valid(pconfig->uid))
|
|
|
|
goto bad_val;
|
2007-07-16 00:40:52 -06:00
|
|
|
break;
|
|
|
|
|
|
|
|
case Opt_gid:
|
|
|
|
if (match_int(&args[0], &option))
|
|
|
|
goto bad_val;
|
2012-02-07 17:19:25 -07:00
|
|
|
pconfig->gid = make_kgid(current_user_ns(), option);
|
|
|
|
if (!gid_valid(pconfig->gid))
|
|
|
|
goto bad_val;
|
2007-07-16 00:40:52 -06:00
|
|
|
break;
|
|
|
|
|
|
|
|
case Opt_mode:
|
|
|
|
if (match_octal(&args[0], &option))
|
|
|
|
goto bad_val;
|
2008-02-04 23:28:36 -07:00
|
|
|
pconfig->mode = option & 01777U;
|
2007-07-16 00:40:52 -06:00
|
|
|
break;
|
|
|
|
|
|
|
|
case Opt_size: {
|
|
|
|
/* memparse() will accept a K/M/G without a digit */
|
|
|
|
if (!isdigit(*args[0].from))
|
|
|
|
goto bad_val;
|
|
|
|
size = memparse(args[0].from, &rest);
|
2008-07-23 22:27:43 -06:00
|
|
|
setsize = SIZE_STD;
|
|
|
|
if (*rest == '%')
|
|
|
|
setsize = SIZE_PERCENT;
|
2007-07-16 00:40:52 -06:00
|
|
|
break;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-07-16 00:40:52 -06:00
|
|
|
case Opt_nr_inodes:
|
|
|
|
/* memparse() will accept a K/M/G without a digit */
|
|
|
|
if (!isdigit(*args[0].from))
|
|
|
|
goto bad_val;
|
|
|
|
pconfig->nr_inodes = memparse(args[0].from, &rest);
|
|
|
|
break;
|
|
|
|
|
2008-07-23 22:27:43 -06:00
|
|
|
case Opt_pagesize: {
|
|
|
|
unsigned long ps;
|
|
|
|
ps = memparse(args[0].from, &rest);
|
|
|
|
pconfig->hstate = size_to_hstate(ps);
|
|
|
|
if (!pconfig->hstate) {
|
2014-06-04 17:07:21 -06:00
|
|
|
pr_err("Unsupported page size %lu MB\n",
|
2008-07-23 22:27:43 -06:00
|
|
|
ps >> 20);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2007-07-16 00:40:52 -06:00
|
|
|
default:
|
2014-06-04 17:07:21 -06:00
|
|
|
pr_err("Bad mount option: \"%s\"\n", p);
|
2007-07-16 00:40:54 -06:00
|
|
|
return -EINVAL;
|
2007-07-16 00:40:52 -06:00
|
|
|
break;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2008-07-23 22:27:43 -06:00
|
|
|
|
|
|
|
/* Do size after hstate is set up */
|
|
|
|
if (setsize > NO_SIZE) {
|
|
|
|
struct hstate *h = pconfig->hstate;
|
|
|
|
if (setsize == SIZE_PERCENT) {
|
|
|
|
size <<= huge_page_shift(h);
|
|
|
|
size *= h->max_huge_pages;
|
|
|
|
do_div(size, 100);
|
|
|
|
}
|
|
|
|
pconfig->nr_blocks = (size >> huge_page_shift(h));
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
2007-07-16 00:40:52 -06:00
|
|
|
|
|
|
|
bad_val:
|
2014-06-04 17:07:21 -06:00
|
|
|
pr_err("Bad value '%s' for mount option '%s'\n", args[0].from, p);
|
2009-04-21 13:24:05 -06:00
|
|
|
return -EINVAL;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
hugetlbfs_fill_super(struct super_block *sb, void *data, int silent)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
struct hugetlbfs_config config;
|
|
|
|
struct hugetlbfs_sb_info *sbinfo;
|
|
|
|
|
2008-02-08 05:21:45 -07:00
|
|
|
save_mount_options(sb, data);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
config.nr_blocks = -1; /* No limit on size by default */
|
|
|
|
config.nr_inodes = -1; /* No limit on number of inodes by default */
|
2008-11-13 16:38:56 -07:00
|
|
|
config.uid = current_fsuid();
|
|
|
|
config.gid = current_fsgid();
|
2005-04-16 16:20:36 -06:00
|
|
|
config.mode = 0755;
|
2008-07-23 22:27:43 -06:00
|
|
|
config.hstate = &default_hstate;
|
2005-04-16 16:20:36 -06:00
|
|
|
ret = hugetlbfs_parse_options(data, &config);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
|
|
|
|
if (!sbinfo)
|
|
|
|
return -ENOMEM;
|
|
|
|
sb->s_fs_info = sbinfo;
|
2008-07-23 22:27:43 -06:00
|
|
|
sbinfo->hstate = config.hstate;
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_lock_init(&sbinfo->stat_lock);
|
|
|
|
sbinfo->max_inodes = config.nr_inodes;
|
|
|
|
sbinfo->free_inodes = config.nr_inodes;
|
hugepages: fix use after free bug in "quota" handling
hugetlbfs_{get,put}_quota() are badly named. They don't interact with the
general quota handling code, and they don't much resemble its behaviour.
Rather than being about maintaining limits on on-disk block usage by
particular users, they are instead about maintaining limits on in-memory
page usage (including anonymous MAP_PRIVATE copied-on-write pages)
associated with a particular hugetlbfs filesystem instance.
Worse, they work by having callbacks to the hugetlbfs filesystem code from
the low-level page handling code, in particular from free_huge_page().
This is a layering violation of itself, but more importantly, if the
kernel does a get_user_pages() on hugepages (which can happen from KVM
amongst others), then the free_huge_page() can be delayed until after the
associated inode has already been freed. If an unmount occurs at the
wrong time, even the hugetlbfs superblock where the "quota" limits are
stored may have been freed.
Andrew Barry proposed a patch to fix this by having hugepages, instead of
storing a pointer to their address_space and reaching the superblock from
there, had the hugepages store pointers directly to the superblock,
bumping the reference count as appropriate to avoid it being freed.
Andrew Morton rejected that version, however, on the grounds that it made
the existing layering violation worse.
This is a reworked version of Andrew's patch, which removes the extra, and
some of the existing, layering violation. It works by introducing the
concept of a hugepage "subpool" at the lower hugepage mm layer - that is a
finite logical pool of hugepages to allocate from. hugetlbfs now creates
a subpool for each filesystem instance with a page limit set, and a
pointer to the subpool gets added to each allocated hugepage, instead of
the address_space pointer used now. The subpool has its own lifetime and
is only freed once all pages in it _and_ all other references to it (i.e.
superblocks) are gone.
subpools are optional - a NULL subpool pointer is taken by the code to
mean that no subpool limits are in effect.
Previous discussion of this bug found in: "Fix refcounting in hugetlbfs
quota handling.". See: https://lkml.org/lkml/2011/8/11/28 or
http://marc.info/?l=linux-mm&m=126928970510627&w=1
v2: Fixed a bug spotted by Hillf Danton, and removed the extra parameter to
alloc_huge_page() - since it already takes the vma, it is not necessary.
Signed-off-by: Andrew Barry <abarry@cray.com>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Paul Mackerras <paulus@samba.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-21 17:34:12 -06:00
|
|
|
sbinfo->spool = NULL;
|
|
|
|
if (config.nr_blocks != -1) {
|
|
|
|
sbinfo->spool = hugepage_new_subpool(config.nr_blocks);
|
|
|
|
if (!sbinfo->spool)
|
|
|
|
goto out_free;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
sb->s_maxbytes = MAX_LFS_FILESIZE;
|
2008-07-23 22:27:43 -06:00
|
|
|
sb->s_blocksize = huge_page_size(config.hstate);
|
|
|
|
sb->s_blocksize_bits = huge_page_shift(config.hstate);
|
2005-04-16 16:20:36 -06:00
|
|
|
sb->s_magic = HUGETLBFS_MAGIC;
|
|
|
|
sb->s_op = &hugetlbfs_ops;
|
|
|
|
sb->s_time_gran = 1;
|
2012-01-08 20:15:13 -07:00
|
|
|
sb->s_root = d_make_root(hugetlbfs_get_root(sb, &config));
|
|
|
|
if (!sb->s_root)
|
2005-04-16 16:20:36 -06:00
|
|
|
goto out_free;
|
|
|
|
return 0;
|
|
|
|
out_free:
|
2014-06-04 17:10:40 -06:00
|
|
|
kfree(sbinfo->spool);
|
2005-04-16 16:20:36 -06:00
|
|
|
kfree(sbinfo);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2010-07-25 01:46:36 -06:00
|
|
|
static struct dentry *hugetlbfs_mount(struct file_system_type *fs_type,
|
|
|
|
int flags, const char *dev_name, void *data)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2010-07-25 01:46:36 -06:00
|
|
|
return mount_nodev(fs_type, flags, data, hugetlbfs_fill_super);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct file_system_type hugetlbfs_fs_type = {
|
|
|
|
.name = "hugetlbfs",
|
2010-07-25 01:46:36 -06:00
|
|
|
.mount = hugetlbfs_mount,
|
2005-04-16 16:20:36 -06:00
|
|
|
.kill_sb = kill_litter_super,
|
|
|
|
};
|
2013-03-02 20:39:14 -07:00
|
|
|
MODULE_ALIAS_FS("hugetlbfs");
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-12-11 17:01:34 -07:00
|
|
|
static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-09-23 16:56:05 -06:00
|
|
|
static int can_do_hugetlb_shm(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2012-02-07 17:19:25 -07:00
|
|
|
kgid_t shm_group;
|
|
|
|
shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
|
|
|
|
return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2012-12-11 17:01:34 -07:00
|
|
|
static int get_hstate_idx(int page_size_log)
|
|
|
|
{
|
2013-05-07 17:18:13 -06:00
|
|
|
struct hstate *h = hstate_sizelog(page_size_log);
|
2012-12-11 17:01:34 -07:00
|
|
|
|
|
|
|
if (!h)
|
|
|
|
return -1;
|
|
|
|
return h - hstates;
|
|
|
|
}
|
|
|
|
|
2014-06-04 17:10:39 -06:00
|
|
|
static const struct dentry_operations anon_ops = {
|
2013-08-24 10:08:17 -06:00
|
|
|
.d_dname = simple_dname
|
2013-02-14 20:39:53 -07:00
|
|
|
};
|
|
|
|
|
2013-05-07 17:18:13 -06:00
|
|
|
/*
|
|
|
|
* Note that size should be aligned to proper hugepage size in caller side,
|
|
|
|
* otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
|
|
|
|
*/
|
|
|
|
struct file *hugetlb_file_setup(const char *name, size_t size,
|
|
|
|
vm_flags_t acctflag, struct user_struct **user,
|
2012-12-11 17:01:34 -07:00
|
|
|
int creat_flags, int page_size_log)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2012-09-12 21:11:55 -06:00
|
|
|
struct file *file = ERR_PTR(-ENOMEM);
|
2005-04-16 16:20:36 -06:00
|
|
|
struct inode *inode;
|
2009-08-08 14:52:35 -06:00
|
|
|
struct path path;
|
2013-02-14 20:39:53 -07:00
|
|
|
struct super_block *sb;
|
2005-04-16 16:20:36 -06:00
|
|
|
struct qstr quick_string;
|
2012-12-11 17:01:34 -07:00
|
|
|
int hstate_idx;
|
|
|
|
|
|
|
|
hstate_idx = get_hstate_idx(page_size_log);
|
|
|
|
if (hstate_idx < 0)
|
|
|
|
return ERR_PTR(-ENODEV);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-08-24 09:30:28 -06:00
|
|
|
*user = NULL;
|
2012-12-11 17:01:34 -07:00
|
|
|
if (!hugetlbfs_vfsmount[hstate_idx])
|
2007-05-06 15:50:18 -06:00
|
|
|
return ERR_PTR(-ENOENT);
|
|
|
|
|
2009-09-23 16:56:05 -06:00
|
|
|
if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
|
2009-08-24 09:30:28 -06:00
|
|
|
*user = current_user();
|
|
|
|
if (user_shm_lock(size, *user)) {
|
2012-03-21 17:34:13 -06:00
|
|
|
task_lock(current);
|
2014-06-04 17:07:21 -06:00
|
|
|
pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is deprecated\n",
|
2012-03-21 17:34:13 -06:00
|
|
|
current->comm, current->pid);
|
|
|
|
task_unlock(current);
|
2009-08-24 09:30:28 -06:00
|
|
|
} else {
|
|
|
|
*user = NULL;
|
2009-03-31 16:21:26 -06:00
|
|
|
return ERR_PTR(-EPERM);
|
2009-08-24 09:30:28 -06:00
|
|
|
}
|
2009-03-31 16:21:26 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2013-02-14 20:39:53 -07:00
|
|
|
sb = hugetlbfs_vfsmount[hstate_idx]->mnt_sb;
|
2007-06-16 11:16:16 -06:00
|
|
|
quick_string.name = name;
|
2005-04-16 16:20:36 -06:00
|
|
|
quick_string.len = strlen(quick_string.name);
|
|
|
|
quick_string.hash = 0;
|
2013-02-14 20:39:53 -07:00
|
|
|
path.dentry = d_alloc_pseudo(sb, &quick_string);
|
2009-08-08 14:52:35 -06:00
|
|
|
if (!path.dentry)
|
2005-04-16 16:20:36 -06:00
|
|
|
goto out_shm_unlock;
|
|
|
|
|
2013-02-14 20:39:53 -07:00
|
|
|
d_set_d_op(path.dentry, &anon_ops);
|
2012-12-11 17:01:34 -07:00
|
|
|
path.mnt = mntget(hugetlbfs_vfsmount[hstate_idx]);
|
2012-09-12 21:11:55 -06:00
|
|
|
file = ERR_PTR(-ENOSPC);
|
2013-02-14 20:39:53 -07:00
|
|
|
inode = hugetlbfs_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!inode)
|
2007-10-17 00:31:13 -06:00
|
|
|
goto out_dentry;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-09-12 21:11:55 -06:00
|
|
|
file = ERR_PTR(-ENOMEM);
|
2013-05-07 17:18:13 -06:00
|
|
|
if (hugetlb_reserve_pages(inode, 0,
|
|
|
|
size >> huge_page_shift(hstate_inode(inode)), NULL,
|
|
|
|
acctflag))
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
goto out_inode;
|
|
|
|
|
2009-08-08 14:52:35 -06:00
|
|
|
d_instantiate(path.dentry, inode);
|
2005-04-16 16:20:36 -06:00
|
|
|
inode->i_size = size;
|
2011-10-28 06:13:28 -06:00
|
|
|
clear_nlink(inode);
|
2007-10-17 00:31:13 -06:00
|
|
|
|
2009-08-08 14:52:35 -06:00
|
|
|
file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
|
2007-10-17 00:31:13 -06:00
|
|
|
&hugetlbfs_file_operations);
|
2012-09-12 21:11:55 -06:00
|
|
|
if (IS_ERR(file))
|
2008-02-23 03:59:19 -07:00
|
|
|
goto out_dentry; /* inode is already attached */
|
2007-10-17 00:31:13 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
return file;
|
|
|
|
|
[PATCH] hugepage: Strict page reservation for hugepage inodes
These days, hugepages are demand-allocated at first fault time. There's a
somewhat dubious (and racy) heuristic when making a new mmap() to check if
there are enough available hugepages to fully satisfy that mapping.
A particularly obvious case where the heuristic breaks down is where a
process maps its hugepages not as a single chunk, but as a bunch of
individually mmap()ed (or shmat()ed) blocks without touching and
instantiating the pages in between allocations. In this case the size of
each block is compared against the total number of available hugepages.
It's thus easy for the process to become overcommitted, because each block
mapping will succeed, although the total number of hugepages required by
all blocks exceeds the number available. In particular, this defeats such
a program which will detect a mapping failure and adjust its hugepage usage
downward accordingly.
The patch below addresses this problem, by strictly reserving a number of
physical hugepages for hugepage inodes which have been mapped, but not
instatiated. MAP_SHARED mappings are thus "safe" - they will fail on
mmap(), not later with an OOM SIGKILL. MAP_PRIVATE mappings can still
trigger an OOM. (Actually SHARED mappings can technically still OOM, but
only if the sysadmin explicitly reduces the hugepage pool between mapping
and instantiation)
This patch appears to address the problem at hand - it allows DB2 to start
correctly, for instance, which previously suffered the failure described
above.
This patch causes no regressions on the libhugetblfs testsuite, and makes a
test (designed to catch this problem) pass which previously failed (ppc64,
POWER5).
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Cc: William Lee Irwin III <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-22 01:08:55 -07:00
|
|
|
out_inode:
|
|
|
|
iput(inode);
|
2005-04-16 16:20:36 -06:00
|
|
|
out_dentry:
|
2009-08-08 14:52:35 -06:00
|
|
|
path_put(&path);
|
2005-04-16 16:20:36 -06:00
|
|
|
out_shm_unlock:
|
2009-08-24 09:30:28 -06:00
|
|
|
if (*user) {
|
|
|
|
user_shm_unlock(size, *user);
|
|
|
|
*user = NULL;
|
|
|
|
}
|
2012-09-12 21:11:55 -06:00
|
|
|
return file;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int __init init_hugetlbfs_fs(void)
|
|
|
|
{
|
2012-12-11 17:01:34 -07:00
|
|
|
struct hstate *h;
|
2005-04-16 16:20:36 -06:00
|
|
|
int error;
|
2012-12-11 17:01:34 -07:00
|
|
|
int i;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
hugetlb: ensure hugepage access is denied if hugepages are not supported
Currently, I am seeing the following when I `mount -t hugetlbfs /none
/dev/hugetlbfs`, and then simply do a `ls /dev/hugetlbfs`. I think it's
related to the fact that hugetlbfs is properly not correctly setting
itself up in this state?:
Unable to handle kernel paging request for data at address 0x00000031
Faulting instruction address: 0xc000000000245710
Oops: Kernel access of bad area, sig: 11 [#1]
SMP NR_CPUS=2048 NUMA pSeries
....
In KVM guests on Power, in a guest not backed by hugepages, we see the
following:
AnonHugePages: 0 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 64 kB
HPAGE_SHIFT == 0 in this configuration, which indicates that hugepages
are not supported at boot-time, but this is only checked in
hugetlb_init(). Extract the check to a helper function, and use it in a
few relevant places.
This does make hugetlbfs not supported (not registered at all) in this
environment. I believe this is fine, as there are no valid hugepages
and that won't change at runtime.
[akpm@linux-foundation.org: use pr_info(), per Mel]
[akpm@linux-foundation.org: fix build when HPAGE_SHIFT is undefined]
Signed-off-by: Nishanth Aravamudan <nacc@linux.vnet.ibm.com>
Reviewed-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Randy Dunlap <rdunlap@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-05-06 13:50:00 -06:00
|
|
|
if (!hugepages_supported()) {
|
2014-06-04 17:07:21 -06:00
|
|
|
pr_info("disabling because there are no supported hugepage sizes\n");
|
hugetlb: ensure hugepage access is denied if hugepages are not supported
Currently, I am seeing the following when I `mount -t hugetlbfs /none
/dev/hugetlbfs`, and then simply do a `ls /dev/hugetlbfs`. I think it's
related to the fact that hugetlbfs is properly not correctly setting
itself up in this state?:
Unable to handle kernel paging request for data at address 0x00000031
Faulting instruction address: 0xc000000000245710
Oops: Kernel access of bad area, sig: 11 [#1]
SMP NR_CPUS=2048 NUMA pSeries
....
In KVM guests on Power, in a guest not backed by hugepages, we see the
following:
AnonHugePages: 0 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 64 kB
HPAGE_SHIFT == 0 in this configuration, which indicates that hugepages
are not supported at boot-time, but this is only checked in
hugetlb_init(). Extract the check to a helper function, and use it in a
few relevant places.
This does make hugetlbfs not supported (not registered at all) in this
environment. I believe this is fine, as there are no valid hugepages
and that won't change at runtime.
[akpm@linux-foundation.org: use pr_info(), per Mel]
[akpm@linux-foundation.org: fix build when HPAGE_SHIFT is undefined]
Signed-off-by: Nishanth Aravamudan <nacc@linux.vnet.ibm.com>
Reviewed-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Randy Dunlap <rdunlap@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-05-06 13:50:00 -06:00
|
|
|
return -ENOTSUPP;
|
|
|
|
}
|
|
|
|
|
2007-10-17 00:25:46 -06:00
|
|
|
error = bdi_init(&hugetlbfs_backing_dev_info);
|
|
|
|
if (error)
|
|
|
|
return error;
|
|
|
|
|
2012-03-21 17:34:15 -06:00
|
|
|
error = -ENOMEM;
|
2005-04-16 16:20:36 -06:00
|
|
|
hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
|
|
|
|
sizeof(struct hugetlbfs_inode_info),
|
2007-07-19 19:11:58 -06:00
|
|
|
0, 0, init_once);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (hugetlbfs_inode_cachep == NULL)
|
2007-10-17 00:25:46 -06:00
|
|
|
goto out2;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
error = register_filesystem(&hugetlbfs_fs_type);
|
|
|
|
if (error)
|
|
|
|
goto out;
|
|
|
|
|
2012-12-11 17:01:34 -07:00
|
|
|
i = 0;
|
|
|
|
for_each_hstate(h) {
|
|
|
|
char buf[50];
|
|
|
|
unsigned ps_kb = 1U << (h->order + PAGE_SHIFT - 10);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-12-11 17:01:34 -07:00
|
|
|
snprintf(buf, sizeof(buf), "pagesize=%uK", ps_kb);
|
|
|
|
hugetlbfs_vfsmount[i] = kern_mount_data(&hugetlbfs_fs_type,
|
|
|
|
buf);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2012-12-11 17:01:34 -07:00
|
|
|
if (IS_ERR(hugetlbfs_vfsmount[i])) {
|
2014-06-04 17:07:21 -06:00
|
|
|
pr_err("Cannot mount internal hugetlbfs for "
|
2012-12-11 17:01:34 -07:00
|
|
|
"page size %uK", ps_kb);
|
|
|
|
error = PTR_ERR(hugetlbfs_vfsmount[i]);
|
|
|
|
hugetlbfs_vfsmount[i] = NULL;
|
|
|
|
}
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
/* Non default hstates are optional */
|
|
|
|
if (!IS_ERR_OR_NULL(hugetlbfs_vfsmount[default_hstate_idx]))
|
|
|
|
return 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
out:
|
2012-03-21 17:34:15 -06:00
|
|
|
kmem_cache_destroy(hugetlbfs_inode_cachep);
|
2007-10-17 00:25:46 -06:00
|
|
|
out2:
|
|
|
|
bdi_destroy(&hugetlbfs_backing_dev_info);
|
2005-04-16 16:20:36 -06:00
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __exit exit_hugetlbfs_fs(void)
|
|
|
|
{
|
2012-12-11 17:01:34 -07:00
|
|
|
struct hstate *h;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
2012-09-25 19:33:07 -06:00
|
|
|
/*
|
|
|
|
* Make sure all delayed rcu free inodes are flushed before we
|
|
|
|
* destroy cache.
|
|
|
|
*/
|
|
|
|
rcu_barrier();
|
2005-04-16 16:20:36 -06:00
|
|
|
kmem_cache_destroy(hugetlbfs_inode_cachep);
|
2012-12-11 17:01:34 -07:00
|
|
|
i = 0;
|
|
|
|
for_each_hstate(h)
|
|
|
|
kern_unmount(hugetlbfs_vfsmount[i++]);
|
2005-04-16 16:20:36 -06:00
|
|
|
unregister_filesystem(&hugetlbfs_fs_type);
|
2007-10-17 00:25:46 -06:00
|
|
|
bdi_destroy(&hugetlbfs_backing_dev_info);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
module_init(init_hugetlbfs_fs)
|
|
|
|
module_exit(exit_hugetlbfs_fs)
|
|
|
|
|
|
|
|
MODULE_LICENSE("GPL");
|