2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* linux/fs/buffer.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 1991, 1992, 2002 Linus Torvalds
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
|
|
|
|
*
|
|
|
|
* Removed a lot of unnecessary code and simplified things now that
|
|
|
|
* the buffer cache isn't our primary cache - Andrew Tridgell 12/96
|
|
|
|
*
|
|
|
|
* Speed up hash, lru, and free list operations. Use gfp() for allocating
|
|
|
|
* hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
|
|
|
|
*
|
|
|
|
* Added 32k buffer block sizes - these are required older ARM systems. - RMK
|
|
|
|
*
|
|
|
|
* async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/syscalls.h>
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/slab.h>
|
2006-01-11 13:17:46 -07:00
|
|
|
#include <linux/capability.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/blkdev.h>
|
|
|
|
#include <linux/file.h>
|
|
|
|
#include <linux/quotaops.h>
|
|
|
|
#include <linux/highmem.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/writeback.h>
|
|
|
|
#include <linux/hash.h>
|
|
|
|
#include <linux/suspend.h>
|
|
|
|
#include <linux/buffer_head.h>
|
2006-12-10 03:19:27 -07:00
|
|
|
#include <linux/task_io_accounting_ops.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/bio.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/bitops.h>
|
|
|
|
#include <linux/mpage.h>
|
[PATCH] spinlock consolidation
This patch (written by me and also containing many suggestions of Arjan van
de Ven) does a major cleanup of the spinlock code. It does the following
things:
- consolidates and enhances the spinlock/rwlock debugging code
- simplifies the asm/spinlock.h files
- encapsulates the raw spinlock type and moves generic spinlock
features (such as ->break_lock) into the generic code.
- cleans up the spinlock code hierarchy to get rid of the spaghetti.
Most notably there's now only a single variant of the debugging code,
located in lib/spinlock_debug.c. (previously we had one SMP debugging
variant per architecture, plus a separate generic one for UP builds)
Also, i've enhanced the rwlock debugging facility, it will now track
write-owners. There is new spinlock-owner/CPU-tracking on SMP builds too.
All locks have lockup detection now, which will work for both soft and hard
spin/rwlock lockups.
The arch-level include files now only contain the minimally necessary
subset of the spinlock code - all the rest that can be generalized now
lives in the generic headers:
include/asm-i386/spinlock_types.h | 16
include/asm-x86_64/spinlock_types.h | 16
I have also split up the various spinlock variants into separate files,
making it easier to see which does what. The new layout is:
SMP | UP
----------------------------|-----------------------------------
asm/spinlock_types_smp.h | linux/spinlock_types_up.h
linux/spinlock_types.h | linux/spinlock_types.h
asm/spinlock_smp.h | linux/spinlock_up.h
linux/spinlock_api_smp.h | linux/spinlock_api_up.h
linux/spinlock.h | linux/spinlock.h
/*
* here's the role of the various spinlock/rwlock related include files:
*
* on SMP builds:
*
* asm/spinlock_types.h: contains the raw_spinlock_t/raw_rwlock_t and the
* initializers
*
* linux/spinlock_types.h:
* defines the generic type and initializers
*
* asm/spinlock.h: contains the __raw_spin_*()/etc. lowlevel
* implementations, mostly inline assembly code
*
* (also included on UP-debug builds:)
*
* linux/spinlock_api_smp.h:
* contains the prototypes for the _spin_*() APIs.
*
* linux/spinlock.h: builds the final spin_*() APIs.
*
* on UP builds:
*
* linux/spinlock_type_up.h:
* contains the generic, simplified UP spinlock type.
* (which is an empty structure on non-debug builds)
*
* linux/spinlock_types.h:
* defines the generic type and initializers
*
* linux/spinlock_up.h:
* contains the __raw_spin_*()/etc. version of UP
* builds. (which are NOPs on non-debug, non-preempt
* builds)
*
* (included on UP-non-debug builds:)
*
* linux/spinlock_api_up.h:
* builds the _spin_*() APIs.
*
* linux/spinlock.h: builds the final spin_*() APIs.
*/
All SMP and UP architectures are converted by this patch.
arm, i386, ia64, ppc, ppc64, s390/s390x, x64 was build-tested via
crosscompilers. m32r, mips, sh, sparc, have not been tested yet, but should
be mostly fine.
From: Grant Grundler <grundler@parisc-linux.org>
Booted and lightly tested on a500-44 (64-bit, SMP kernel, dual CPU).
Builds 32-bit SMP kernel (not booted or tested). I did not try to build
non-SMP kernels. That should be trivial to fix up later if necessary.
I converted bit ops atomic_hash lock to raw_spinlock_t. Doing so avoids
some ugly nesting of linux/*.h and asm/*.h files. Those particular locks
are well tested and contained entirely inside arch specific code. I do NOT
expect any new issues to arise with them.
If someone does ever need to use debug/metrics with them, then they will
need to unravel this hairball between spinlocks, atomic ops, and bit ops
that exist only because parisc has exactly one atomic instruction: LDCW
(load and clear word).
From: "Luck, Tony" <tony.luck@intel.com>
ia64 fix
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Arjan van de Ven <arjanv@infradead.org>
Signed-off-by: Grant Grundler <grundler@parisc-linux.org>
Cc: Matthew Wilcox <willy@debian.org>
Signed-off-by: Hirokazu Takata <takata@linux-m32r.org>
Signed-off-by: Mikael Pettersson <mikpe@csd.uu.se>
Signed-off-by: Benoit Boissinot <benoit.boissinot@ens-lyon.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-10 01:25:56 -06:00
|
|
|
#include <linux/bit_spinlock.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
|
|
|
|
|
|
|
|
#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
|
|
|
|
|
|
|
|
inline void
|
|
|
|
init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
|
|
|
|
{
|
|
|
|
bh->b_end_io = handler;
|
|
|
|
bh->b_private = private;
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(init_buffer);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
static int sync_buffer(void *word)
|
|
|
|
{
|
|
|
|
struct block_device *bd;
|
|
|
|
struct buffer_head *bh
|
|
|
|
= container_of(word, struct buffer_head, b_state);
|
|
|
|
|
|
|
|
smp_mb();
|
|
|
|
bd = bh->b_bdev;
|
|
|
|
if (bd)
|
|
|
|
blk_run_address_space(bd->bd_inode->i_mapping);
|
|
|
|
io_schedule();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-02-08 05:19:52 -07:00
|
|
|
void __lock_buffer(struct buffer_head *bh)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
|
|
|
|
TASK_UNINTERRUPTIBLE);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__lock_buffer);
|
|
|
|
|
2008-02-08 05:19:52 -07:00
|
|
|
void unlock_buffer(struct buffer_head *bh)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-10-18 21:27:00 -06:00
|
|
|
clear_bit_unlock(BH_Lock, &bh->b_state);
|
2005-04-16 16:20:36 -06:00
|
|
|
smp_mb__after_clear_bit();
|
|
|
|
wake_up_bit(&bh->b_state, BH_Lock);
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(unlock_buffer);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Block until a buffer comes unlocked. This doesn't stop it
|
|
|
|
* from becoming locked again - you have to lock it yourself
|
|
|
|
* if you want to preserve its state.
|
|
|
|
*/
|
|
|
|
void __wait_on_buffer(struct buffer_head * bh)
|
|
|
|
{
|
|
|
|
wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(__wait_on_buffer);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
static void
|
|
|
|
__clear_page_buffers(struct page *page)
|
|
|
|
{
|
|
|
|
ClearPagePrivate(page);
|
[PATCH] mm: split page table lock
Christoph Lameter demonstrated very poor scalability on the SGI 512-way, with
a many-threaded application which concurrently initializes different parts of
a large anonymous area.
This patch corrects that, by using a separate spinlock per page table page, to
guard the page table entries in that page, instead of using the mm's single
page_table_lock. (But even then, page_table_lock is still used to guard page
table allocation, and anon_vma allocation.)
In this implementation, the spinlock is tucked inside the struct page of the
page table page: with a BUILD_BUG_ON in case it overflows - which it would in
the case of 32-bit PA-RISC with spinlock debugging enabled.
Splitting the lock is not quite for free: another cacheline access. Ideally,
I suppose we would use split ptlock only for multi-threaded processes on
multi-cpu machines; but deciding that dynamically would have its own costs.
So for now enable it by config, at some number of cpus - since the Kconfig
language doesn't support inequalities, let preprocessor compare that with
NR_CPUS. But I don't think it's worth being user-configurable: for good
testing of both split and unsplit configs, split now at 4 cpus, and perhaps
change that to 8 later.
There is a benefit even for singly threaded processes: kswapd can be attacking
one part of the mm while another part is busy faulting.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-29 19:16:40 -06:00
|
|
|
set_page_private(page, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
page_cache_release(page);
|
|
|
|
}
|
|
|
|
|
block: Supress Buffer I/O errors when SCSI REQ_QUIET flag set
Allow the scsi request REQ_QUIET flag to be propagated to the buffer
file system layer. The basic ideas is to pass the flag from the scsi
request to the bio (block IO) and then to the buffer layer. The buffer
layer can then suppress needless printks.
This patch declutters the kernel log by removed the 40-50 (per lun)
buffer io error messages seen during a boot in my multipath setup . It
is a good chance any real errors will be missed in the "noise" it the
logs without this patch.
During boot I see blocks of messages like
"
__ratelimit: 211 callbacks suppressed
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242847
Buffer I/O error on device sdm, logical block 1
Buffer I/O error on device sdm, logical block 5242878
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242872
"
in my logs.
My disk environment is multipath fiber channel using the SCSI_DH_RDAC
code and multipathd. This topology includes an "active" and "ghost"
path for each lun. IO's to the "ghost" path will never complete and the
SCSI layer, via the scsi device handler rdac code, quick returns the IOs
to theses paths and sets the REQ_QUIET scsi flag to suppress the scsi
layer messages.
I am wanting to extend the QUIET behavior to include the buffer file
system layer to deal with these errors as well. I have been running this
patch for a while now on several boxes without issue. A few runs of
bonnie++ show no noticeable difference in performance in my setup.
Thanks for John Stultz for the quiet_error finalization.
Submitted-by: Keith Mannthey <kmannth@us.ibm.com>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
2008-11-25 02:24:35 -07:00
|
|
|
|
|
|
|
static int quiet_error(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
if (!test_bit(BH_Quiet, &bh->b_state) && printk_ratelimit())
|
|
|
|
return 0;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
static void buffer_io_error(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
char b[BDEVNAME_SIZE];
|
|
|
|
printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
|
|
|
|
bdevname(bh->b_bdev, b),
|
|
|
|
(unsigned long long)bh->b_blocknr);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2007-10-16 02:24:47 -06:00
|
|
|
* End-of-IO handler helper function which does not touch the bh after
|
|
|
|
* unlocking it.
|
|
|
|
* Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
|
|
|
|
* a race there is benign: unlock_buffer() only use the bh's address for
|
|
|
|
* hashing after unlocking the buffer, so it doesn't actually touch the bh
|
|
|
|
* itself.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2007-10-16 02:24:47 -06:00
|
|
|
static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
|
|
|
/* This happens, due to failed READA attempts. */
|
|
|
|
clear_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
unlock_buffer(bh);
|
2007-10-16 02:24:47 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Default synchronous end-of-IO handler.. Just mark it up-to-date and
|
|
|
|
* unlock the buffer. This is what ll_rw_block uses too.
|
|
|
|
*/
|
|
|
|
void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
|
|
|
__end_buffer_read_notouch(bh, uptodate);
|
2005-04-16 16:20:36 -06:00
|
|
|
put_bh(bh);
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(end_buffer_read_sync);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
|
|
|
char b[BDEVNAME_SIZE];
|
|
|
|
|
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
2010-08-18 03:29:23 -06:00
|
|
|
if (!quiet_error(bh)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
buffer_io_error(bh);
|
|
|
|
printk(KERN_WARNING "lost page write due to "
|
|
|
|
"I/O error on %s\n",
|
|
|
|
bdevname(bh->b_bdev, b));
|
|
|
|
}
|
|
|
|
set_buffer_write_io_error(bh);
|
|
|
|
clear_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
unlock_buffer(bh);
|
|
|
|
put_bh(bh);
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(end_buffer_write_sync);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Various filesystems appear to want __find_get_block to be non-blocking.
|
|
|
|
* But it's the page lock which protects the buffers. To get around this,
|
|
|
|
* we get exclusion from try_to_free_buffers with the blockdev mapping's
|
|
|
|
* private_lock.
|
|
|
|
*
|
|
|
|
* Hack idea: for the blockdev mapping, i_bufferlist_lock contention
|
|
|
|
* may be quite high. This code could TryLock the page, and if that
|
|
|
|
* succeeds, there is no need to take private_lock. (But if
|
|
|
|
* private_lock is contended then so is mapping->tree_lock).
|
|
|
|
*/
|
|
|
|
static struct buffer_head *
|
2005-11-07 01:59:39 -07:00
|
|
|
__find_get_block_slow(struct block_device *bdev, sector_t block)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct inode *bd_inode = bdev->bd_inode;
|
|
|
|
struct address_space *bd_mapping = bd_inode->i_mapping;
|
|
|
|
struct buffer_head *ret = NULL;
|
|
|
|
pgoff_t index;
|
|
|
|
struct buffer_head *bh;
|
|
|
|
struct buffer_head *head;
|
|
|
|
struct page *page;
|
|
|
|
int all_mapped = 1;
|
|
|
|
|
|
|
|
index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
|
|
|
|
page = find_get_page(bd_mapping, index);
|
|
|
|
if (!page)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
spin_lock(&bd_mapping->private_lock);
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
goto out_unlock;
|
|
|
|
head = page_buffers(page);
|
|
|
|
bh = head;
|
|
|
|
do {
|
2009-04-02 17:56:46 -06:00
|
|
|
if (!buffer_mapped(bh))
|
|
|
|
all_mapped = 0;
|
|
|
|
else if (bh->b_blocknr == block) {
|
2005-04-16 16:20:36 -06:00
|
|
|
ret = bh;
|
|
|
|
get_bh(bh);
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
/* we might be here because some of the buffers on this page are
|
|
|
|
* not mapped. This is due to various races between
|
|
|
|
* file io on the block device and getblk. It gets dealt with
|
|
|
|
* elsewhere, don't buffer_error if we had some unmapped buffers
|
|
|
|
*/
|
|
|
|
if (all_mapped) {
|
|
|
|
printk("__find_get_block_slow() failed. "
|
|
|
|
"block=%llu, b_blocknr=%llu\n",
|
2006-03-26 02:38:00 -07:00
|
|
|
(unsigned long long)block,
|
|
|
|
(unsigned long long)bh->b_blocknr);
|
|
|
|
printk("b_state=0x%08lx, b_size=%zu\n",
|
|
|
|
bh->b_state, bh->b_size);
|
2005-04-16 16:20:36 -06:00
|
|
|
printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
|
|
|
|
}
|
|
|
|
out_unlock:
|
|
|
|
spin_unlock(&bd_mapping->private_lock);
|
|
|
|
page_cache_release(page);
|
|
|
|
out:
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If invalidate_buffers() will trash dirty buffers, it means some kind
|
|
|
|
of fs corruption is going on. Trashing dirty data always imply losing
|
|
|
|
information that was supposed to be just stored on the physical layer
|
|
|
|
by the user.
|
|
|
|
|
|
|
|
Thus invalidate_buffers in general usage is not allwowed to trash
|
|
|
|
dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
|
|
|
|
be preserved. These buffers are simply skipped.
|
|
|
|
|
|
|
|
We also skip buffers which are still in use. For example this can
|
|
|
|
happen if a userspace program is reading the block device.
|
|
|
|
|
|
|
|
NOTE: In the case where the user removed a removable-media-disk even if
|
|
|
|
there's still dirty data not synced on disk (due a bug in the device driver
|
|
|
|
or due an error of the user), by not destroying the dirty buffers we could
|
|
|
|
generate corruption also on the next media inserted, thus a parameter is
|
|
|
|
necessary to handle this case in the most safe way possible (trying
|
|
|
|
to not corrupt also the new disk inserted with the data belonging to
|
|
|
|
the old now corrupted disk). Also for the ramdisk the natural thing
|
|
|
|
to do in order to release the ramdisk memory is to destroy dirty buffers.
|
|
|
|
|
|
|
|
These are two special cases. Normal usage imply the device driver
|
|
|
|
to issue a sync on the device (without waiting I/O completion) and
|
|
|
|
then an invalidate_buffers call that doesn't trash dirty buffers.
|
|
|
|
|
|
|
|
For handling cache coherency with the blkdev pagecache the 'update' case
|
|
|
|
is been introduced. It is needed to re-read from disk any pinned
|
|
|
|
buffer. NOTE: re-reading from disk is destructive so we can do it only
|
|
|
|
when we assume nobody is changing the buffercache under our I/O and when
|
|
|
|
we think the disk contains more recent information than the buffercache.
|
|
|
|
The update == 1 pass marks the buffers we need to update, the update == 2
|
|
|
|
pass does the actual I/O. */
|
2007-05-06 15:49:54 -06:00
|
|
|
void invalidate_bdev(struct block_device *bdev)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2006-07-30 04:03:28 -06:00
|
|
|
struct address_space *mapping = bdev->bd_inode->i_mapping;
|
|
|
|
|
|
|
|
if (mapping->nrpages == 0)
|
|
|
|
return;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
invalidate_bh_lrus();
|
2010-05-15 12:09:27 -06:00
|
|
|
lru_add_drain_all(); /* make sure all lru add caches are flushed */
|
2007-02-10 02:45:39 -07:00
|
|
|
invalidate_mapping_pages(mapping, 0, -1);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(invalidate_bdev);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
2009-09-23 11:37:09 -06:00
|
|
|
* Kick the writeback threads then try to free up some ZONE_NORMAL memory.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
static void free_more_memory(void)
|
|
|
|
{
|
2008-04-28 03:12:18 -06:00
|
|
|
struct zone *zone;
|
2008-04-28 03:12:14 -06:00
|
|
|
int nid;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-09-09 01:08:54 -06:00
|
|
|
wakeup_flusher_threads(1024);
|
2005-04-16 16:20:36 -06:00
|
|
|
yield();
|
|
|
|
|
2008-04-28 03:12:14 -06:00
|
|
|
for_each_online_node(nid) {
|
2008-04-28 03:12:18 -06:00
|
|
|
(void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
|
|
|
|
gfp_zone(GFP_NOFS), NULL,
|
|
|
|
&zone);
|
|
|
|
if (zone)
|
2008-04-28 03:12:16 -06:00
|
|
|
try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
|
2009-03-31 16:23:31 -06:00
|
|
|
GFP_NOFS, NULL);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* I/O completion handler for block_read_full_page() - pages
|
|
|
|
* which come unlocked at the end of I/O.
|
|
|
|
*/
|
|
|
|
static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
2005-07-07 18:56:56 -06:00
|
|
|
struct buffer_head *first;
|
2005-04-16 16:20:36 -06:00
|
|
|
struct buffer_head *tmp;
|
|
|
|
struct page *page;
|
|
|
|
int page_uptodate = 1;
|
|
|
|
|
|
|
|
BUG_ON(!buffer_async_read(bh));
|
|
|
|
|
|
|
|
page = bh->b_page;
|
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
|
|
|
clear_buffer_uptodate(bh);
|
block: Supress Buffer I/O errors when SCSI REQ_QUIET flag set
Allow the scsi request REQ_QUIET flag to be propagated to the buffer
file system layer. The basic ideas is to pass the flag from the scsi
request to the bio (block IO) and then to the buffer layer. The buffer
layer can then suppress needless printks.
This patch declutters the kernel log by removed the 40-50 (per lun)
buffer io error messages seen during a boot in my multipath setup . It
is a good chance any real errors will be missed in the "noise" it the
logs without this patch.
During boot I see blocks of messages like
"
__ratelimit: 211 callbacks suppressed
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242847
Buffer I/O error on device sdm, logical block 1
Buffer I/O error on device sdm, logical block 5242878
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242872
"
in my logs.
My disk environment is multipath fiber channel using the SCSI_DH_RDAC
code and multipathd. This topology includes an "active" and "ghost"
path for each lun. IO's to the "ghost" path will never complete and the
SCSI layer, via the scsi device handler rdac code, quick returns the IOs
to theses paths and sets the REQ_QUIET scsi flag to suppress the scsi
layer messages.
I am wanting to extend the QUIET behavior to include the buffer file
system layer to deal with these errors as well. I have been running this
patch for a while now on several boxes without issue. A few runs of
bonnie++ show no noticeable difference in performance in my setup.
Thanks for John Stultz for the quiet_error finalization.
Submitted-by: Keith Mannthey <kmannth@us.ibm.com>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
2008-11-25 02:24:35 -07:00
|
|
|
if (!quiet_error(bh))
|
2005-04-16 16:20:36 -06:00
|
|
|
buffer_io_error(bh);
|
|
|
|
SetPageError(page);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Be _very_ careful from here on. Bad things can happen if
|
|
|
|
* two buffer heads end IO at almost the same time and both
|
|
|
|
* decide that the page is now completely done.
|
|
|
|
*/
|
2005-07-07 18:56:56 -06:00
|
|
|
first = page_buffers(page);
|
|
|
|
local_irq_save(flags);
|
|
|
|
bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
|
2005-04-16 16:20:36 -06:00
|
|
|
clear_buffer_async_read(bh);
|
|
|
|
unlock_buffer(bh);
|
|
|
|
tmp = bh;
|
|
|
|
do {
|
|
|
|
if (!buffer_uptodate(tmp))
|
|
|
|
page_uptodate = 0;
|
|
|
|
if (buffer_async_read(tmp)) {
|
|
|
|
BUG_ON(!buffer_locked(tmp));
|
|
|
|
goto still_busy;
|
|
|
|
}
|
|
|
|
tmp = tmp->b_this_page;
|
|
|
|
} while (tmp != bh);
|
2005-07-07 18:56:56 -06:00
|
|
|
bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
|
|
|
|
local_irq_restore(flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If none of the buffers had errors and they are all
|
|
|
|
* uptodate then we can set the page uptodate.
|
|
|
|
*/
|
|
|
|
if (page_uptodate && !PageError(page))
|
|
|
|
SetPageUptodate(page);
|
|
|
|
unlock_page(page);
|
|
|
|
return;
|
|
|
|
|
|
|
|
still_busy:
|
2005-07-07 18:56:56 -06:00
|
|
|
bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
|
|
|
|
local_irq_restore(flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Completion handler for block_write_full_page() - pages which are unlocked
|
|
|
|
* during I/O, and which have PageWriteback cleared upon I/O completion.
|
|
|
|
*/
|
2009-04-15 11:22:38 -06:00
|
|
|
void end_buffer_async_write(struct buffer_head *bh, int uptodate)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
char b[BDEVNAME_SIZE];
|
|
|
|
unsigned long flags;
|
2005-07-07 18:56:56 -06:00
|
|
|
struct buffer_head *first;
|
2005-04-16 16:20:36 -06:00
|
|
|
struct buffer_head *tmp;
|
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
BUG_ON(!buffer_async_write(bh));
|
|
|
|
|
|
|
|
page = bh->b_page;
|
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
block: Supress Buffer I/O errors when SCSI REQ_QUIET flag set
Allow the scsi request REQ_QUIET flag to be propagated to the buffer
file system layer. The basic ideas is to pass the flag from the scsi
request to the bio (block IO) and then to the buffer layer. The buffer
layer can then suppress needless printks.
This patch declutters the kernel log by removed the 40-50 (per lun)
buffer io error messages seen during a boot in my multipath setup . It
is a good chance any real errors will be missed in the "noise" it the
logs without this patch.
During boot I see blocks of messages like
"
__ratelimit: 211 callbacks suppressed
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242847
Buffer I/O error on device sdm, logical block 1
Buffer I/O error on device sdm, logical block 5242878
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242872
"
in my logs.
My disk environment is multipath fiber channel using the SCSI_DH_RDAC
code and multipathd. This topology includes an "active" and "ghost"
path for each lun. IO's to the "ghost" path will never complete and the
SCSI layer, via the scsi device handler rdac code, quick returns the IOs
to theses paths and sets the REQ_QUIET scsi flag to suppress the scsi
layer messages.
I am wanting to extend the QUIET behavior to include the buffer file
system layer to deal with these errors as well. I have been running this
patch for a while now on several boxes without issue. A few runs of
bonnie++ show no noticeable difference in performance in my setup.
Thanks for John Stultz for the quiet_error finalization.
Submitted-by: Keith Mannthey <kmannth@us.ibm.com>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
2008-11-25 02:24:35 -07:00
|
|
|
if (!quiet_error(bh)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
buffer_io_error(bh);
|
|
|
|
printk(KERN_WARNING "lost page write due to "
|
|
|
|
"I/O error on %s\n",
|
|
|
|
bdevname(bh->b_bdev, b));
|
|
|
|
}
|
|
|
|
set_bit(AS_EIO, &page->mapping->flags);
|
2006-10-17 01:10:19 -06:00
|
|
|
set_buffer_write_io_error(bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
clear_buffer_uptodate(bh);
|
|
|
|
SetPageError(page);
|
|
|
|
}
|
|
|
|
|
2005-07-07 18:56:56 -06:00
|
|
|
first = page_buffers(page);
|
|
|
|
local_irq_save(flags);
|
|
|
|
bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
clear_buffer_async_write(bh);
|
|
|
|
unlock_buffer(bh);
|
|
|
|
tmp = bh->b_this_page;
|
|
|
|
while (tmp != bh) {
|
|
|
|
if (buffer_async_write(tmp)) {
|
|
|
|
BUG_ON(!buffer_locked(tmp));
|
|
|
|
goto still_busy;
|
|
|
|
}
|
|
|
|
tmp = tmp->b_this_page;
|
|
|
|
}
|
2005-07-07 18:56:56 -06:00
|
|
|
bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
|
|
|
|
local_irq_restore(flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
end_page_writeback(page);
|
|
|
|
return;
|
|
|
|
|
|
|
|
still_busy:
|
2005-07-07 18:56:56 -06:00
|
|
|
bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
|
|
|
|
local_irq_restore(flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
return;
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(end_buffer_async_write);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If a page's buffers are under async readin (end_buffer_async_read
|
|
|
|
* completion) then there is a possibility that another thread of
|
|
|
|
* control could lock one of the buffers after it has completed
|
|
|
|
* but while some of the other buffers have not completed. This
|
|
|
|
* locked buffer would confuse end_buffer_async_read() into not unlocking
|
|
|
|
* the page. So the absence of BH_Async_Read tells end_buffer_async_read()
|
|
|
|
* that this buffer is not under async I/O.
|
|
|
|
*
|
|
|
|
* The page comes unlocked when it has no locked buffer_async buffers
|
|
|
|
* left.
|
|
|
|
*
|
|
|
|
* PageLocked prevents anyone starting new async I/O reads any of
|
|
|
|
* the buffers.
|
|
|
|
*
|
|
|
|
* PageWriteback is used to prevent simultaneous writeout of the same
|
|
|
|
* page.
|
|
|
|
*
|
|
|
|
* PageLocked prevents anyone from starting writeback of a page which is
|
|
|
|
* under read I/O (PageWriteback is only ever set against a locked page).
|
|
|
|
*/
|
|
|
|
static void mark_buffer_async_read(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
bh->b_end_io = end_buffer_async_read;
|
|
|
|
set_buffer_async_read(bh);
|
|
|
|
}
|
|
|
|
|
2009-09-22 17:43:51 -06:00
|
|
|
static void mark_buffer_async_write_endio(struct buffer_head *bh,
|
|
|
|
bh_end_io_t *handler)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2009-04-15 11:22:38 -06:00
|
|
|
bh->b_end_io = handler;
|
2005-04-16 16:20:36 -06:00
|
|
|
set_buffer_async_write(bh);
|
|
|
|
}
|
2009-04-15 11:22:38 -06:00
|
|
|
|
|
|
|
void mark_buffer_async_write(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
mark_buffer_async_write_endio(bh, end_buffer_async_write);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
EXPORT_SYMBOL(mark_buffer_async_write);
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fs/buffer.c contains helper functions for buffer-backed address space's
|
|
|
|
* fsync functions. A common requirement for buffer-based filesystems is
|
|
|
|
* that certain data from the backing blockdev needs to be written out for
|
|
|
|
* a successful fsync(). For example, ext2 indirect blocks need to be
|
|
|
|
* written back and waited upon before fsync() returns.
|
|
|
|
*
|
|
|
|
* The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
|
|
|
|
* inode_has_buffers() and invalidate_inode_buffers() are provided for the
|
|
|
|
* management of a list of dependent buffers at ->i_mapping->private_list.
|
|
|
|
*
|
|
|
|
* Locking is a little subtle: try_to_free_buffers() will remove buffers
|
|
|
|
* from their controlling inode's queue when they are being freed. But
|
|
|
|
* try_to_free_buffers() will be operating against the *blockdev* mapping
|
|
|
|
* at the time, not against the S_ISREG file which depends on those buffers.
|
|
|
|
* So the locking for private_list is via the private_lock in the address_space
|
|
|
|
* which backs the buffers. Which is different from the address_space
|
|
|
|
* against which the buffers are listed. So for a particular address_space,
|
|
|
|
* mapping->private_lock does *not* protect mapping->private_list! In fact,
|
|
|
|
* mapping->private_list will always be protected by the backing blockdev's
|
|
|
|
* ->private_lock.
|
|
|
|
*
|
|
|
|
* Which introduces a requirement: all buffers on an address_space's
|
|
|
|
* ->private_list must be from the same address_space: the blockdev's.
|
|
|
|
*
|
|
|
|
* address_spaces which do not place buffers at ->private_list via these
|
|
|
|
* utility functions are free to use private_lock and private_list for
|
|
|
|
* whatever they want. The only requirement is that list_empty(private_list)
|
|
|
|
* be true at clear_inode() time.
|
|
|
|
*
|
|
|
|
* FIXME: clear_inode should not call invalidate_inode_buffers(). The
|
|
|
|
* filesystems should do that. invalidate_inode_buffers() should just go
|
|
|
|
* BUG_ON(!list_empty).
|
|
|
|
*
|
|
|
|
* FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
|
|
|
|
* take an address_space, not an inode. And it should be called
|
|
|
|
* mark_buffer_dirty_fsync() to clearly define why those buffers are being
|
|
|
|
* queued up.
|
|
|
|
*
|
|
|
|
* FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
|
|
|
|
* list if it is already on a list. Because if the buffer is on a list,
|
|
|
|
* it *must* already be on the right one. If not, the filesystem is being
|
|
|
|
* silly. This will save a ton of locking. But first we have to ensure
|
|
|
|
* that buffers are taken *off* the old inode's list when they are freed
|
|
|
|
* (presumably in truncate). That requires careful auditing of all
|
|
|
|
* filesystems (do it inside bforget()). It could also be done by bringing
|
|
|
|
* b_inode back.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The buffer's backing address_space's private_lock must be held
|
|
|
|
*/
|
2008-07-29 23:33:47 -06:00
|
|
|
static void __remove_assoc_queue(struct buffer_head *bh)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
list_del_init(&bh->b_assoc_buffers);
|
2006-10-17 01:10:19 -06:00
|
|
|
WARN_ON(!bh->b_assoc_map);
|
|
|
|
if (buffer_write_io_error(bh))
|
|
|
|
set_bit(AS_EIO, &bh->b_assoc_map->flags);
|
|
|
|
bh->b_assoc_map = NULL;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
int inode_has_buffers(struct inode *inode)
|
|
|
|
{
|
|
|
|
return !list_empty(&inode->i_data.private_list);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* osync is designed to support O_SYNC io. It waits synchronously for
|
|
|
|
* all already-submitted IO to complete, but does not queue any new
|
|
|
|
* writes to the disk.
|
|
|
|
*
|
|
|
|
* To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
|
|
|
|
* you dirty the buffers, and then use osync_inode_buffers to wait for
|
|
|
|
* completion. Any other dirty buffers which are not yet queued for
|
|
|
|
* write will not be flushed to disk by the osync.
|
|
|
|
*/
|
|
|
|
static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh;
|
|
|
|
struct list_head *p;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
spin_lock(lock);
|
|
|
|
repeat:
|
|
|
|
list_for_each_prev(p, list) {
|
|
|
|
bh = BH_ENTRY(p);
|
|
|
|
if (buffer_locked(bh)) {
|
|
|
|
get_bh(bh);
|
|
|
|
spin_unlock(lock);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
err = -EIO;
|
|
|
|
brelse(bh);
|
|
|
|
spin_lock(lock);
|
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
spin_unlock(lock);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2010-03-23 04:06:58 -06:00
|
|
|
static void do_thaw_one(struct super_block *sb, void *unused)
|
2009-03-31 16:23:46 -06:00
|
|
|
{
|
|
|
|
char b[BDEVNAME_SIZE];
|
2010-03-23 04:06:58 -06:00
|
|
|
while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
|
|
|
|
printk(KERN_WARNING "Emergency Thaw on %s\n",
|
|
|
|
bdevname(sb->s_bdev, b));
|
|
|
|
}
|
2009-03-31 16:23:46 -06:00
|
|
|
|
2010-03-23 04:06:58 -06:00
|
|
|
static void do_thaw_all(struct work_struct *work)
|
|
|
|
{
|
|
|
|
iterate_supers(do_thaw_one, NULL);
|
2009-04-08 05:44:08 -06:00
|
|
|
kfree(work);
|
2009-03-31 16:23:46 -06:00
|
|
|
printk(KERN_WARNING "Emergency Thaw complete\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* emergency_thaw_all -- forcibly thaw every frozen filesystem
|
|
|
|
*
|
|
|
|
* Used for emergency unfreeze of all filesystems via SysRq
|
|
|
|
*/
|
|
|
|
void emergency_thaw_all(void)
|
|
|
|
{
|
2009-04-08 05:44:08 -06:00
|
|
|
struct work_struct *work;
|
|
|
|
|
|
|
|
work = kmalloc(sizeof(*work), GFP_ATOMIC);
|
|
|
|
if (work) {
|
|
|
|
INIT_WORK(work, do_thaw_all);
|
|
|
|
schedule_work(work);
|
|
|
|
}
|
2009-03-31 16:23:46 -06:00
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/**
|
2008-02-29 23:02:31 -07:00
|
|
|
* sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
|
2005-05-01 09:59:26 -06:00
|
|
|
* @mapping: the mapping which wants those buffers written
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* Starts I/O against the buffers at mapping->private_list, and waits upon
|
|
|
|
* that I/O.
|
|
|
|
*
|
2005-05-01 09:59:26 -06:00
|
|
|
* Basically, this is a convenience function for fsync().
|
|
|
|
* @mapping is a file or directory which needs those buffers to be written for
|
|
|
|
* a successful fsync().
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
int sync_mapping_buffers(struct address_space *mapping)
|
|
|
|
{
|
|
|
|
struct address_space *buffer_mapping = mapping->assoc_mapping;
|
|
|
|
|
|
|
|
if (buffer_mapping == NULL || list_empty(&mapping->private_list))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return fsync_buffers_list(&buffer_mapping->private_lock,
|
|
|
|
&mapping->private_list);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(sync_mapping_buffers);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Called when we've recently written block `bblock', and it is known that
|
|
|
|
* `bblock' was for a buffer_boundary() buffer. This means that the block at
|
|
|
|
* `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
|
|
|
|
* dirty, schedule it for IO. So that indirects merge nicely with their data.
|
|
|
|
*/
|
|
|
|
void write_boundary_block(struct block_device *bdev,
|
|
|
|
sector_t bblock, unsigned blocksize)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
|
|
|
|
if (bh) {
|
|
|
|
if (buffer_dirty(bh))
|
|
|
|
ll_rw_block(WRITE, 1, &bh);
|
|
|
|
put_bh(bh);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
|
|
|
|
{
|
|
|
|
struct address_space *mapping = inode->i_mapping;
|
|
|
|
struct address_space *buffer_mapping = bh->b_page->mapping;
|
|
|
|
|
|
|
|
mark_buffer_dirty(bh);
|
|
|
|
if (!mapping->assoc_mapping) {
|
|
|
|
mapping->assoc_mapping = buffer_mapping;
|
|
|
|
} else {
|
2006-03-26 09:24:46 -07:00
|
|
|
BUG_ON(mapping->assoc_mapping != buffer_mapping);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2008-02-08 05:21:59 -07:00
|
|
|
if (!bh->b_assoc_map) {
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_lock(&buffer_mapping->private_lock);
|
|
|
|
list_move_tail(&bh->b_assoc_buffers,
|
|
|
|
&mapping->private_list);
|
2006-10-17 01:10:19 -06:00
|
|
|
bh->b_assoc_map = mapping;
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_unlock(&buffer_mapping->private_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mark_buffer_dirty_inode);
|
|
|
|
|
2007-07-17 05:03:34 -06:00
|
|
|
/*
|
|
|
|
* Mark the page dirty, and set it dirty in the radix tree, and mark the inode
|
|
|
|
* dirty.
|
|
|
|
*
|
|
|
|
* If warn is true, then emit a warning if the page is not uptodate and has
|
|
|
|
* not been truncated.
|
|
|
|
*/
|
2009-03-19 12:32:05 -06:00
|
|
|
static void __set_page_dirty(struct page *page,
|
2007-07-17 05:03:34 -06:00
|
|
|
struct address_space *mapping, int warn)
|
|
|
|
{
|
2008-07-25 20:45:32 -06:00
|
|
|
spin_lock_irq(&mapping->tree_lock);
|
2007-07-17 05:03:34 -06:00
|
|
|
if (page->mapping) { /* Race with truncate? */
|
|
|
|
WARN_ON_ONCE(warn && !PageUptodate(page));
|
2009-03-31 16:19:39 -06:00
|
|
|
account_page_dirtied(page, mapping);
|
2007-07-17 05:03:34 -06:00
|
|
|
radix_tree_tag_set(&mapping->page_tree,
|
|
|
|
page_index(page), PAGECACHE_TAG_DIRTY);
|
|
|
|
}
|
2008-07-25 20:45:32 -06:00
|
|
|
spin_unlock_irq(&mapping->tree_lock);
|
2007-07-17 05:03:34 -06:00
|
|
|
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* Add a page to the dirty page list.
|
|
|
|
*
|
|
|
|
* It is a sad fact of life that this function is called from several places
|
|
|
|
* deeply under spinlocking. It may not sleep.
|
|
|
|
*
|
|
|
|
* If the page has buffers, the uptodate buffers are set dirty, to preserve
|
|
|
|
* dirty-state coherency between the page and the buffers. It the page does
|
|
|
|
* not have buffers then when they are later attached they will all be set
|
|
|
|
* dirty.
|
|
|
|
*
|
|
|
|
* The buffers are dirtied before the page is dirtied. There's a small race
|
|
|
|
* window in which a writepage caller may see the page cleanness but not the
|
|
|
|
* buffer dirtiness. That's fine. If this code were to set the page dirty
|
|
|
|
* before the buffers, a concurrent writepage caller could clear the page dirty
|
|
|
|
* bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
|
|
|
|
* page on the dirty page list.
|
|
|
|
*
|
|
|
|
* We use private_lock to lock against try_to_free_buffers while using the
|
|
|
|
* page's buffer list. Also use this to protect against clean buffers being
|
|
|
|
* added to the page after it was set dirty.
|
|
|
|
*
|
|
|
|
* FIXME: may need to call ->reservepage here as well. That's rather up to the
|
|
|
|
* address_space though.
|
|
|
|
*/
|
|
|
|
int __set_page_dirty_buffers(struct page *page)
|
|
|
|
{
|
2009-03-19 12:32:05 -06:00
|
|
|
int newly_dirty;
|
2007-07-17 05:03:34 -06:00
|
|
|
struct address_space *mapping = page_mapping(page);
|
2006-10-09 20:36:54 -06:00
|
|
|
|
|
|
|
if (unlikely(!mapping))
|
|
|
|
return !TestSetPageDirty(page);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
spin_lock(&mapping->private_lock);
|
|
|
|
if (page_has_buffers(page)) {
|
|
|
|
struct buffer_head *head = page_buffers(page);
|
|
|
|
struct buffer_head *bh = head;
|
|
|
|
|
|
|
|
do {
|
|
|
|
set_buffer_dirty(bh);
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
2009-03-19 12:32:05 -06:00
|
|
|
newly_dirty = !TestSetPageDirty(page);
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_unlock(&mapping->private_lock);
|
|
|
|
|
2009-03-19 12:32:05 -06:00
|
|
|
if (newly_dirty)
|
|
|
|
__set_page_dirty(page, mapping, 1);
|
|
|
|
return newly_dirty;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__set_page_dirty_buffers);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Write out and wait upon a list of buffers.
|
|
|
|
*
|
|
|
|
* We have conflicting pressures: we want to make sure that all
|
|
|
|
* initially dirty buffers get waited on, but that any subsequently
|
|
|
|
* dirtied buffers don't. After all, we don't want fsync to last
|
|
|
|
* forever if somebody is actively writing to the file.
|
|
|
|
*
|
|
|
|
* Do this in two main stages: first we copy dirty buffers to a
|
|
|
|
* temporary inode list, queueing the writes as we go. Then we clean
|
|
|
|
* up, waiting for those writes to complete.
|
|
|
|
*
|
|
|
|
* During this second stage, any subsequent updates to the file may end
|
|
|
|
* up refiling the buffer on the original inode's dirty list again, so
|
|
|
|
* there is a chance we will end up with a buffer queued for write but
|
|
|
|
* not yet completed on that list. So, as a final cleanup we go through
|
|
|
|
* the osync code to catch these locked, dirty buffers without requeuing
|
|
|
|
* any newly dirty buffers for write.
|
|
|
|
*/
|
|
|
|
static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh;
|
|
|
|
struct list_head tmp;
|
2009-04-06 06:48:03 -06:00
|
|
|
struct address_space *mapping, *prev_mapping = NULL;
|
2005-04-16 16:20:36 -06:00
|
|
|
int err = 0, err2;
|
|
|
|
|
|
|
|
INIT_LIST_HEAD(&tmp);
|
|
|
|
|
|
|
|
spin_lock(lock);
|
|
|
|
while (!list_empty(list)) {
|
|
|
|
bh = BH_ENTRY(list->next);
|
2008-02-08 05:21:59 -07:00
|
|
|
mapping = bh->b_assoc_map;
|
2006-10-17 01:10:19 -06:00
|
|
|
__remove_assoc_queue(bh);
|
2008-02-08 05:21:59 -07:00
|
|
|
/* Avoid race with mark_buffer_dirty_inode() which does
|
|
|
|
* a lockless check and we rely on seeing the dirty bit */
|
|
|
|
smp_mb();
|
2005-04-16 16:20:36 -06:00
|
|
|
if (buffer_dirty(bh) || buffer_locked(bh)) {
|
|
|
|
list_add(&bh->b_assoc_buffers, &tmp);
|
2008-02-08 05:21:59 -07:00
|
|
|
bh->b_assoc_map = mapping;
|
2005-04-16 16:20:36 -06:00
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
get_bh(bh);
|
|
|
|
spin_unlock(lock);
|
|
|
|
/*
|
|
|
|
* Ensure any pending I/O completes so that
|
2010-08-11 09:06:24 -06:00
|
|
|
* write_dirty_buffer() actually writes the
|
|
|
|
* current contents - it is a noop if I/O is
|
|
|
|
* still in flight on potentially older
|
|
|
|
* contents.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2010-08-11 09:06:24 -06:00
|
|
|
write_dirty_buffer(bh, WRITE_SYNC_PLUG);
|
2009-04-06 06:48:03 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Kick off IO for the previous mapping. Note
|
|
|
|
* that we will not run the very last mapping,
|
|
|
|
* wait_on_buffer() will do that for us
|
|
|
|
* through sync_buffer().
|
|
|
|
*/
|
|
|
|
if (prev_mapping && prev_mapping != mapping)
|
|
|
|
blk_run_address_space(prev_mapping);
|
|
|
|
prev_mapping = mapping;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
brelse(bh);
|
|
|
|
spin_lock(lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
while (!list_empty(&tmp)) {
|
|
|
|
bh = BH_ENTRY(tmp.prev);
|
|
|
|
get_bh(bh);
|
2008-02-08 05:21:59 -07:00
|
|
|
mapping = bh->b_assoc_map;
|
|
|
|
__remove_assoc_queue(bh);
|
|
|
|
/* Avoid race with mark_buffer_dirty_inode() which does
|
|
|
|
* a lockless check and we rely on seeing the dirty bit */
|
|
|
|
smp_mb();
|
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
list_add(&bh->b_assoc_buffers,
|
2008-03-04 15:28:33 -07:00
|
|
|
&mapping->private_list);
|
2008-02-08 05:21:59 -07:00
|
|
|
bh->b_assoc_map = mapping;
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_unlock(lock);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
err = -EIO;
|
|
|
|
brelse(bh);
|
|
|
|
spin_lock(lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock(lock);
|
|
|
|
err2 = osync_buffers_list(lock, list);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
else
|
|
|
|
return err2;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Invalidate any and all dirty buffers on a given inode. We are
|
|
|
|
* probably unmounting the fs, but that doesn't mean we have already
|
|
|
|
* done a sync(). Just drop the buffers from the inode list.
|
|
|
|
*
|
|
|
|
* NOTE: we take the inode's blockdev's mapping's private_lock. Which
|
|
|
|
* assumes that all the buffers are against the blockdev. Not true
|
|
|
|
* for reiserfs.
|
|
|
|
*/
|
|
|
|
void invalidate_inode_buffers(struct inode *inode)
|
|
|
|
{
|
|
|
|
if (inode_has_buffers(inode)) {
|
|
|
|
struct address_space *mapping = &inode->i_data;
|
|
|
|
struct list_head *list = &mapping->private_list;
|
|
|
|
struct address_space *buffer_mapping = mapping->assoc_mapping;
|
|
|
|
|
|
|
|
spin_lock(&buffer_mapping->private_lock);
|
|
|
|
while (!list_empty(list))
|
|
|
|
__remove_assoc_queue(BH_ENTRY(list->next));
|
|
|
|
spin_unlock(&buffer_mapping->private_lock);
|
|
|
|
}
|
|
|
|
}
|
2008-09-23 10:24:08 -06:00
|
|
|
EXPORT_SYMBOL(invalidate_inode_buffers);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove any clean buffers from the inode's buffer list. This is called
|
|
|
|
* when we're trying to free the inode itself. Those buffers can pin it.
|
|
|
|
*
|
|
|
|
* Returns true if all buffers were removed.
|
|
|
|
*/
|
|
|
|
int remove_inode_buffers(struct inode *inode)
|
|
|
|
{
|
|
|
|
int ret = 1;
|
|
|
|
|
|
|
|
if (inode_has_buffers(inode)) {
|
|
|
|
struct address_space *mapping = &inode->i_data;
|
|
|
|
struct list_head *list = &mapping->private_list;
|
|
|
|
struct address_space *buffer_mapping = mapping->assoc_mapping;
|
|
|
|
|
|
|
|
spin_lock(&buffer_mapping->private_lock);
|
|
|
|
while (!list_empty(list)) {
|
|
|
|
struct buffer_head *bh = BH_ENTRY(list->next);
|
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
ret = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
__remove_assoc_queue(bh);
|
|
|
|
}
|
|
|
|
spin_unlock(&buffer_mapping->private_lock);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create the appropriate buffers when given a page for data area and
|
|
|
|
* the size of each buffer.. Use the bh->b_this_page linked list to
|
|
|
|
* follow the buffers created. Return NULL if unable to create more
|
|
|
|
* buffers.
|
|
|
|
*
|
|
|
|
* The retry flag is used to differentiate async IO (paging, swapping)
|
|
|
|
* which may not fail from ordinary buffer allocations.
|
|
|
|
*/
|
|
|
|
struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
|
|
|
|
int retry)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh, *head;
|
|
|
|
long offset;
|
|
|
|
|
|
|
|
try_again:
|
|
|
|
head = NULL;
|
|
|
|
offset = PAGE_SIZE;
|
|
|
|
while ((offset -= size) >= 0) {
|
|
|
|
bh = alloc_buffer_head(GFP_NOFS);
|
|
|
|
if (!bh)
|
|
|
|
goto no_grow;
|
|
|
|
|
|
|
|
bh->b_bdev = NULL;
|
|
|
|
bh->b_this_page = head;
|
|
|
|
bh->b_blocknr = -1;
|
|
|
|
head = bh;
|
|
|
|
|
|
|
|
bh->b_state = 0;
|
|
|
|
atomic_set(&bh->b_count, 0);
|
|
|
|
bh->b_size = size;
|
|
|
|
|
|
|
|
/* Link the buffer to its page */
|
|
|
|
set_bh_page(bh, page, offset);
|
|
|
|
|
2006-01-16 15:02:07 -07:00
|
|
|
init_buffer(bh, NULL, NULL);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
return head;
|
|
|
|
/*
|
|
|
|
* In case anything failed, we just free everything we got.
|
|
|
|
*/
|
|
|
|
no_grow:
|
|
|
|
if (head) {
|
|
|
|
do {
|
|
|
|
bh = head;
|
|
|
|
head = head->b_this_page;
|
|
|
|
free_buffer_head(bh);
|
|
|
|
} while (head);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return failure for non-async IO requests. Async IO requests
|
|
|
|
* are not allowed to fail, so we have to wait until buffer heads
|
|
|
|
* become available. But we don't want tasks sleeping with
|
|
|
|
* partially complete buffers, so all were released above.
|
|
|
|
*/
|
|
|
|
if (!retry)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/* We're _really_ low on memory. Now we just
|
|
|
|
* wait for old buffer heads to become free due to
|
|
|
|
* finishing IO. Since this is an async request and
|
|
|
|
* the reserve list is empty, we're sure there are
|
|
|
|
* async buffer heads in use.
|
|
|
|
*/
|
|
|
|
free_more_memory();
|
|
|
|
goto try_again;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(alloc_page_buffers);
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
link_dev_buffers(struct page *page, struct buffer_head *head)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh, *tail;
|
|
|
|
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
tail = bh;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh);
|
|
|
|
tail->b_this_page = head;
|
|
|
|
attach_page_buffers(page, head);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialise the state of a blockdev page's buffers.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
init_page_buffers(struct page *page, struct block_device *bdev,
|
|
|
|
sector_t block, int size)
|
|
|
|
{
|
|
|
|
struct buffer_head *head = page_buffers(page);
|
|
|
|
struct buffer_head *bh = head;
|
|
|
|
int uptodate = PageUptodate(page);
|
|
|
|
|
|
|
|
do {
|
|
|
|
if (!buffer_mapped(bh)) {
|
|
|
|
init_buffer(bh, NULL, NULL);
|
|
|
|
bh->b_bdev = bdev;
|
|
|
|
bh->b_blocknr = block;
|
|
|
|
if (uptodate)
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
set_buffer_mapped(bh);
|
|
|
|
}
|
|
|
|
block++;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create the page-cache page that contains the requested block.
|
|
|
|
*
|
|
|
|
* This is user purely for blockdev mappings.
|
|
|
|
*/
|
|
|
|
static struct page *
|
|
|
|
grow_dev_page(struct block_device *bdev, sector_t block,
|
|
|
|
pgoff_t index, int size)
|
|
|
|
{
|
|
|
|
struct inode *inode = bdev->bd_inode;
|
|
|
|
struct page *page;
|
|
|
|
struct buffer_head *bh;
|
|
|
|
|
2007-05-16 23:11:21 -06:00
|
|
|
page = find_or_create_page(inode->i_mapping, index,
|
2007-07-17 05:03:05 -06:00
|
|
|
(mapping_gfp_mask(inode->i_mapping) & ~__GFP_FS)|__GFP_MOVABLE);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!page)
|
|
|
|
return NULL;
|
|
|
|
|
2006-03-26 09:24:46 -07:00
|
|
|
BUG_ON(!PageLocked(page));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (page_has_buffers(page)) {
|
|
|
|
bh = page_buffers(page);
|
|
|
|
if (bh->b_size == size) {
|
|
|
|
init_page_buffers(page, bdev, block, size);
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
if (!try_to_free_buffers(page))
|
|
|
|
goto failed;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate some buffers for this page
|
|
|
|
*/
|
|
|
|
bh = alloc_page_buffers(page, size, 0);
|
|
|
|
if (!bh)
|
|
|
|
goto failed;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Link the page to the buffers and initialise them. Take the
|
|
|
|
* lock to be atomic wrt __find_get_block(), which does not
|
|
|
|
* run under the page lock.
|
|
|
|
*/
|
|
|
|
spin_lock(&inode->i_mapping->private_lock);
|
|
|
|
link_dev_buffers(page, bh);
|
|
|
|
init_page_buffers(page, bdev, block, size);
|
|
|
|
spin_unlock(&inode->i_mapping->private_lock);
|
|
|
|
return page;
|
|
|
|
|
|
|
|
failed:
|
|
|
|
BUG();
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create buffers for the specified block device block's page. If
|
|
|
|
* that page was dirty, the buffers are set dirty also.
|
|
|
|
*/
|
2006-01-14 14:20:43 -07:00
|
|
|
static int
|
2005-04-16 16:20:36 -06:00
|
|
|
grow_buffers(struct block_device *bdev, sector_t block, int size)
|
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
pgoff_t index;
|
|
|
|
int sizebits;
|
|
|
|
|
|
|
|
sizebits = -1;
|
|
|
|
do {
|
|
|
|
sizebits++;
|
|
|
|
} while ((size << sizebits) < PAGE_SIZE);
|
|
|
|
|
|
|
|
index = block >> sizebits;
|
|
|
|
|
2006-10-11 02:21:46 -06:00
|
|
|
/*
|
|
|
|
* Check for a block which wants to lie outside our maximum possible
|
|
|
|
* pagecache index. (this comparison is done using sector_t types).
|
|
|
|
*/
|
|
|
|
if (unlikely(index != block >> sizebits)) {
|
|
|
|
char b[BDEVNAME_SIZE];
|
|
|
|
|
|
|
|
printk(KERN_ERR "%s: requested out-of-range block %llu for "
|
|
|
|
"device %s\n",
|
2008-04-30 01:55:09 -06:00
|
|
|
__func__, (unsigned long long)block,
|
2006-10-11 02:21:46 -06:00
|
|
|
bdevname(bdev, b));
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
block = index << sizebits;
|
2005-04-16 16:20:36 -06:00
|
|
|
/* Create a page with the proper size buffers.. */
|
|
|
|
page = grow_dev_page(bdev, block, index, size);
|
|
|
|
if (!page)
|
|
|
|
return 0;
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2005-05-05 17:16:09 -06:00
|
|
|
static struct buffer_head *
|
2005-04-16 16:20:36 -06:00
|
|
|
__getblk_slow(struct block_device *bdev, sector_t block, int size)
|
|
|
|
{
|
|
|
|
/* Size must be multiple of hard sectorsize */
|
2009-05-22 15:17:49 -06:00
|
|
|
if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
|
2005-04-16 16:20:36 -06:00
|
|
|
(size < 512 || size > PAGE_SIZE))) {
|
|
|
|
printk(KERN_ERR "getblk(): invalid block size %d requested\n",
|
|
|
|
size);
|
2009-05-22 15:17:49 -06:00
|
|
|
printk(KERN_ERR "logical block size: %d\n",
|
|
|
|
bdev_logical_block_size(bdev));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
dump_stack();
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
struct buffer_head * bh;
|
2006-10-11 02:21:46 -06:00
|
|
|
int ret;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
bh = __find_get_block(bdev, block, size);
|
|
|
|
if (bh)
|
|
|
|
return bh;
|
|
|
|
|
2006-10-11 02:21:46 -06:00
|
|
|
ret = grow_buffers(bdev, block, size);
|
|
|
|
if (ret < 0)
|
|
|
|
return NULL;
|
|
|
|
if (ret == 0)
|
2005-04-16 16:20:36 -06:00
|
|
|
free_more_memory();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The relationship between dirty buffers and dirty pages:
|
|
|
|
*
|
|
|
|
* Whenever a page has any dirty buffers, the page's dirty bit is set, and
|
|
|
|
* the page is tagged dirty in its radix tree.
|
|
|
|
*
|
|
|
|
* At all times, the dirtiness of the buffers represents the dirtiness of
|
|
|
|
* subsections of the page. If the page has buffers, the page dirty bit is
|
|
|
|
* merely a hint about the true dirty state.
|
|
|
|
*
|
|
|
|
* When a page is set dirty in its entirety, all its buffers are marked dirty
|
|
|
|
* (if the page has buffers).
|
|
|
|
*
|
|
|
|
* When a buffer is marked dirty, its page is dirtied, but the page's other
|
|
|
|
* buffers are not.
|
|
|
|
*
|
|
|
|
* Also. When blockdev buffers are explicitly read with bread(), they
|
|
|
|
* individually become uptodate. But their backing page remains not
|
|
|
|
* uptodate - even if all of its buffers are uptodate. A subsequent
|
|
|
|
* block_read_full_page() against that page will discover all the uptodate
|
|
|
|
* buffers, will set the page uptodate and will perform no I/O.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* mark_buffer_dirty - mark a buffer_head as needing writeout
|
2005-05-01 09:59:26 -06:00
|
|
|
* @bh: the buffer_head to mark dirty
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* mark_buffer_dirty() will set the dirty bit against the buffer, then set its
|
|
|
|
* backing page dirty, then tag the page as dirty in its address_space's radix
|
|
|
|
* tree and then attach the address_space's inode to its superblock's dirty
|
|
|
|
* inode list.
|
|
|
|
*
|
|
|
|
* mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
|
2011-03-22 05:23:36 -06:00
|
|
|
* mapping->tree_lock and mapping->host->i_lock.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2008-02-08 05:19:52 -07:00
|
|
|
void mark_buffer_dirty(struct buffer_head *bh)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-07-17 05:03:34 -06:00
|
|
|
WARN_ON_ONCE(!buffer_uptodate(bh));
|
2008-04-04 15:38:17 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Very *carefully* optimize the it-is-already-dirty case.
|
|
|
|
*
|
|
|
|
* Don't let the final "is it dirty" escape to before we
|
|
|
|
* perhaps modified the buffer.
|
|
|
|
*/
|
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
smp_mb();
|
|
|
|
if (buffer_dirty(bh))
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2009-03-19 12:32:05 -06:00
|
|
|
if (!test_set_buffer_dirty(bh)) {
|
|
|
|
struct page *page = bh->b_page;
|
2009-08-21 18:40:08 -06:00
|
|
|
if (!TestSetPageDirty(page)) {
|
|
|
|
struct address_space *mapping = page_mapping(page);
|
|
|
|
if (mapping)
|
|
|
|
__set_page_dirty(page, mapping, 0);
|
|
|
|
}
|
2009-03-19 12:32:05 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(mark_buffer_dirty);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Decrement a buffer_head's reference count. If all buffers against a page
|
|
|
|
* have zero reference count, are clean and unlocked, and if the page is clean
|
|
|
|
* and unlocked then try_to_free_buffers() may strip the buffers from the page
|
|
|
|
* in preparation for freeing it (sometimes, rarely, buffers are removed from
|
|
|
|
* a page but it ends up not being freed, and buffers may later be reattached).
|
|
|
|
*/
|
|
|
|
void __brelse(struct buffer_head * buf)
|
|
|
|
{
|
|
|
|
if (atomic_read(&buf->b_count)) {
|
|
|
|
put_bh(buf);
|
|
|
|
return;
|
|
|
|
}
|
2008-07-25 20:45:40 -06:00
|
|
|
WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(__brelse);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* bforget() is like brelse(), except it discards any
|
|
|
|
* potentially dirty data.
|
|
|
|
*/
|
|
|
|
void __bforget(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
clear_buffer_dirty(bh);
|
2008-02-08 05:21:59 -07:00
|
|
|
if (bh->b_assoc_map) {
|
2005-04-16 16:20:36 -06:00
|
|
|
struct address_space *buffer_mapping = bh->b_page->mapping;
|
|
|
|
|
|
|
|
spin_lock(&buffer_mapping->private_lock);
|
|
|
|
list_del_init(&bh->b_assoc_buffers);
|
2006-10-17 01:10:19 -06:00
|
|
|
bh->b_assoc_map = NULL;
|
2005-04-16 16:20:36 -06:00
|
|
|
spin_unlock(&buffer_mapping->private_lock);
|
|
|
|
}
|
|
|
|
__brelse(bh);
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(__bforget);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
static struct buffer_head *__bread_slow(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (buffer_uptodate(bh)) {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
return bh;
|
|
|
|
} else {
|
|
|
|
get_bh(bh);
|
|
|
|
bh->b_end_io = end_buffer_read_sync;
|
|
|
|
submit_bh(READ, bh);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
return bh;
|
|
|
|
}
|
|
|
|
brelse(bh);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
|
|
|
|
* The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
|
|
|
|
* refcount elevated by one when they're in an LRU. A buffer can only appear
|
|
|
|
* once in a particular CPU's LRU. A single buffer can be present in multiple
|
|
|
|
* CPU's LRUs at the same time.
|
|
|
|
*
|
|
|
|
* This is a transparent caching front-end to sb_bread(), sb_getblk() and
|
|
|
|
* sb_find_get_block().
|
|
|
|
*
|
|
|
|
* The LRUs themselves only need locking against invalidate_bh_lrus. We use
|
|
|
|
* a local interrupt disable for that.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define BH_LRU_SIZE 8
|
|
|
|
|
|
|
|
struct bh_lru {
|
|
|
|
struct buffer_head *bhs[BH_LRU_SIZE];
|
|
|
|
};
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
#define bh_lru_lock() local_irq_disable()
|
|
|
|
#define bh_lru_unlock() local_irq_enable()
|
|
|
|
#else
|
|
|
|
#define bh_lru_lock() preempt_disable()
|
|
|
|
#define bh_lru_unlock() preempt_enable()
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static inline void check_irqs_on(void)
|
|
|
|
{
|
|
|
|
#ifdef irqs_disabled
|
|
|
|
BUG_ON(irqs_disabled());
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The LRU management algorithm is dopey-but-simple. Sorry.
|
|
|
|
*/
|
|
|
|
static void bh_lru_install(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
struct buffer_head *evictee = NULL;
|
|
|
|
|
|
|
|
check_irqs_on();
|
|
|
|
bh_lru_lock();
|
2010-12-06 10:16:28 -07:00
|
|
|
if (__this_cpu_read(bh_lrus.bhs[0]) != bh) {
|
2005-04-16 16:20:36 -06:00
|
|
|
struct buffer_head *bhs[BH_LRU_SIZE];
|
|
|
|
int in;
|
|
|
|
int out = 0;
|
|
|
|
|
|
|
|
get_bh(bh);
|
|
|
|
bhs[out++] = bh;
|
|
|
|
for (in = 0; in < BH_LRU_SIZE; in++) {
|
2010-12-06 10:16:28 -07:00
|
|
|
struct buffer_head *bh2 =
|
|
|
|
__this_cpu_read(bh_lrus.bhs[in]);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (bh2 == bh) {
|
|
|
|
__brelse(bh2);
|
|
|
|
} else {
|
|
|
|
if (out >= BH_LRU_SIZE) {
|
|
|
|
BUG_ON(evictee != NULL);
|
|
|
|
evictee = bh2;
|
|
|
|
} else {
|
|
|
|
bhs[out++] = bh2;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
while (out < BH_LRU_SIZE)
|
|
|
|
bhs[out++] = NULL;
|
2010-12-06 10:16:28 -07:00
|
|
|
memcpy(__this_cpu_ptr(&bh_lrus.bhs), bhs, sizeof(bhs));
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
bh_lru_unlock();
|
|
|
|
|
|
|
|
if (evictee)
|
|
|
|
__brelse(evictee);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Look up the bh in this cpu's LRU. If it's there, move it to the head.
|
|
|
|
*/
|
2006-01-14 14:20:43 -07:00
|
|
|
static struct buffer_head *
|
2007-02-12 01:52:14 -07:00
|
|
|
lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct buffer_head *ret = NULL;
|
2007-02-12 01:52:14 -07:00
|
|
|
unsigned int i;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
check_irqs_on();
|
|
|
|
bh_lru_lock();
|
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
2010-12-06 10:16:28 -07:00
|
|
|
struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
if (bh && bh->b_bdev == bdev &&
|
|
|
|
bh->b_blocknr == block && bh->b_size == size) {
|
|
|
|
if (i) {
|
|
|
|
while (i) {
|
2010-12-06 10:16:28 -07:00
|
|
|
__this_cpu_write(bh_lrus.bhs[i],
|
|
|
|
__this_cpu_read(bh_lrus.bhs[i - 1]));
|
2005-04-16 16:20:36 -06:00
|
|
|
i--;
|
|
|
|
}
|
2010-12-06 10:16:28 -07:00
|
|
|
__this_cpu_write(bh_lrus.bhs[0], bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
get_bh(bh);
|
|
|
|
ret = bh;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
bh_lru_unlock();
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Perform a pagecache lookup for the matching buffer. If it's there, refresh
|
|
|
|
* it in the LRU and mark it as accessed. If it is not present then return
|
|
|
|
* NULL
|
|
|
|
*/
|
|
|
|
struct buffer_head *
|
2007-02-12 01:52:14 -07:00
|
|
|
__find_get_block(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
|
|
|
|
|
|
|
|
if (bh == NULL) {
|
2005-11-07 01:59:39 -07:00
|
|
|
bh = __find_get_block_slow(bdev, block);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (bh)
|
|
|
|
bh_lru_install(bh);
|
|
|
|
}
|
|
|
|
if (bh)
|
|
|
|
touch_buffer(bh);
|
|
|
|
return bh;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__find_get_block);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* __getblk will locate (and, if necessary, create) the buffer_head
|
|
|
|
* which corresponds to the passed block_device, block and size. The
|
|
|
|
* returned buffer has its reference count incremented.
|
|
|
|
*
|
|
|
|
* __getblk() cannot fail - it just keeps trying. If you pass it an
|
|
|
|
* illegal block number, __getblk() will happily return a buffer_head
|
|
|
|
* which represents the non-existent block. Very weird.
|
|
|
|
*
|
|
|
|
* __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
|
|
|
|
* attempt is failing. FIXME, perhaps?
|
|
|
|
*/
|
|
|
|
struct buffer_head *
|
2007-02-12 01:52:14 -07:00
|
|
|
__getblk(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = __find_get_block(bdev, block, size);
|
|
|
|
|
|
|
|
might_sleep();
|
|
|
|
if (bh == NULL)
|
|
|
|
bh = __getblk_slow(bdev, block, size);
|
|
|
|
return bh;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__getblk);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Do async read-ahead on a buffer..
|
|
|
|
*/
|
2007-02-12 01:52:14 -07:00
|
|
|
void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = __getblk(bdev, block, size);
|
2005-10-30 16:03:15 -07:00
|
|
|
if (likely(bh)) {
|
|
|
|
ll_rw_block(READA, 1, &bh);
|
|
|
|
brelse(bh);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__breadahead);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* __bread() - reads a specified block and returns the bh
|
2005-05-01 09:59:26 -06:00
|
|
|
* @bdev: the block_device to read from
|
2005-04-16 16:20:36 -06:00
|
|
|
* @block: number of block
|
|
|
|
* @size: size (in bytes) to read
|
|
|
|
*
|
|
|
|
* Reads a specified block, and returns buffer head that contains it.
|
|
|
|
* It returns NULL if the block was unreadable.
|
|
|
|
*/
|
|
|
|
struct buffer_head *
|
2007-02-12 01:52:14 -07:00
|
|
|
__bread(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = __getblk(bdev, block, size);
|
|
|
|
|
2005-10-30 16:03:15 -07:00
|
|
|
if (likely(bh) && !buffer_uptodate(bh))
|
2005-04-16 16:20:36 -06:00
|
|
|
bh = __bread_slow(bh);
|
|
|
|
return bh;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__bread);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* invalidate_bh_lrus() is called rarely - but not only at unmount.
|
|
|
|
* This doesn't race because it runs in each cpu either in irq
|
|
|
|
* or with preempt disabled.
|
|
|
|
*/
|
|
|
|
static void invalidate_bh_lru(void *arg)
|
|
|
|
{
|
|
|
|
struct bh_lru *b = &get_cpu_var(bh_lrus);
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
|
|
|
brelse(b->bhs[i]);
|
|
|
|
b->bhs[i] = NULL;
|
|
|
|
}
|
|
|
|
put_cpu_var(bh_lrus);
|
|
|
|
}
|
|
|
|
|
2007-05-06 15:49:55 -06:00
|
|
|
void invalidate_bh_lrus(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-05-09 01:39:44 -06:00
|
|
|
on_each_cpu(invalidate_bh_lru, NULL, 1);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
rewrite rd
This is a rewrite of the ramdisk block device driver.
The old one is really difficult because it effectively implements a block
device which serves data out of its own buffer cache. It relies on the dirty
bit being set, to pin its backing store in cache, however there are non
trivial paths which can clear the dirty bit (eg. try_to_free_buffers()),
which had recently lead to data corruption. And in general it is completely
wrong for a block device driver to do this.
The new one is more like a regular block device driver. It has no idea about
vm/vfs stuff. It's backing store is similar to the buffer cache (a simple
radix-tree of pages), but it doesn't know anything about page cache (the pages
in the radix tree are not pagecache pages).
There is one slight downside -- direct block device access and filesystem
metadata access goes through an extra copy and gets stored in RAM twice.
However, this downside is only slight, because the real buffercache of the
device is now reclaimable (because we're not playing crazy games with it), so
under memory intensive situations, footprint should effectively be the same --
maybe even a slight advantage to the new driver because it can also reclaim
buffer heads.
The fact that it now goes through all the regular vm/fs paths makes it
much more useful for testing, too.
text data bss dec hex filename
2837 849 384 4070 fe6 drivers/block/rd.o
3528 371 12 3911 f47 drivers/block/brd.o
Text is larger, but data and bss are smaller, making total size smaller.
A few other nice things about it:
- Similar structure and layout to the new loop device handlinag.
- Dynamic ramdisk creation.
- Runtime flexible buffer head size (because it is no longer part of the
ramdisk code).
- Boot / load time flexible ramdisk size, which could easily be extended
to a per-ramdisk runtime changeable size (eg. with an ioctl).
- Can use highmem for the backing store.
[akpm@linux-foundation.org: fix build]
[byron.bbradley@gmail.com: make rd_size non-static]
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Byron Bradley <byron.bbradley@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-08 05:19:49 -07:00
|
|
|
EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
void set_bh_page(struct buffer_head *bh,
|
|
|
|
struct page *page, unsigned long offset)
|
|
|
|
{
|
|
|
|
bh->b_page = page;
|
2006-03-26 09:24:46 -07:00
|
|
|
BUG_ON(offset >= PAGE_SIZE);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (PageHighMem(page))
|
|
|
|
/*
|
|
|
|
* This catches illegal uses and preserves the offset:
|
|
|
|
*/
|
|
|
|
bh->b_data = (char *)(0 + offset);
|
|
|
|
else
|
|
|
|
bh->b_data = page_address(page) + offset;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(set_bh_page);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Called when truncating a buffer on a page completely.
|
|
|
|
*/
|
2006-01-14 14:20:43 -07:00
|
|
|
static void discard_buffer(struct buffer_head * bh)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
lock_buffer(bh);
|
|
|
|
clear_buffer_dirty(bh);
|
|
|
|
bh->b_bdev = NULL;
|
|
|
|
clear_buffer_mapped(bh);
|
|
|
|
clear_buffer_req(bh);
|
|
|
|
clear_buffer_new(bh);
|
|
|
|
clear_buffer_delay(bh);
|
2007-02-12 01:51:41 -07:00
|
|
|
clear_buffer_unwritten(bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* block_invalidatepage - invalidate part of all of a buffer-backed page
|
|
|
|
*
|
|
|
|
* @page: the page which is affected
|
|
|
|
* @offset: the index of the truncation point
|
|
|
|
*
|
|
|
|
* block_invalidatepage() is called when all or part of the page has become
|
|
|
|
* invalidatedby a truncate operation.
|
|
|
|
*
|
|
|
|
* block_invalidatepage() does not have to release all buffers, but it must
|
|
|
|
* ensure that no dirty buffer is left outside @offset and that no I/O
|
|
|
|
* is underway against any of the blocks which are outside the truncation
|
|
|
|
* point. Because the caller is about to free (and possibly reuse) those
|
|
|
|
* blocks on-disk.
|
|
|
|
*/
|
2006-03-26 02:37:18 -07:00
|
|
|
void block_invalidatepage(struct page *page, unsigned long offset)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct buffer_head *head, *bh, *next;
|
|
|
|
unsigned int curr_off = 0;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
head = page_buffers(page);
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
unsigned int next_off = curr_off + bh->b_size;
|
|
|
|
next = bh->b_this_page;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* is this block fully invalidated?
|
|
|
|
*/
|
|
|
|
if (offset <= curr_off)
|
|
|
|
discard_buffer(bh);
|
|
|
|
curr_off = next_off;
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We release buffers only if the entire page is being invalidated.
|
|
|
|
* The get_block cached value has been unconditionally invalidated,
|
|
|
|
* so real IO is not possible anymore.
|
|
|
|
*/
|
|
|
|
if (offset == 0)
|
2006-03-26 02:37:18 -07:00
|
|
|
try_to_release_page(page, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
out:
|
2006-03-26 02:37:18 -07:00
|
|
|
return;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_invalidatepage);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We attach and possibly dirty the buffers atomically wrt
|
|
|
|
* __set_page_dirty_buffers() via private_lock. try_to_free_buffers
|
|
|
|
* is already excluded via the page lock.
|
|
|
|
*/
|
|
|
|
void create_empty_buffers(struct page *page,
|
|
|
|
unsigned long blocksize, unsigned long b_state)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh, *head, *tail;
|
|
|
|
|
|
|
|
head = alloc_page_buffers(page, blocksize, 1);
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
bh->b_state |= b_state;
|
|
|
|
tail = bh;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh);
|
|
|
|
tail->b_this_page = head;
|
|
|
|
|
|
|
|
spin_lock(&page->mapping->private_lock);
|
|
|
|
if (PageUptodate(page) || PageDirty(page)) {
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
if (PageDirty(page))
|
|
|
|
set_buffer_dirty(bh);
|
|
|
|
if (PageUptodate(page))
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
|
|
|
attach_page_buffers(page, head);
|
|
|
|
spin_unlock(&page->mapping->private_lock);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(create_empty_buffers);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We are taking a block for data and we don't want any output from any
|
|
|
|
* buffer-cache aliases starting from return from that function and
|
|
|
|
* until the moment when something will explicitly mark the buffer
|
|
|
|
* dirty (hopefully that will not happen until we will free that block ;-)
|
|
|
|
* We don't even need to mark it not-uptodate - nobody can expect
|
|
|
|
* anything from a newly allocated buffer anyway. We used to used
|
|
|
|
* unmap_buffer() for such invalidation, but that was wrong. We definitely
|
|
|
|
* don't want to mark the alias unmapped, for example - it would confuse
|
|
|
|
* anyone who might pick it with bread() afterwards...
|
|
|
|
*
|
|
|
|
* Also.. Note that bforget() doesn't lock the buffer. So there can
|
|
|
|
* be writeout I/O going on against recently-freed buffers. We don't
|
|
|
|
* wait on that I/O in bforget() - it's more efficient to wait on the I/O
|
|
|
|
* only if we really need to. That happens here.
|
|
|
|
*/
|
|
|
|
void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
|
|
|
|
{
|
|
|
|
struct buffer_head *old_bh;
|
|
|
|
|
|
|
|
might_sleep();
|
|
|
|
|
2005-11-07 01:59:39 -07:00
|
|
|
old_bh = __find_get_block_slow(bdev, block);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (old_bh) {
|
|
|
|
clear_buffer_dirty(old_bh);
|
|
|
|
wait_on_buffer(old_bh);
|
|
|
|
clear_buffer_req(old_bh);
|
|
|
|
__brelse(old_bh);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(unmap_underlying_metadata);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* NOTE! All mapped/uptodate combinations are valid:
|
|
|
|
*
|
|
|
|
* Mapped Uptodate Meaning
|
|
|
|
*
|
|
|
|
* No No "unknown" - must do get_block()
|
|
|
|
* No Yes "hole" - zero-filled
|
|
|
|
* Yes No "allocated" - allocated on disk, not read in
|
|
|
|
* Yes Yes "valid" - allocated and up-to-date in memory.
|
|
|
|
*
|
|
|
|
* "Dirty" is valid only with the last case (mapped+uptodate).
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* While block_write_full_page is writing back the dirty buffers under
|
|
|
|
* the page lock, whoever dirtied the buffers may decide to clean them
|
|
|
|
* again at any time. We handle that by only looking at the buffer
|
|
|
|
* state inside lock_buffer().
|
|
|
|
*
|
|
|
|
* If block_write_full_page() is called for regular writeback
|
|
|
|
* (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
|
|
|
|
* locked buffer. This only can happen if someone has written the buffer
|
|
|
|
* directly, with submit_bh(). At the address_space level PageWriteback
|
|
|
|
* prevents this contention from occurring.
|
2009-04-07 16:12:43 -06:00
|
|
|
*
|
|
|
|
* If block_write_full_page() is called with wbc->sync_mode ==
|
|
|
|
* WB_SYNC_ALL, the writes are posted using WRITE_SYNC_PLUG; this
|
|
|
|
* causes the writes to be flagged as synchronous writes, but the
|
|
|
|
* block device queue will NOT be unplugged, since usually many pages
|
|
|
|
* will be pushed to the out before the higher-level caller actually
|
|
|
|
* waits for the writes to be completed. The various wait functions,
|
|
|
|
* such as wait_on_writeback_range() will ultimately call sync_page()
|
|
|
|
* which will ultimately call blk_run_backing_dev(), which will end up
|
|
|
|
* unplugging the device queue.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
|
|
|
static int __block_write_full_page(struct inode *inode, struct page *page,
|
2009-04-15 11:22:38 -06:00
|
|
|
get_block_t *get_block, struct writeback_control *wbc,
|
|
|
|
bh_end_io_t *handler)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
sector_t block;
|
|
|
|
sector_t last_block;
|
2005-05-05 17:15:48 -06:00
|
|
|
struct buffer_head *bh, *head;
|
2006-03-26 02:38:00 -07:00
|
|
|
const unsigned blocksize = 1 << inode->i_blkbits;
|
2005-04-16 16:20:36 -06:00
|
|
|
int nr_underway = 0;
|
2009-04-07 16:12:43 -06:00
|
|
|
int write_op = (wbc->sync_mode == WB_SYNC_ALL ?
|
|
|
|
WRITE_SYNC_PLUG : WRITE);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
|
|
|
|
last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
|
|
|
|
|
|
|
|
if (!page_has_buffers(page)) {
|
2006-03-26 02:38:00 -07:00
|
|
|
create_empty_buffers(page, blocksize,
|
2005-04-16 16:20:36 -06:00
|
|
|
(1 << BH_Dirty)|(1 << BH_Uptodate));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Be very careful. We have no exclusion from __set_page_dirty_buffers
|
|
|
|
* here, and the (potentially unmapped) buffers may become dirty at
|
|
|
|
* any time. If a buffer becomes dirty here after we've inspected it
|
|
|
|
* then we just miss that fact, and the page stays dirty.
|
|
|
|
*
|
|
|
|
* Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
|
|
|
|
* handle that here by just cleaning them.
|
|
|
|
*/
|
|
|
|
|
2006-01-08 02:03:05 -07:00
|
|
|
block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
|
2005-04-16 16:20:36 -06:00
|
|
|
head = page_buffers(page);
|
|
|
|
bh = head;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get all the dirty buffers mapped to disk addresses and
|
|
|
|
* handle any aliases from the underlying blockdev's mapping.
|
|
|
|
*/
|
|
|
|
do {
|
|
|
|
if (block > last_block) {
|
|
|
|
/*
|
|
|
|
* mapped buffers outside i_size will occur, because
|
|
|
|
* this page can be outside i_size when there is a
|
|
|
|
* truncate in progress.
|
|
|
|
*/
|
|
|
|
/*
|
|
|
|
* The buffer was zeroed by block_write_full_page()
|
|
|
|
*/
|
|
|
|
clear_buffer_dirty(bh);
|
|
|
|
set_buffer_uptodate(bh);
|
2008-07-11 17:27:31 -06:00
|
|
|
} else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
|
|
|
|
buffer_dirty(bh)) {
|
2006-03-26 02:38:00 -07:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-04-16 16:20:36 -06:00
|
|
|
err = get_block(inode, block, bh, 1);
|
|
|
|
if (err)
|
|
|
|
goto recover;
|
2008-07-11 17:27:31 -06:00
|
|
|
clear_buffer_delay(bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (buffer_new(bh)) {
|
|
|
|
/* blockdev mappings never come here */
|
|
|
|
clear_buffer_new(bh);
|
|
|
|
unmap_underlying_metadata(bh->b_bdev,
|
|
|
|
bh->b_blocknr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
block++;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
do {
|
|
|
|
if (!buffer_mapped(bh))
|
|
|
|
continue;
|
|
|
|
/*
|
|
|
|
* If it's a fully non-blocking write attempt and we cannot
|
|
|
|
* lock the buffer then redirty the page. Note that this can
|
2009-09-23 11:37:09 -06:00
|
|
|
* potentially cause a busy-wait loop from writeback threads
|
|
|
|
* and kswapd activity, but those code paths have their own
|
|
|
|
* higher-level throttling.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2010-10-26 15:21:26 -06:00
|
|
|
if (wbc->sync_mode != WB_SYNC_NONE) {
|
2005-04-16 16:20:36 -06:00
|
|
|
lock_buffer(bh);
|
2008-08-02 04:02:13 -06:00
|
|
|
} else if (!trylock_buffer(bh)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
redirty_page_for_writepage(wbc, page);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (test_clear_buffer_dirty(bh)) {
|
2009-04-15 11:22:38 -06:00
|
|
|
mark_buffer_async_write_endio(bh, handler);
|
2005-04-16 16:20:36 -06:00
|
|
|
} else {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
} while ((bh = bh->b_this_page) != head);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The page and its buffers are protected by PageWriteback(), so we can
|
|
|
|
* drop the bh refcounts early.
|
|
|
|
*/
|
|
|
|
BUG_ON(PageWriteback(page));
|
|
|
|
set_page_writeback(page);
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
if (buffer_async_write(bh)) {
|
2009-03-27 20:14:10 -06:00
|
|
|
submit_bh(write_op, bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
nr_underway++;
|
|
|
|
}
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
2005-05-05 17:15:47 -06:00
|
|
|
unlock_page(page);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
err = 0;
|
|
|
|
done:
|
|
|
|
if (nr_underway == 0) {
|
|
|
|
/*
|
|
|
|
* The page was marked dirty, but the buffers were
|
|
|
|
* clean. Someone wrote them back by hand with
|
|
|
|
* ll_rw_block/submit_bh. A rare case.
|
|
|
|
*/
|
|
|
|
end_page_writeback(page);
|
2007-05-06 15:49:05 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* The page and buffer_heads can be released at any time from
|
|
|
|
* here on.
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
return err;
|
|
|
|
|
|
|
|
recover:
|
|
|
|
/*
|
|
|
|
* ENOSPC, or some other error. We may already have added some
|
|
|
|
* blocks to the file, so we need to write these out to avoid
|
|
|
|
* exposing stale data.
|
|
|
|
* The page is currently locked and not marked for writeback
|
|
|
|
*/
|
|
|
|
bh = head;
|
|
|
|
/* Recovery: lock and submit the mapped buffers */
|
|
|
|
do {
|
2008-07-11 17:27:31 -06:00
|
|
|
if (buffer_mapped(bh) && buffer_dirty(bh) &&
|
|
|
|
!buffer_delay(bh)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
lock_buffer(bh);
|
2009-04-15 11:22:38 -06:00
|
|
|
mark_buffer_async_write_endio(bh, handler);
|
2005-04-16 16:20:36 -06:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* The buffer may have been set dirty during
|
|
|
|
* attachment to a dirty page.
|
|
|
|
*/
|
|
|
|
clear_buffer_dirty(bh);
|
|
|
|
}
|
|
|
|
} while ((bh = bh->b_this_page) != head);
|
|
|
|
SetPageError(page);
|
|
|
|
BUG_ON(PageWriteback(page));
|
2007-05-08 01:23:27 -06:00
|
|
|
mapping_set_error(page->mapping, err);
|
2005-04-16 16:20:36 -06:00
|
|
|
set_page_writeback(page);
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
if (buffer_async_write(bh)) {
|
|
|
|
clear_buffer_dirty(bh);
|
2009-03-27 20:14:10 -06:00
|
|
|
submit_bh(write_op, bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
nr_underway++;
|
|
|
|
}
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
2007-02-20 14:57:54 -07:00
|
|
|
unlock_page(page);
|
2005-04-16 16:20:36 -06:00
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
2007-10-16 02:25:01 -06:00
|
|
|
/*
|
|
|
|
* If a page has any new buffers, zero them out here, and mark them uptodate
|
|
|
|
* and dirty so they'll be written out (in order to prevent uninitialised
|
|
|
|
* block data from leaking). And clear the new bit.
|
|
|
|
*/
|
|
|
|
void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
|
|
|
|
{
|
|
|
|
unsigned int block_start, block_end;
|
|
|
|
struct buffer_head *head, *bh;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
return;
|
|
|
|
|
|
|
|
bh = head = page_buffers(page);
|
|
|
|
block_start = 0;
|
|
|
|
do {
|
|
|
|
block_end = block_start + bh->b_size;
|
|
|
|
|
|
|
|
if (buffer_new(bh)) {
|
|
|
|
if (block_end > from && block_start < to) {
|
|
|
|
if (!PageUptodate(page)) {
|
|
|
|
unsigned start, size;
|
|
|
|
|
|
|
|
start = max(from, block_start);
|
|
|
|
size = min(to, block_end) - start;
|
|
|
|
|
2008-02-04 23:28:29 -07:00
|
|
|
zero_user(page, start, size);
|
2007-10-16 02:25:01 -06:00
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
clear_buffer_new(bh);
|
|
|
|
mark_buffer_dirty(bh);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
block_start = block_end;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(page_zero_new_buffers);
|
|
|
|
|
2010-10-06 02:47:23 -06:00
|
|
|
int __block_write_begin(struct page *page, loff_t pos, unsigned len,
|
2010-06-04 03:29:57 -06:00
|
|
|
get_block_t *get_block)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2010-10-06 02:47:23 -06:00
|
|
|
unsigned from = pos & (PAGE_CACHE_SIZE - 1);
|
|
|
|
unsigned to = from + len;
|
2010-06-04 03:29:57 -06:00
|
|
|
struct inode *inode = page->mapping->host;
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned block_start, block_end;
|
|
|
|
sector_t block;
|
|
|
|
int err = 0;
|
|
|
|
unsigned blocksize, bbits;
|
|
|
|
struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
BUG_ON(from > PAGE_CACHE_SIZE);
|
|
|
|
BUG_ON(to > PAGE_CACHE_SIZE);
|
|
|
|
BUG_ON(from > to);
|
|
|
|
|
|
|
|
blocksize = 1 << inode->i_blkbits;
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
create_empty_buffers(page, blocksize, 0);
|
|
|
|
head = page_buffers(page);
|
|
|
|
|
|
|
|
bbits = inode->i_blkbits;
|
|
|
|
block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
|
|
|
|
|
|
|
|
for(bh = head, block_start = 0; bh != head || !block_start;
|
|
|
|
block++, block_start=block_end, bh = bh->b_this_page) {
|
|
|
|
block_end = block_start + blocksize;
|
|
|
|
if (block_end <= from || block_start >= to) {
|
|
|
|
if (PageUptodate(page)) {
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (buffer_new(bh))
|
|
|
|
clear_buffer_new(bh);
|
|
|
|
if (!buffer_mapped(bh)) {
|
2006-03-26 02:38:00 -07:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-04-16 16:20:36 -06:00
|
|
|
err = get_block(inode, block, bh, 1);
|
|
|
|
if (err)
|
2005-05-05 17:15:45 -06:00
|
|
|
break;
|
2005-04-16 16:20:36 -06:00
|
|
|
if (buffer_new(bh)) {
|
|
|
|
unmap_underlying_metadata(bh->b_bdev,
|
|
|
|
bh->b_blocknr);
|
|
|
|
if (PageUptodate(page)) {
|
2007-10-16 02:25:00 -06:00
|
|
|
clear_buffer_new(bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
set_buffer_uptodate(bh);
|
2007-10-16 02:25:00 -06:00
|
|
|
mark_buffer_dirty(bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
continue;
|
|
|
|
}
|
2008-02-04 23:28:29 -07:00
|
|
|
if (block_end > to || block_start < from)
|
|
|
|
zero_user_segments(page,
|
|
|
|
to, block_end,
|
|
|
|
block_start, from);
|
2005-04-16 16:20:36 -06:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (PageUptodate(page)) {
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
|
2007-02-12 01:51:41 -07:00
|
|
|
!buffer_unwritten(bh) &&
|
2005-04-16 16:20:36 -06:00
|
|
|
(block_start < from || block_end > to)) {
|
|
|
|
ll_rw_block(READ, 1, &bh);
|
|
|
|
*wait_bh++=bh;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* If we issued read requests - let them complete.
|
|
|
|
*/
|
|
|
|
while(wait_bh > wait) {
|
|
|
|
wait_on_buffer(*--wait_bh);
|
|
|
|
if (!buffer_uptodate(*wait_bh))
|
2005-05-05 17:15:45 -06:00
|
|
|
err = -EIO;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2010-06-04 03:29:57 -06:00
|
|
|
if (unlikely(err)) {
|
2007-10-16 02:25:01 -06:00
|
|
|
page_zero_new_buffers(page, from, to);
|
2010-06-04 03:29:57 -06:00
|
|
|
ClearPageUptodate(page);
|
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
return err;
|
|
|
|
}
|
2010-10-06 02:47:23 -06:00
|
|
|
EXPORT_SYMBOL(__block_write_begin);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
static int __block_commit_write(struct inode *inode, struct page *page,
|
|
|
|
unsigned from, unsigned to)
|
|
|
|
{
|
|
|
|
unsigned block_start, block_end;
|
|
|
|
int partial = 0;
|
|
|
|
unsigned blocksize;
|
|
|
|
struct buffer_head *bh, *head;
|
|
|
|
|
|
|
|
blocksize = 1 << inode->i_blkbits;
|
|
|
|
|
|
|
|
for(bh = head = page_buffers(page), block_start = 0;
|
|
|
|
bh != head || !block_start;
|
|
|
|
block_start=block_end, bh = bh->b_this_page) {
|
|
|
|
block_end = block_start + blocksize;
|
|
|
|
if (block_end <= from || block_start >= to) {
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
partial = 1;
|
|
|
|
} else {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
mark_buffer_dirty(bh);
|
|
|
|
}
|
2007-10-16 02:25:01 -06:00
|
|
|
clear_buffer_new(bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If this is a partial write which happened to make all buffers
|
|
|
|
* uptodate then we can optimize away a bogus readpage() for
|
|
|
|
* the next read(). Here we 'discover' whether the page went
|
|
|
|
* uptodate as a result of this (potentially partial) write.
|
|
|
|
*/
|
|
|
|
if (!partial)
|
|
|
|
SetPageUptodate(page);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-10-16 02:25:01 -06:00
|
|
|
/*
|
2010-06-04 03:29:58 -06:00
|
|
|
* block_write_begin takes care of the basic task of block allocation and
|
|
|
|
* bringing partial write blocks uptodate first.
|
|
|
|
*
|
fs: introduce new truncate sequence
Introduce a new truncate calling sequence into fs/mm subsystems. Rather than
setattr > vmtruncate > truncate, have filesystems call their truncate sequence
from ->setattr if filesystem specific operations are required. vmtruncate is
deprecated, and truncate_pagecache and inode_newsize_ok helpers introduced
previously should be used.
simple_setattr is introduced for simple in-ram filesystems to implement
the new truncate sequence. Eventually all filesystems should be converted
to implement a setattr, and the default code in notify_change should go
away.
simple_setsize is also introduced to perform just the ATTR_SIZE portion
of simple_setattr (ie. changing i_size and trimming pagecache).
To implement the new truncate sequence:
- filesystem specific manipulations (eg freeing blocks) must be done in
the setattr method rather than ->truncate.
- vmtruncate can not be used by core code to trim blocks past i_size in
the event of write failure after allocation, so this must be performed
in the fs code.
- convert usage of helpers block_write_begin, nobh_write_begin,
cont_write_begin, and *blockdev_direct_IO* to use _newtrunc postfixed
variants. These avoid calling vmtruncate to trim blocks (see previous).
- inode_setattr should not be used. generic_setattr is a new function
to be used to copy simple attributes into the generic inode.
- make use of the better opportunity to handle errors with the new sequence.
Big problem with the previous calling sequence: the filesystem is not called
until i_size has already changed. This means it is not allowed to fail the
call, and also it does not know what the previous i_size was. Also, generic
code calling vmtruncate to truncate allocated blocks in case of error had
no good way to return a meaningful error (or, for example, atomically handle
block deallocation).
Cc: Christoph Hellwig <hch@lst.de>
Acked-by: Jan Kara <jack@suse.cz>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2010-05-26 09:05:33 -06:00
|
|
|
* The filesystem needs to handle block truncation upon failure.
|
2007-10-16 02:25:01 -06:00
|
|
|
*/
|
2010-06-04 03:29:58 -06:00
|
|
|
int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
|
|
|
|
unsigned flags, struct page **pagep, get_block_t *get_block)
|
2007-10-16 02:25:01 -06:00
|
|
|
{
|
2010-06-04 03:29:57 -06:00
|
|
|
pgoff_t index = pos >> PAGE_CACHE_SHIFT;
|
2007-10-16 02:25:01 -06:00
|
|
|
struct page *page;
|
2010-06-04 03:29:57 -06:00
|
|
|
int status;
|
2007-10-16 02:25:01 -06:00
|
|
|
|
2010-06-04 03:29:57 -06:00
|
|
|
page = grab_cache_page_write_begin(mapping, index, flags);
|
|
|
|
if (!page)
|
|
|
|
return -ENOMEM;
|
2007-10-16 02:25:01 -06:00
|
|
|
|
2010-06-04 03:29:57 -06:00
|
|
|
status = __block_write_begin(page, pos, len, get_block);
|
2007-10-16 02:25:01 -06:00
|
|
|
if (unlikely(status)) {
|
2010-06-04 03:29:57 -06:00
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
page = NULL;
|
2007-10-16 02:25:01 -06:00
|
|
|
}
|
|
|
|
|
2010-06-04 03:29:57 -06:00
|
|
|
*pagep = page;
|
2007-10-16 02:25:01 -06:00
|
|
|
return status;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_write_begin);
|
|
|
|
|
|
|
|
int block_write_end(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
|
|
struct page *page, void *fsdata)
|
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
unsigned start;
|
|
|
|
|
|
|
|
start = pos & (PAGE_CACHE_SIZE - 1);
|
|
|
|
|
|
|
|
if (unlikely(copied < len)) {
|
|
|
|
/*
|
|
|
|
* The buffers that were written will now be uptodate, so we
|
|
|
|
* don't have to worry about a readpage reading them and
|
|
|
|
* overwriting a partial write. However if we have encountered
|
|
|
|
* a short write and only partially written into a buffer, it
|
|
|
|
* will not be marked uptodate, so a readpage might come in and
|
|
|
|
* destroy our partial write.
|
|
|
|
*
|
|
|
|
* Do the simplest thing, and just treat any short write to a
|
|
|
|
* non uptodate page as a zero-length write, and force the
|
|
|
|
* caller to redo the whole thing.
|
|
|
|
*/
|
|
|
|
if (!PageUptodate(page))
|
|
|
|
copied = 0;
|
|
|
|
|
|
|
|
page_zero_new_buffers(page, start+copied, start+len);
|
|
|
|
}
|
|
|
|
flush_dcache_page(page);
|
|
|
|
|
|
|
|
/* This could be a short (even 0-length) commit */
|
|
|
|
__block_commit_write(inode, page, start, start+copied);
|
|
|
|
|
|
|
|
return copied;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_write_end);
|
|
|
|
|
|
|
|
int generic_write_end(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
|
|
struct page *page, void *fsdata)
|
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
2008-07-11 17:27:31 -06:00
|
|
|
int i_size_changed = 0;
|
2007-10-16 02:25:01 -06:00
|
|
|
|
|
|
|
copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* No need to use i_size_read() here, the i_size
|
|
|
|
* cannot change under us because we hold i_mutex.
|
|
|
|
*
|
|
|
|
* But it's important to update i_size while still holding page lock:
|
|
|
|
* page writeout could otherwise come in and zero beyond i_size.
|
|
|
|
*/
|
|
|
|
if (pos+copied > inode->i_size) {
|
|
|
|
i_size_write(inode, pos+copied);
|
2008-07-11 17:27:31 -06:00
|
|
|
i_size_changed = 1;
|
2007-10-16 02:25:01 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
|
2008-07-11 17:27:31 -06:00
|
|
|
/*
|
|
|
|
* Don't mark the inode dirty under page lock. First, it unnecessarily
|
|
|
|
* makes the holding time of page lock longer. Second, it forces lock
|
|
|
|
* ordering of page lock and transaction start for journaling
|
|
|
|
* filesystems.
|
|
|
|
*/
|
|
|
|
if (i_size_changed)
|
|
|
|
mark_inode_dirty(inode);
|
|
|
|
|
2007-10-16 02:25:01 -06:00
|
|
|
return copied;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(generic_write_end);
|
|
|
|
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-28 16:46:36 -06:00
|
|
|
/*
|
|
|
|
* block_is_partially_uptodate checks whether buffers within a page are
|
|
|
|
* uptodate or not.
|
|
|
|
*
|
|
|
|
* Returns true if all buffers which correspond to a file portion
|
|
|
|
* we want to read are uptodate.
|
|
|
|
*/
|
|
|
|
int block_is_partially_uptodate(struct page *page, read_descriptor_t *desc,
|
|
|
|
unsigned long from)
|
|
|
|
{
|
|
|
|
struct inode *inode = page->mapping->host;
|
|
|
|
unsigned block_start, block_end, blocksize;
|
|
|
|
unsigned to;
|
|
|
|
struct buffer_head *bh, *head;
|
|
|
|
int ret = 1;
|
|
|
|
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
blocksize = 1 << inode->i_blkbits;
|
|
|
|
to = min_t(unsigned, PAGE_CACHE_SIZE - from, desc->count);
|
|
|
|
to = from + to;
|
|
|
|
if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
head = page_buffers(page);
|
|
|
|
bh = head;
|
|
|
|
block_start = 0;
|
|
|
|
do {
|
|
|
|
block_end = block_start + blocksize;
|
|
|
|
if (block_end > from && block_start < to) {
|
|
|
|
if (!buffer_uptodate(bh)) {
|
|
|
|
ret = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (block_end >= to)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
block_start = block_end;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_is_partially_uptodate);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* Generic "read page" function for block devices that have the normal
|
|
|
|
* get_block functionality. This is most of the block device filesystems.
|
|
|
|
* Reads the page asynchronously --- the unlock_buffer() and
|
|
|
|
* set/clear_buffer_uptodate() functions propagate buffer state into the
|
|
|
|
* page struct once IO has completed.
|
|
|
|
*/
|
|
|
|
int block_read_full_page(struct page *page, get_block_t *get_block)
|
|
|
|
{
|
|
|
|
struct inode *inode = page->mapping->host;
|
|
|
|
sector_t iblock, lblock;
|
|
|
|
struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
|
|
|
|
unsigned int blocksize;
|
|
|
|
int nr, i;
|
|
|
|
int fully_mapped = 1;
|
|
|
|
|
2005-05-01 09:59:01 -06:00
|
|
|
BUG_ON(!PageLocked(page));
|
2005-04-16 16:20:36 -06:00
|
|
|
blocksize = 1 << inode->i_blkbits;
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
create_empty_buffers(page, blocksize, 0);
|
|
|
|
head = page_buffers(page);
|
|
|
|
|
|
|
|
iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
|
|
|
|
lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
|
|
|
|
bh = head;
|
|
|
|
nr = 0;
|
|
|
|
i = 0;
|
|
|
|
|
|
|
|
do {
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (!buffer_mapped(bh)) {
|
2005-05-16 22:53:49 -06:00
|
|
|
int err = 0;
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
fully_mapped = 0;
|
|
|
|
if (iblock < lblock) {
|
2006-03-26 02:38:00 -07:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-05-16 22:53:49 -06:00
|
|
|
err = get_block(inode, iblock, bh, 0);
|
|
|
|
if (err)
|
2005-04-16 16:20:36 -06:00
|
|
|
SetPageError(page);
|
|
|
|
}
|
|
|
|
if (!buffer_mapped(bh)) {
|
2008-02-04 23:28:29 -07:00
|
|
|
zero_user(page, i * blocksize, blocksize);
|
2005-05-16 22:53:49 -06:00
|
|
|
if (!err)
|
|
|
|
set_buffer_uptodate(bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* get_block() might have updated the buffer
|
|
|
|
* synchronously
|
|
|
|
*/
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
arr[nr++] = bh;
|
|
|
|
} while (i++, iblock++, (bh = bh->b_this_page) != head);
|
|
|
|
|
|
|
|
if (fully_mapped)
|
|
|
|
SetPageMappedToDisk(page);
|
|
|
|
|
|
|
|
if (!nr) {
|
|
|
|
/*
|
|
|
|
* All buffers are uptodate - we can set the page uptodate
|
|
|
|
* as well. But not if get_block() returned an error.
|
|
|
|
*/
|
|
|
|
if (!PageError(page))
|
|
|
|
SetPageUptodate(page);
|
|
|
|
unlock_page(page);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Stage two: lock the buffers */
|
|
|
|
for (i = 0; i < nr; i++) {
|
|
|
|
bh = arr[i];
|
|
|
|
lock_buffer(bh);
|
|
|
|
mark_buffer_async_read(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Stage 3: start the IO. Check for uptodateness
|
|
|
|
* inside the buffer lock in case another process reading
|
|
|
|
* the underlying blockdev brought it uptodate (the sct fix).
|
|
|
|
*/
|
|
|
|
for (i = 0; i < nr; i++) {
|
|
|
|
bh = arr[i];
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
end_buffer_async_read(bh, 1);
|
|
|
|
else
|
|
|
|
submit_bh(READ, bh);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(block_read_full_page);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* utility function for filesystems that need to do work on expanding
|
2007-10-16 02:25:07 -06:00
|
|
|
* truncates. Uses filesystem pagecache writes to allow the filesystem to
|
2005-04-16 16:20:36 -06:00
|
|
|
* deal with the hole.
|
|
|
|
*/
|
2007-10-16 02:25:07 -06:00
|
|
|
int generic_cont_expand_simple(struct inode *inode, loff_t size)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct address_space *mapping = inode->i_mapping;
|
|
|
|
struct page *page;
|
2007-10-16 02:25:07 -06:00
|
|
|
void *fsdata;
|
2005-04-16 16:20:36 -06:00
|
|
|
int err;
|
|
|
|
|
2009-08-20 10:35:06 -06:00
|
|
|
err = inode_newsize_ok(inode, size);
|
|
|
|
if (err)
|
2005-04-16 16:20:36 -06:00
|
|
|
goto out;
|
|
|
|
|
2007-10-16 02:25:07 -06:00
|
|
|
err = pagecache_write_begin(NULL, mapping, size, 0,
|
|
|
|
AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
|
|
|
|
&page, &fsdata);
|
|
|
|
if (err)
|
2006-01-08 02:02:13 -07:00
|
|
|
goto out;
|
|
|
|
|
2007-10-16 02:25:07 -06:00
|
|
|
err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
|
|
|
|
BUG_ON(err > 0);
|
2006-01-08 02:02:13 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(generic_cont_expand_simple);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-04-29 01:59:01 -06:00
|
|
|
static int cont_expand_zero(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, loff_t *bytes)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
unsigned blocksize = 1 << inode->i_blkbits;
|
2007-10-16 02:25:07 -06:00
|
|
|
struct page *page;
|
|
|
|
void *fsdata;
|
|
|
|
pgoff_t index, curidx;
|
|
|
|
loff_t curpos;
|
|
|
|
unsigned zerofrom, offset, len;
|
|
|
|
int err = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-16 02:25:07 -06:00
|
|
|
index = pos >> PAGE_CACHE_SHIFT;
|
|
|
|
offset = pos & ~PAGE_CACHE_MASK;
|
|
|
|
|
|
|
|
while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) {
|
|
|
|
zerofrom = curpos & ~PAGE_CACHE_MASK;
|
2005-04-16 16:20:36 -06:00
|
|
|
if (zerofrom & (blocksize-1)) {
|
|
|
|
*bytes |= (blocksize-1);
|
|
|
|
(*bytes)++;
|
|
|
|
}
|
2007-10-16 02:25:07 -06:00
|
|
|
len = PAGE_CACHE_SIZE - zerofrom;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-16 02:25:07 -06:00
|
|
|
err = pagecache_write_begin(file, mapping, curpos, len,
|
|
|
|
AOP_FLAG_UNINTERRUPTIBLE,
|
|
|
|
&page, &fsdata);
|
|
|
|
if (err)
|
|
|
|
goto out;
|
2008-02-04 23:28:29 -07:00
|
|
|
zero_user(page, zerofrom, len);
|
2007-10-16 02:25:07 -06:00
|
|
|
err = pagecache_write_end(file, mapping, curpos, len, len,
|
|
|
|
page, fsdata);
|
|
|
|
if (err < 0)
|
|
|
|
goto out;
|
|
|
|
BUG_ON(err != len);
|
|
|
|
err = 0;
|
2008-04-28 03:16:28 -06:00
|
|
|
|
|
|
|
balance_dirty_pages_ratelimited(mapping);
|
2007-10-16 02:25:07 -06:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-16 02:25:07 -06:00
|
|
|
/* page covers the boundary, find the boundary offset */
|
|
|
|
if (index == curidx) {
|
|
|
|
zerofrom = curpos & ~PAGE_CACHE_MASK;
|
2005-04-16 16:20:36 -06:00
|
|
|
/* if we will expand the thing last block will be filled */
|
2007-10-16 02:25:07 -06:00
|
|
|
if (offset <= zerofrom) {
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (zerofrom & (blocksize-1)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
*bytes |= (blocksize-1);
|
|
|
|
(*bytes)++;
|
|
|
|
}
|
2007-10-16 02:25:07 -06:00
|
|
|
len = offset - zerofrom;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-16 02:25:07 -06:00
|
|
|
err = pagecache_write_begin(file, mapping, curpos, len,
|
|
|
|
AOP_FLAG_UNINTERRUPTIBLE,
|
|
|
|
&page, &fsdata);
|
|
|
|
if (err)
|
|
|
|
goto out;
|
2008-02-04 23:28:29 -07:00
|
|
|
zero_user(page, zerofrom, len);
|
2007-10-16 02:25:07 -06:00
|
|
|
err = pagecache_write_end(file, mapping, curpos, len, len,
|
|
|
|
page, fsdata);
|
|
|
|
if (err < 0)
|
|
|
|
goto out;
|
|
|
|
BUG_ON(err != len);
|
|
|
|
err = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2007-10-16 02:25:07 -06:00
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For moronic filesystems that do not allow holes in file.
|
|
|
|
* We may have to extend the file.
|
|
|
|
*/
|
2010-06-04 03:29:55 -06:00
|
|
|
int cont_write_begin(struct file *file, struct address_space *mapping,
|
2007-10-16 02:25:07 -06:00
|
|
|
loff_t pos, unsigned len, unsigned flags,
|
|
|
|
struct page **pagep, void **fsdata,
|
|
|
|
get_block_t *get_block, loff_t *bytes)
|
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
unsigned blocksize = 1 << inode->i_blkbits;
|
|
|
|
unsigned zerofrom;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = cont_expand_zero(file, mapping, pos, bytes);
|
|
|
|
if (err)
|
2010-06-04 03:29:58 -06:00
|
|
|
return err;
|
2007-10-16 02:25:07 -06:00
|
|
|
|
|
|
|
zerofrom = *bytes & ~PAGE_CACHE_MASK;
|
|
|
|
if (pos+len > *bytes && zerofrom & (blocksize-1)) {
|
|
|
|
*bytes |= (blocksize-1);
|
|
|
|
(*bytes)++;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2010-06-04 03:29:58 -06:00
|
|
|
return block_write_begin(mapping, pos, len, flags, pagep, get_block);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(cont_write_begin);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
int block_commit_write(struct page *page, unsigned from, unsigned to)
|
|
|
|
{
|
|
|
|
struct inode *inode = page->mapping->host;
|
|
|
|
__block_commit_write(inode,page,from,to);
|
|
|
|
return 0;
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(block_commit_write);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-07-19 01:39:55 -06:00
|
|
|
/*
|
|
|
|
* block_page_mkwrite() is not allowed to change the file size as it gets
|
|
|
|
* called from a page fault handler when a page is first dirtied. Hence we must
|
|
|
|
* be careful to check for EOF conditions here. We set the page up correctly
|
|
|
|
* for a written page which means we get ENOSPC checking when writing into
|
|
|
|
* holes and correct delalloc and unwritten extent mapping on filesystems that
|
|
|
|
* support these features.
|
|
|
|
*
|
|
|
|
* We are not allowed to take the i_mutex here so we have to play games to
|
|
|
|
* protect against truncate races as the page could now be beyond EOF. Because
|
fs: introduce new truncate sequence
Introduce a new truncate calling sequence into fs/mm subsystems. Rather than
setattr > vmtruncate > truncate, have filesystems call their truncate sequence
from ->setattr if filesystem specific operations are required. vmtruncate is
deprecated, and truncate_pagecache and inode_newsize_ok helpers introduced
previously should be used.
simple_setattr is introduced for simple in-ram filesystems to implement
the new truncate sequence. Eventually all filesystems should be converted
to implement a setattr, and the default code in notify_change should go
away.
simple_setsize is also introduced to perform just the ATTR_SIZE portion
of simple_setattr (ie. changing i_size and trimming pagecache).
To implement the new truncate sequence:
- filesystem specific manipulations (eg freeing blocks) must be done in
the setattr method rather than ->truncate.
- vmtruncate can not be used by core code to trim blocks past i_size in
the event of write failure after allocation, so this must be performed
in the fs code.
- convert usage of helpers block_write_begin, nobh_write_begin,
cont_write_begin, and *blockdev_direct_IO* to use _newtrunc postfixed
variants. These avoid calling vmtruncate to trim blocks (see previous).
- inode_setattr should not be used. generic_setattr is a new function
to be used to copy simple attributes into the generic inode.
- make use of the better opportunity to handle errors with the new sequence.
Big problem with the previous calling sequence: the filesystem is not called
until i_size has already changed. This means it is not allowed to fail the
call, and also it does not know what the previous i_size was. Also, generic
code calling vmtruncate to truncate allocated blocks in case of error had
no good way to return a meaningful error (or, for example, atomically handle
block deallocation).
Cc: Christoph Hellwig <hch@lst.de>
Acked-by: Jan Kara <jack@suse.cz>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2010-05-26 09:05:33 -06:00
|
|
|
* truncate writes the inode size before removing pages, once we have the
|
2007-07-19 01:39:55 -06:00
|
|
|
* page lock we can determine safely if the page is beyond EOF. If it is not
|
|
|
|
* beyond EOF, then the page is guaranteed safe against truncation until we
|
|
|
|
* unlock the page.
|
|
|
|
*/
|
|
|
|
int
|
2009-03-31 16:23:21 -06:00
|
|
|
block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
|
2007-07-19 01:39:55 -06:00
|
|
|
get_block_t get_block)
|
|
|
|
{
|
2009-03-31 16:23:21 -06:00
|
|
|
struct page *page = vmf->page;
|
2007-07-19 01:39:55 -06:00
|
|
|
struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
|
|
|
|
unsigned long end;
|
|
|
|
loff_t size;
|
2009-03-31 16:23:23 -06:00
|
|
|
int ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
|
2007-07-19 01:39:55 -06:00
|
|
|
|
|
|
|
lock_page(page);
|
|
|
|
size = i_size_read(inode);
|
|
|
|
if ((page->mapping != inode->i_mapping) ||
|
2007-07-20 01:31:45 -06:00
|
|
|
(page_offset(page) > size)) {
|
2007-07-19 01:39:55 -06:00
|
|
|
/* page got truncated out from underneath us */
|
mm: close page_mkwrite races
Change page_mkwrite to allow implementations to return with the page
locked, and also change it's callers (in page fault paths) to hold the
lock until the page is marked dirty. This allows the filesystem to have
full control of page dirtying events coming from the VM.
Rather than simply hold the page locked over the page_mkwrite call, we
call page_mkwrite with the page unlocked and allow callers to return with
it locked, so filesystems can avoid LOR conditions with page lock.
The problem with the current scheme is this: a filesystem that wants to
associate some metadata with a page as long as the page is dirty, will
perform this manipulation in its ->page_mkwrite. It currently then must
return with the page unlocked and may not hold any other locks (according
to existing page_mkwrite convention).
In this window, the VM could write out the page, clearing page-dirty. The
filesystem has no good way to detect that a dirty pte is about to be
attached, so it will happily write out the page, at which point, the
filesystem may manipulate the metadata to reflect that the page is no
longer dirty.
It is not always possible to perform the required metadata manipulation in
->set_page_dirty, because that function cannot block or fail. The
filesystem may need to allocate some data structure, for example.
And the VM cannot mark the pte dirty before page_mkwrite, because
page_mkwrite is allowed to fail, so we must not allow any window where the
page could be written to if page_mkwrite does fail.
This solution of holding the page locked over the 3 critical operations
(page_mkwrite, setting the pte dirty, and finally setting the page dirty)
closes out races nicely, preventing page cleaning for writeout being
initiated in that window. This provides the filesystem with a strong
synchronisation against the VM here.
- Sage needs this race closed for ceph filesystem.
- Trond for NFS (http://bugzilla.kernel.org/show_bug.cgi?id=12913).
- I need it for fsblock.
- I suspect other filesystems may need it too (eg. btrfs).
- I have converted buffer.c to the new locking. Even simple block allocation
under dirty pages might be susceptible to i_size changing under partial page
at the end of file (we also have a buffer.c-side problem here, but it cannot
be fixed properly without this patch).
- Other filesystems (eg. NFS, maybe btrfs) will need to change their
page_mkwrite functions themselves.
[ This also moves page_mkwrite another step closer to fault, which should
eventually allow page_mkwrite to be moved into ->fault, and thus avoiding a
filesystem calldown and page lock/unlock cycle in __do_fault. ]
[akpm@linux-foundation.org: fix derefs of NULL ->mapping]
Cc: Sage Weil <sage@newdream.net>
Cc: Trond Myklebust <trond.myklebust@fys.uio.no>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: Valdis Kletnieks <Valdis.Kletnieks@vt.edu>
Cc: <stable@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-30 16:08:16 -06:00
|
|
|
unlock_page(page);
|
|
|
|
goto out;
|
2007-07-19 01:39:55 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/* page is wholly or partially inside EOF */
|
|
|
|
if (((page->index + 1) << PAGE_CACHE_SHIFT) > size)
|
|
|
|
end = size & ~PAGE_CACHE_MASK;
|
|
|
|
else
|
|
|
|
end = PAGE_CACHE_SIZE;
|
|
|
|
|
2010-10-06 02:47:23 -06:00
|
|
|
ret = __block_write_begin(page, 0, end, get_block);
|
2007-07-19 01:39:55 -06:00
|
|
|
if (!ret)
|
|
|
|
ret = block_commit_write(page, 0, end);
|
|
|
|
|
2009-03-31 16:23:23 -06:00
|
|
|
if (unlikely(ret)) {
|
mm: close page_mkwrite races
Change page_mkwrite to allow implementations to return with the page
locked, and also change it's callers (in page fault paths) to hold the
lock until the page is marked dirty. This allows the filesystem to have
full control of page dirtying events coming from the VM.
Rather than simply hold the page locked over the page_mkwrite call, we
call page_mkwrite with the page unlocked and allow callers to return with
it locked, so filesystems can avoid LOR conditions with page lock.
The problem with the current scheme is this: a filesystem that wants to
associate some metadata with a page as long as the page is dirty, will
perform this manipulation in its ->page_mkwrite. It currently then must
return with the page unlocked and may not hold any other locks (according
to existing page_mkwrite convention).
In this window, the VM could write out the page, clearing page-dirty. The
filesystem has no good way to detect that a dirty pte is about to be
attached, so it will happily write out the page, at which point, the
filesystem may manipulate the metadata to reflect that the page is no
longer dirty.
It is not always possible to perform the required metadata manipulation in
->set_page_dirty, because that function cannot block or fail. The
filesystem may need to allocate some data structure, for example.
And the VM cannot mark the pte dirty before page_mkwrite, because
page_mkwrite is allowed to fail, so we must not allow any window where the
page could be written to if page_mkwrite does fail.
This solution of holding the page locked over the 3 critical operations
(page_mkwrite, setting the pte dirty, and finally setting the page dirty)
closes out races nicely, preventing page cleaning for writeout being
initiated in that window. This provides the filesystem with a strong
synchronisation against the VM here.
- Sage needs this race closed for ceph filesystem.
- Trond for NFS (http://bugzilla.kernel.org/show_bug.cgi?id=12913).
- I need it for fsblock.
- I suspect other filesystems may need it too (eg. btrfs).
- I have converted buffer.c to the new locking. Even simple block allocation
under dirty pages might be susceptible to i_size changing under partial page
at the end of file (we also have a buffer.c-side problem here, but it cannot
be fixed properly without this patch).
- Other filesystems (eg. NFS, maybe btrfs) will need to change their
page_mkwrite functions themselves.
[ This also moves page_mkwrite another step closer to fault, which should
eventually allow page_mkwrite to be moved into ->fault, and thus avoiding a
filesystem calldown and page lock/unlock cycle in __do_fault. ]
[akpm@linux-foundation.org: fix derefs of NULL ->mapping]
Cc: Sage Weil <sage@newdream.net>
Cc: Trond Myklebust <trond.myklebust@fys.uio.no>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: Valdis Kletnieks <Valdis.Kletnieks@vt.edu>
Cc: <stable@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-30 16:08:16 -06:00
|
|
|
unlock_page(page);
|
2009-03-31 16:23:23 -06:00
|
|
|
if (ret == -ENOMEM)
|
|
|
|
ret = VM_FAULT_OOM;
|
|
|
|
else /* -ENOSPC, -EIO, etc */
|
|
|
|
ret = VM_FAULT_SIGBUS;
|
mm: close page_mkwrite races
Change page_mkwrite to allow implementations to return with the page
locked, and also change it's callers (in page fault paths) to hold the
lock until the page is marked dirty. This allows the filesystem to have
full control of page dirtying events coming from the VM.
Rather than simply hold the page locked over the page_mkwrite call, we
call page_mkwrite with the page unlocked and allow callers to return with
it locked, so filesystems can avoid LOR conditions with page lock.
The problem with the current scheme is this: a filesystem that wants to
associate some metadata with a page as long as the page is dirty, will
perform this manipulation in its ->page_mkwrite. It currently then must
return with the page unlocked and may not hold any other locks (according
to existing page_mkwrite convention).
In this window, the VM could write out the page, clearing page-dirty. The
filesystem has no good way to detect that a dirty pte is about to be
attached, so it will happily write out the page, at which point, the
filesystem may manipulate the metadata to reflect that the page is no
longer dirty.
It is not always possible to perform the required metadata manipulation in
->set_page_dirty, because that function cannot block or fail. The
filesystem may need to allocate some data structure, for example.
And the VM cannot mark the pte dirty before page_mkwrite, because
page_mkwrite is allowed to fail, so we must not allow any window where the
page could be written to if page_mkwrite does fail.
This solution of holding the page locked over the 3 critical operations
(page_mkwrite, setting the pte dirty, and finally setting the page dirty)
closes out races nicely, preventing page cleaning for writeout being
initiated in that window. This provides the filesystem with a strong
synchronisation against the VM here.
- Sage needs this race closed for ceph filesystem.
- Trond for NFS (http://bugzilla.kernel.org/show_bug.cgi?id=12913).
- I need it for fsblock.
- I suspect other filesystems may need it too (eg. btrfs).
- I have converted buffer.c to the new locking. Even simple block allocation
under dirty pages might be susceptible to i_size changing under partial page
at the end of file (we also have a buffer.c-side problem here, but it cannot
be fixed properly without this patch).
- Other filesystems (eg. NFS, maybe btrfs) will need to change their
page_mkwrite functions themselves.
[ This also moves page_mkwrite another step closer to fault, which should
eventually allow page_mkwrite to be moved into ->fault, and thus avoiding a
filesystem calldown and page lock/unlock cycle in __do_fault. ]
[akpm@linux-foundation.org: fix derefs of NULL ->mapping]
Cc: Sage Weil <sage@newdream.net>
Cc: Trond Myklebust <trond.myklebust@fys.uio.no>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: Valdis Kletnieks <Valdis.Kletnieks@vt.edu>
Cc: <stable@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-30 16:08:16 -06:00
|
|
|
} else
|
|
|
|
ret = VM_FAULT_LOCKED;
|
2009-03-31 16:23:21 -06:00
|
|
|
|
mm: close page_mkwrite races
Change page_mkwrite to allow implementations to return with the page
locked, and also change it's callers (in page fault paths) to hold the
lock until the page is marked dirty. This allows the filesystem to have
full control of page dirtying events coming from the VM.
Rather than simply hold the page locked over the page_mkwrite call, we
call page_mkwrite with the page unlocked and allow callers to return with
it locked, so filesystems can avoid LOR conditions with page lock.
The problem with the current scheme is this: a filesystem that wants to
associate some metadata with a page as long as the page is dirty, will
perform this manipulation in its ->page_mkwrite. It currently then must
return with the page unlocked and may not hold any other locks (according
to existing page_mkwrite convention).
In this window, the VM could write out the page, clearing page-dirty. The
filesystem has no good way to detect that a dirty pte is about to be
attached, so it will happily write out the page, at which point, the
filesystem may manipulate the metadata to reflect that the page is no
longer dirty.
It is not always possible to perform the required metadata manipulation in
->set_page_dirty, because that function cannot block or fail. The
filesystem may need to allocate some data structure, for example.
And the VM cannot mark the pte dirty before page_mkwrite, because
page_mkwrite is allowed to fail, so we must not allow any window where the
page could be written to if page_mkwrite does fail.
This solution of holding the page locked over the 3 critical operations
(page_mkwrite, setting the pte dirty, and finally setting the page dirty)
closes out races nicely, preventing page cleaning for writeout being
initiated in that window. This provides the filesystem with a strong
synchronisation against the VM here.
- Sage needs this race closed for ceph filesystem.
- Trond for NFS (http://bugzilla.kernel.org/show_bug.cgi?id=12913).
- I need it for fsblock.
- I suspect other filesystems may need it too (eg. btrfs).
- I have converted buffer.c to the new locking. Even simple block allocation
under dirty pages might be susceptible to i_size changing under partial page
at the end of file (we also have a buffer.c-side problem here, but it cannot
be fixed properly without this patch).
- Other filesystems (eg. NFS, maybe btrfs) will need to change their
page_mkwrite functions themselves.
[ This also moves page_mkwrite another step closer to fault, which should
eventually allow page_mkwrite to be moved into ->fault, and thus avoiding a
filesystem calldown and page lock/unlock cycle in __do_fault. ]
[akpm@linux-foundation.org: fix derefs of NULL ->mapping]
Cc: Sage Weil <sage@newdream.net>
Cc: Trond Myklebust <trond.myklebust@fys.uio.no>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: Valdis Kletnieks <Valdis.Kletnieks@vt.edu>
Cc: <stable@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-30 16:08:16 -06:00
|
|
|
out:
|
2007-07-19 01:39:55 -06:00
|
|
|
return ret;
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(block_page_mkwrite);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
2007-10-16 02:25:25 -06:00
|
|
|
* nobh_write_begin()'s prereads are special: the buffer_heads are freed
|
2005-04-16 16:20:36 -06:00
|
|
|
* immediately, while under the page lock. So it needs a special end_io
|
|
|
|
* handler which does not touch the bh after unlocking it.
|
|
|
|
*/
|
|
|
|
static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
2007-10-16 02:24:47 -06:00
|
|
|
__end_buffer_read_notouch(bh, uptodate);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2007-10-16 02:25:25 -06:00
|
|
|
/*
|
|
|
|
* Attach the singly-linked list of buffers created by nobh_write_begin, to
|
|
|
|
* the page (converting it to circular linked list and taking care of page
|
|
|
|
* dirty races).
|
|
|
|
*/
|
|
|
|
static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
|
|
|
|
spin_lock(&page->mapping->private_lock);
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
if (PageDirty(page))
|
|
|
|
set_buffer_dirty(bh);
|
|
|
|
if (!bh->b_this_page)
|
|
|
|
bh->b_this_page = head;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
attach_page_buffers(page, head);
|
|
|
|
spin_unlock(&page->mapping->private_lock);
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
2010-06-04 03:29:54 -06:00
|
|
|
* On entry, the page is fully not uptodate.
|
|
|
|
* On exit the page is fully uptodate in the areas outside (from,to)
|
fs: introduce new truncate sequence
Introduce a new truncate calling sequence into fs/mm subsystems. Rather than
setattr > vmtruncate > truncate, have filesystems call their truncate sequence
from ->setattr if filesystem specific operations are required. vmtruncate is
deprecated, and truncate_pagecache and inode_newsize_ok helpers introduced
previously should be used.
simple_setattr is introduced for simple in-ram filesystems to implement
the new truncate sequence. Eventually all filesystems should be converted
to implement a setattr, and the default code in notify_change should go
away.
simple_setsize is also introduced to perform just the ATTR_SIZE portion
of simple_setattr (ie. changing i_size and trimming pagecache).
To implement the new truncate sequence:
- filesystem specific manipulations (eg freeing blocks) must be done in
the setattr method rather than ->truncate.
- vmtruncate can not be used by core code to trim blocks past i_size in
the event of write failure after allocation, so this must be performed
in the fs code.
- convert usage of helpers block_write_begin, nobh_write_begin,
cont_write_begin, and *blockdev_direct_IO* to use _newtrunc postfixed
variants. These avoid calling vmtruncate to trim blocks (see previous).
- inode_setattr should not be used. generic_setattr is a new function
to be used to copy simple attributes into the generic inode.
- make use of the better opportunity to handle errors with the new sequence.
Big problem with the previous calling sequence: the filesystem is not called
until i_size has already changed. This means it is not allowed to fail the
call, and also it does not know what the previous i_size was. Also, generic
code calling vmtruncate to truncate allocated blocks in case of error had
no good way to return a meaningful error (or, for example, atomically handle
block deallocation).
Cc: Christoph Hellwig <hch@lst.de>
Acked-by: Jan Kara <jack@suse.cz>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2010-05-26 09:05:33 -06:00
|
|
|
* The filesystem needs to handle block truncation upon failure.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2010-06-04 03:29:54 -06:00
|
|
|
int nobh_write_begin(struct address_space *mapping,
|
2007-10-16 02:25:25 -06:00
|
|
|
loff_t pos, unsigned len, unsigned flags,
|
|
|
|
struct page **pagep, void **fsdata,
|
2005-04-16 16:20:36 -06:00
|
|
|
get_block_t *get_block)
|
|
|
|
{
|
2007-10-16 02:25:25 -06:00
|
|
|
struct inode *inode = mapping->host;
|
2005-04-16 16:20:36 -06:00
|
|
|
const unsigned blkbits = inode->i_blkbits;
|
|
|
|
const unsigned blocksize = 1 << blkbits;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
struct buffer_head *head, *bh;
|
2007-10-16 02:25:25 -06:00
|
|
|
struct page *page;
|
|
|
|
pgoff_t index;
|
|
|
|
unsigned from, to;
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned block_in_page;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
unsigned block_start, block_end;
|
2005-04-16 16:20:36 -06:00
|
|
|
sector_t block_in_file;
|
|
|
|
int nr_reads = 0;
|
|
|
|
int ret = 0;
|
|
|
|
int is_mapped_to_disk = 1;
|
|
|
|
|
2007-10-16 02:25:25 -06:00
|
|
|
index = pos >> PAGE_CACHE_SHIFT;
|
|
|
|
from = pos & (PAGE_CACHE_SIZE - 1);
|
|
|
|
to = from + len;
|
|
|
|
|
fs: symlink write_begin allocation context fix
With the write_begin/write_end aops, page_symlink was broken because it
could no longer pass a GFP_NOFS type mask into the point where the
allocations happened. They are done in write_begin, which would always
assume that the filesystem can be entered from reclaim. This bug could
cause filesystem deadlocks.
The funny thing with having a gfp_t mask there is that it doesn't really
allow the caller to arbitrarily tinker with the context in which it can be
called. It couldn't ever be GFP_ATOMIC, for example, because it needs to
take the page lock. The only thing any callers care about is __GFP_FS
anyway, so turn that into a single flag.
Add a new flag for write_begin, AOP_FLAG_NOFS. Filesystems can now act on
this flag in their write_begin function. Change __grab_cache_page to
accept a nofs argument as well, to honour that flag (while we're there,
change the name to grab_cache_page_write_begin which is more instructive
and does away with random leading underscores).
This is really a more flexible way to go in the end anyway -- if a
filesystem happens to want any extra allocations aside from the pagecache
ones in ints write_begin function, it may now use GFP_KERNEL (rather than
GFP_NOFS) for common case allocations (eg. ocfs2_alloc_write_ctxt, for a
random example).
[kosaki.motohiro@jp.fujitsu.com: fix ubifs]
[kosaki.motohiro@jp.fujitsu.com: fix fuse]
Signed-off-by: Nick Piggin <npiggin@suse.de>
Reviewed-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: <stable@kernel.org> [2.6.28.x]
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
[ Cleaned up the calling convention: just pass in the AOP flags
untouched to the grab_cache_page_write_begin() function. That
just simplifies everybody, and may even allow future expansion of the
logic. - Linus ]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-01-04 13:00:53 -07:00
|
|
|
page = grab_cache_page_write_begin(mapping, index, flags);
|
2007-10-16 02:25:25 -06:00
|
|
|
if (!page)
|
|
|
|
return -ENOMEM;
|
|
|
|
*pagep = page;
|
|
|
|
*fsdata = NULL;
|
|
|
|
|
|
|
|
if (page_has_buffers(page)) {
|
2010-10-25 00:01:12 -06:00
|
|
|
ret = __block_write_begin(page, pos, len, get_block);
|
|
|
|
if (unlikely(ret))
|
|
|
|
goto out_release;
|
|
|
|
return ret;
|
2007-10-16 02:25:25 -06:00
|
|
|
}
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
if (PageMappedToDisk(page))
|
|
|
|
return 0;
|
|
|
|
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
/*
|
|
|
|
* Allocate buffers so that we can keep track of state, and potentially
|
|
|
|
* attach them to the page if an error occurs. In the common case of
|
|
|
|
* no error, they will just be freed again without ever being attached
|
|
|
|
* to the page (which is all OK, because we're under the page lock).
|
|
|
|
*
|
|
|
|
* Be careful: the buffer linked list is a NULL terminated one, rather
|
|
|
|
* than the circular one we're used to.
|
|
|
|
*/
|
|
|
|
head = alloc_page_buffers(page, blocksize, 0);
|
2007-10-16 02:25:25 -06:00
|
|
|
if (!head) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto out_release;
|
|
|
|
}
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We loop across all blocks in the page, whether or not they are
|
|
|
|
* part of the affected region. This is so we can discover if the
|
|
|
|
* page is fully mapped-to-disk.
|
|
|
|
*/
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
for (block_start = 0, block_in_page = 0, bh = head;
|
2005-04-16 16:20:36 -06:00
|
|
|
block_start < PAGE_CACHE_SIZE;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
|
2005-04-16 16:20:36 -06:00
|
|
|
int create;
|
|
|
|
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
block_end = block_start + blocksize;
|
|
|
|
bh->b_state = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
create = 1;
|
|
|
|
if (block_start >= to)
|
|
|
|
create = 0;
|
|
|
|
ret = get_block(inode, block_in_file + block_in_page,
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
bh, create);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (ret)
|
|
|
|
goto failed;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
if (!buffer_mapped(bh))
|
2005-04-16 16:20:36 -06:00
|
|
|
is_mapped_to_disk = 0;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
if (buffer_new(bh))
|
|
|
|
unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
|
|
|
|
if (PageUptodate(page)) {
|
|
|
|
set_buffer_uptodate(bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
continue;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
}
|
|
|
|
if (buffer_new(bh) || !buffer_mapped(bh)) {
|
2008-02-04 23:28:29 -07:00
|
|
|
zero_user_segments(page, block_start, from,
|
|
|
|
to, block_end);
|
2005-04-16 16:20:36 -06:00
|
|
|
continue;
|
|
|
|
}
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
if (buffer_uptodate(bh))
|
2005-04-16 16:20:36 -06:00
|
|
|
continue; /* reiserfs does this */
|
|
|
|
if (block_start < from || block_end > to) {
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
lock_buffer(bh);
|
|
|
|
bh->b_end_io = end_buffer_read_nobh;
|
|
|
|
submit_bh(READ, bh);
|
|
|
|
nr_reads++;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (nr_reads) {
|
|
|
|
/*
|
|
|
|
* The page is locked, so these buffers are protected from
|
|
|
|
* any VM or truncate activity. Hence we don't need to care
|
|
|
|
* for the buffer_head refcounts.
|
|
|
|
*/
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
for (bh = head; bh; bh = bh->b_this_page) {
|
2005-04-16 16:20:36 -06:00
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
ret = -EIO;
|
|
|
|
}
|
|
|
|
if (ret)
|
|
|
|
goto failed;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (is_mapped_to_disk)
|
|
|
|
SetPageMappedToDisk(page);
|
|
|
|
|
2007-10-16 02:25:25 -06:00
|
|
|
*fsdata = head; /* to be released by nobh_write_end */
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
failed:
|
2007-10-16 02:25:25 -06:00
|
|
|
BUG_ON(!ret);
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
* Error recovery is a bit difficult. We need to zero out blocks that
|
|
|
|
* were newly allocated, and dirty them to ensure they get written out.
|
|
|
|
* Buffers need to be attached to the page at this point, otherwise
|
|
|
|
* the handling of potential IO errors during writeout would be hard
|
|
|
|
* (could try doing synchronous writeout, but what if that fails too?)
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2007-10-16 02:25:25 -06:00
|
|
|
attach_nobh_buffers(page, head);
|
|
|
|
page_zero_new_buffers(page, from, to);
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
|
2007-10-16 02:25:25 -06:00
|
|
|
out_release:
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
*pagep = NULL;
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
|
fs: introduce new truncate sequence
Introduce a new truncate calling sequence into fs/mm subsystems. Rather than
setattr > vmtruncate > truncate, have filesystems call their truncate sequence
from ->setattr if filesystem specific operations are required. vmtruncate is
deprecated, and truncate_pagecache and inode_newsize_ok helpers introduced
previously should be used.
simple_setattr is introduced for simple in-ram filesystems to implement
the new truncate sequence. Eventually all filesystems should be converted
to implement a setattr, and the default code in notify_change should go
away.
simple_setsize is also introduced to perform just the ATTR_SIZE portion
of simple_setattr (ie. changing i_size and trimming pagecache).
To implement the new truncate sequence:
- filesystem specific manipulations (eg freeing blocks) must be done in
the setattr method rather than ->truncate.
- vmtruncate can not be used by core code to trim blocks past i_size in
the event of write failure after allocation, so this must be performed
in the fs code.
- convert usage of helpers block_write_begin, nobh_write_begin,
cont_write_begin, and *blockdev_direct_IO* to use _newtrunc postfixed
variants. These avoid calling vmtruncate to trim blocks (see previous).
- inode_setattr should not be used. generic_setattr is a new function
to be used to copy simple attributes into the generic inode.
- make use of the better opportunity to handle errors with the new sequence.
Big problem with the previous calling sequence: the filesystem is not called
until i_size has already changed. This means it is not allowed to fail the
call, and also it does not know what the previous i_size was. Also, generic
code calling vmtruncate to truncate allocated blocks in case of error had
no good way to return a meaningful error (or, for example, atomically handle
block deallocation).
Cc: Christoph Hellwig <hch@lst.de>
Acked-by: Jan Kara <jack@suse.cz>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2010-05-26 09:05:33 -06:00
|
|
|
return ret;
|
|
|
|
}
|
2007-10-16 02:25:25 -06:00
|
|
|
EXPORT_SYMBOL(nobh_write_begin);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-16 02:25:25 -06:00
|
|
|
int nobh_write_end(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
|
|
struct page *page, void *fsdata)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct inode *inode = page->mapping->host;
|
2007-10-20 22:57:41 -06:00
|
|
|
struct buffer_head *head = fsdata;
|
2007-10-16 02:25:25 -06:00
|
|
|
struct buffer_head *bh;
|
vfs: fix data leak in nobh_write_end()
Current nobh_write_end() implementation ignore partial writes(copied < len)
case if page was fully mapped and simply mark page as Uptodate, which is
totally wrong because area [pos+copied, pos+len) wasn't updated explicitly in
previous write_begin call. It simply contains garbage from pagecache and
result in data leakage.
#TEST_CASE_BEGIN:
~~~~~~~~~~~~~~~~
In fact issue triggered by classical testcase
open("/mnt/test", O_RDWR|O_CREAT|O_TRUNC, 0666) = 3
ftruncate(3, 409600) = 0
writev(3, [{"a", 1}, {NULL, 4095}], 2) = 1
##TESTCASE_SOURCE:
~~~~~~~~~~~~~~~~~
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <errno.h>
int main(int argc, char **argv)
{
int fd, ret;
void* p;
struct iovec iov[2];
fd = open(argv[1], O_RDWR|O_CREAT|O_TRUNC, 0666);
ftruncate(fd, 409600);
iov[0].iov_base="a";
iov[0].iov_len=1;
iov[1].iov_base=NULL;
iov[1].iov_len=4096;
ret = writev(fd, iov, sizeof(iov)/sizeof(struct iovec));
printf("writev = %d, err = %d\n", ret, errno);
return 0;
}
##TESTCASE RESULT:
~~~~~~~~~~~~~~~~~~
[root@ts63 ~]# mount | grep mnt2
/dev/mapper/test on /mnt2 type ext2 (rw,nobh)
[root@ts63 ~]# /tmp/writev /mnt2/test
writev = 1, err = 0
[root@ts63 ~]# hexdump -C /mnt2/test
00000000 61 65 62 6f 6f 74 00 00 f0 b9 b4 59 3a 00 00 00 |aeboot.....Y:...|
00000010 20 00 00 00 00 00 00 00 21 00 00 00 00 00 00 00 | .......!.......|
00000020 df df df df df df df df df df df df df df df df |................|
00000030 3a 00 00 00 2a 00 00 00 21 00 00 00 00 00 00 00 |:...*...!.......|
00000040 60 c0 8c 00 00 00 00 00 40 4a 8d 00 00 00 00 00 |`.......@J......|
00000050 00 00 00 00 00 00 00 00 41 00 00 00 00 00 00 00 |........A.......|
00000060 74 69 6d 65 20 64 64 20 69 66 3d 2f 64 65 76 2f |time dd if=/dev/|
00000070 6c 6f 6f 70 30 20 20 6f 66 3d 2f 64 65 76 2f 6e |loop0 of=/dev/n|
skip..
00000f50 00 00 00 00 00 00 00 00 31 00 00 00 00 00 00 00 |........1.......|
00000f60 6d 6b 66 73 2e 65 78 74 33 20 2f 64 65 76 2f 76 |mkfs.ext3 /dev/v|
00000f70 7a 76 67 2f 74 65 73 74 20 2d 62 34 30 39 36 00 |zvg/test -b4096.|
00000f80 a0 fe 8c 00 00 00 00 00 21 00 00 00 00 00 00 00 |........!.......|
00000f90 23 31 32 30 35 39 35 30 34 30 34 00 3a 00 00 00 |#1205950404.:...|
00000fa0 20 00 8d 00 00 00 00 00 21 00 00 00 00 00 00 00 | .......!.......|
00000fb0 d0 cf 8c 00 00 00 00 00 10 d0 8c 00 00 00 00 00 |................|
00000fc0 00 00 00 00 00 00 00 00 41 00 00 00 00 00 00 00 |........A.......|
00000fd0 6d 6f 75 6e 74 20 2f 64 65 76 2f 76 7a 76 67 2f |mount /dev/vzvg/|
00000fe0 74 65 73 74 20 20 2f 76 7a 20 2d 6f 20 64 61 74 |test /vz -o dat|
00000ff0 61 3d 77 72 69 74 65 62 61 63 6b 00 00 00 00 00 |a=writeback.....|
00001000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
As you can see file's page contains garbage from pagecache instead of zeros.
#TEST_CASE_END
Attached patch:
- Add sanity check BUG_ON in order to prevent incorrect usage by caller,
This is function invariant because page can has buffers and in no zero
*fadata pointer at the same time.
- Always attach buffers to page is it is partial write case.
- Always switch back to generic_write_end if page has buffers.
This is reasonable because if page already has buffer then generic_write_begin
was called previously.
Signed-off-by: Dmitri Monakhov <dmonakhov@openvz.org>
Reviewed-by: Nick Piggin <npiggin@suse.de>
Cc: <stable@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-03-28 15:15:52 -06:00
|
|
|
BUG_ON(fsdata != NULL && page_has_buffers(page));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-02-06 13:59:26 -07:00
|
|
|
if (unlikely(copied < len) && head)
|
vfs: fix data leak in nobh_write_end()
Current nobh_write_end() implementation ignore partial writes(copied < len)
case if page was fully mapped and simply mark page as Uptodate, which is
totally wrong because area [pos+copied, pos+len) wasn't updated explicitly in
previous write_begin call. It simply contains garbage from pagecache and
result in data leakage.
#TEST_CASE_BEGIN:
~~~~~~~~~~~~~~~~
In fact issue triggered by classical testcase
open("/mnt/test", O_RDWR|O_CREAT|O_TRUNC, 0666) = 3
ftruncate(3, 409600) = 0
writev(3, [{"a", 1}, {NULL, 4095}], 2) = 1
##TESTCASE_SOURCE:
~~~~~~~~~~~~~~~~~
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/uio.h>
#include <sys/mman.h>
#include <errno.h>
int main(int argc, char **argv)
{
int fd, ret;
void* p;
struct iovec iov[2];
fd = open(argv[1], O_RDWR|O_CREAT|O_TRUNC, 0666);
ftruncate(fd, 409600);
iov[0].iov_base="a";
iov[0].iov_len=1;
iov[1].iov_base=NULL;
iov[1].iov_len=4096;
ret = writev(fd, iov, sizeof(iov)/sizeof(struct iovec));
printf("writev = %d, err = %d\n", ret, errno);
return 0;
}
##TESTCASE RESULT:
~~~~~~~~~~~~~~~~~~
[root@ts63 ~]# mount | grep mnt2
/dev/mapper/test on /mnt2 type ext2 (rw,nobh)
[root@ts63 ~]# /tmp/writev /mnt2/test
writev = 1, err = 0
[root@ts63 ~]# hexdump -C /mnt2/test
00000000 61 65 62 6f 6f 74 00 00 f0 b9 b4 59 3a 00 00 00 |aeboot.....Y:...|
00000010 20 00 00 00 00 00 00 00 21 00 00 00 00 00 00 00 | .......!.......|
00000020 df df df df df df df df df df df df df df df df |................|
00000030 3a 00 00 00 2a 00 00 00 21 00 00 00 00 00 00 00 |:...*...!.......|
00000040 60 c0 8c 00 00 00 00 00 40 4a 8d 00 00 00 00 00 |`.......@J......|
00000050 00 00 00 00 00 00 00 00 41 00 00 00 00 00 00 00 |........A.......|
00000060 74 69 6d 65 20 64 64 20 69 66 3d 2f 64 65 76 2f |time dd if=/dev/|
00000070 6c 6f 6f 70 30 20 20 6f 66 3d 2f 64 65 76 2f 6e |loop0 of=/dev/n|
skip..
00000f50 00 00 00 00 00 00 00 00 31 00 00 00 00 00 00 00 |........1.......|
00000f60 6d 6b 66 73 2e 65 78 74 33 20 2f 64 65 76 2f 76 |mkfs.ext3 /dev/v|
00000f70 7a 76 67 2f 74 65 73 74 20 2d 62 34 30 39 36 00 |zvg/test -b4096.|
00000f80 a0 fe 8c 00 00 00 00 00 21 00 00 00 00 00 00 00 |........!.......|
00000f90 23 31 32 30 35 39 35 30 34 30 34 00 3a 00 00 00 |#1205950404.:...|
00000fa0 20 00 8d 00 00 00 00 00 21 00 00 00 00 00 00 00 | .......!.......|
00000fb0 d0 cf 8c 00 00 00 00 00 10 d0 8c 00 00 00 00 00 |................|
00000fc0 00 00 00 00 00 00 00 00 41 00 00 00 00 00 00 00 |........A.......|
00000fd0 6d 6f 75 6e 74 20 2f 64 65 76 2f 76 7a 76 67 2f |mount /dev/vzvg/|
00000fe0 74 65 73 74 20 20 2f 76 7a 20 2d 6f 20 64 61 74 |test /vz -o dat|
00000ff0 61 3d 77 72 69 74 65 62 61 63 6b 00 00 00 00 00 |a=writeback.....|
00001000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
As you can see file's page contains garbage from pagecache instead of zeros.
#TEST_CASE_END
Attached patch:
- Add sanity check BUG_ON in order to prevent incorrect usage by caller,
This is function invariant because page can has buffers and in no zero
*fadata pointer at the same time.
- Always attach buffers to page is it is partial write case.
- Always switch back to generic_write_end if page has buffers.
This is reasonable because if page already has buffer then generic_write_begin
was called previously.
Signed-off-by: Dmitri Monakhov <dmonakhov@openvz.org>
Reviewed-by: Nick Piggin <npiggin@suse.de>
Cc: <stable@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-03-28 15:15:52 -06:00
|
|
|
attach_nobh_buffers(page, head);
|
|
|
|
if (page_has_buffers(page))
|
|
|
|
return generic_write_end(file, mapping, pos, len,
|
|
|
|
copied, page, fsdata);
|
fs: fix nobh error handling
nobh mode error handling is not just pretty slack, it's wrong.
One cannot zero out the whole page to ensure new blocks are zeroed, because
it just brings the whole page "uptodate" with zeroes even if that may not
be the correct uptodate data. Also, other parts of the page may already
contain dirty data which would get lost by zeroing it out. Thirdly, the
writeback of zeroes to the new blocks will also erase existing blocks. All
these conditions are pagecache and/or filesystem corruption.
The problem comes about because we didn't keep track of which buffers
actually are new or old. However it is not enough just to keep only this
state, because at the point we start dirtying parts of the page (new
blocks, with zeroes), the handling of IO errors becomes impossible without
buffers because the page may only be partially uptodate, in which case the
page flags allone cannot capture the state of the parts of the page.
So allocate all buffers for the page upfront, but leave them unattached so
that they don't pick up any other references and can be freed when we're
done. If the error path is hit, then zero the new buffers as the regular
buffer path does, then attach the buffers to the page so that it can
actually be written out correctly and be subject to the normal IO error
handling paths.
As an upshot, we save 1K of kernel stack on ia64 or powerpc 64K page
systems.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-16 02:24:48 -06:00
|
|
|
|
2007-02-20 14:58:09 -07:00
|
|
|
SetPageUptodate(page);
|
2005-04-16 16:20:36 -06:00
|
|
|
set_page_dirty(page);
|
2007-10-16 02:25:25 -06:00
|
|
|
if (pos+copied > inode->i_size) {
|
|
|
|
i_size_write(inode, pos+copied);
|
2005-04-16 16:20:36 -06:00
|
|
|
mark_inode_dirty(inode);
|
|
|
|
}
|
2007-10-16 02:25:25 -06:00
|
|
|
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
|
|
|
|
while (head) {
|
|
|
|
bh = head;
|
|
|
|
head = head->b_this_page;
|
|
|
|
free_buffer_head(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
return copied;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2007-10-16 02:25:25 -06:00
|
|
|
EXPORT_SYMBOL(nobh_write_end);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* nobh_writepage() - based on block_full_write_page() except
|
|
|
|
* that it tries to operate without attaching bufferheads to
|
|
|
|
* the page.
|
|
|
|
*/
|
|
|
|
int nobh_writepage(struct page *page, get_block_t *get_block,
|
|
|
|
struct writeback_control *wbc)
|
|
|
|
{
|
|
|
|
struct inode * const inode = page->mapping->host;
|
|
|
|
loff_t i_size = i_size_read(inode);
|
|
|
|
const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
|
|
|
|
unsigned offset;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
/* Is the page fully inside i_size? */
|
|
|
|
if (page->index < end_index)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
/* Is the page fully outside i_size? (truncate in progress) */
|
|
|
|
offset = i_size & (PAGE_CACHE_SIZE-1);
|
|
|
|
if (page->index >= end_index+1 || !offset) {
|
|
|
|
/*
|
|
|
|
* The page may have dirty, unmapped buffers. For example,
|
|
|
|
* they may have been added in ext3_writepage(). Make them
|
|
|
|
* freeable here, so the page does not leak.
|
|
|
|
*/
|
|
|
|
#if 0
|
|
|
|
/* Not really sure about this - do we need this ? */
|
|
|
|
if (page->mapping->a_ops->invalidatepage)
|
|
|
|
page->mapping->a_ops->invalidatepage(page, offset);
|
|
|
|
#endif
|
|
|
|
unlock_page(page);
|
|
|
|
return 0; /* don't care */
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The page straddles i_size. It must be zeroed out on each and every
|
|
|
|
* writepage invocation because it may be mmapped. "A file is mapped
|
|
|
|
* in multiples of the page size. For a file that is not a multiple of
|
|
|
|
* the page size, the remaining memory is zeroed when mapped, and
|
|
|
|
* writes to that region are not written out to the file."
|
|
|
|
*/
|
2008-02-04 23:28:29 -07:00
|
|
|
zero_user_segment(page, offset, PAGE_CACHE_SIZE);
|
2005-04-16 16:20:36 -06:00
|
|
|
out:
|
|
|
|
ret = mpage_writepage(page, get_block, wbc);
|
|
|
|
if (ret == -EAGAIN)
|
2009-04-15 11:22:38 -06:00
|
|
|
ret = __block_write_full_page(inode, page, get_block, wbc,
|
|
|
|
end_buffer_async_write);
|
2005-04-16 16:20:36 -06:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(nobh_writepage);
|
|
|
|
|
2007-10-16 02:25:25 -06:00
|
|
|
int nobh_truncate_page(struct address_space *mapping,
|
|
|
|
loff_t from, get_block_t *get_block)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
pgoff_t index = from >> PAGE_CACHE_SHIFT;
|
|
|
|
unsigned offset = from & (PAGE_CACHE_SIZE-1);
|
2007-10-16 02:25:25 -06:00
|
|
|
unsigned blocksize;
|
|
|
|
sector_t iblock;
|
|
|
|
unsigned length, pos;
|
|
|
|
struct inode *inode = mapping->host;
|
2005-04-16 16:20:36 -06:00
|
|
|
struct page *page;
|
2007-10-16 02:25:25 -06:00
|
|
|
struct buffer_head map_bh;
|
|
|
|
int err;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-16 02:25:25 -06:00
|
|
|
blocksize = 1 << inode->i_blkbits;
|
|
|
|
length = offset & (blocksize - 1);
|
|
|
|
|
|
|
|
/* Block boundary? Nothing to do */
|
|
|
|
if (!length)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
length = blocksize - length;
|
|
|
|
iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
page = grab_cache_page(mapping, index);
|
2007-10-16 02:25:25 -06:00
|
|
|
err = -ENOMEM;
|
2005-04-16 16:20:36 -06:00
|
|
|
if (!page)
|
|
|
|
goto out;
|
|
|
|
|
2007-10-16 02:25:25 -06:00
|
|
|
if (page_has_buffers(page)) {
|
|
|
|
has_buffers:
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
return block_truncate_page(mapping, from, get_block);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Find the buffer that contains "offset" */
|
|
|
|
pos = blocksize;
|
|
|
|
while (offset >= pos) {
|
|
|
|
iblock++;
|
|
|
|
pos += blocksize;
|
|
|
|
}
|
|
|
|
|
Fix nobh_truncate_page() to not pass stack garbage to get_block()
The nobh_truncate_page() function is used by ext2, exofs, and jfs. Of
these three, only ext2 and jfs's get_block() function pays attention
to bh->b_size --- which is normally always the filesystem blocksize
except when the get_block() function is called by either
mpage_readpage(), mpage_readpages(), or the direct I/O routines in
fs/direct_io.c.
Unfortunately, nobh_truncate_page() does not initialize map_bh before
calling the filesystem-supplied get_block() function. So ext2 and jfs
will try to calculate the number of blocks to map by taking stack
garbage and shifting it left by inode->i_blkbits. This should be
*mostly* harmless (except the filesystem will do some unnneeded work)
unless the stack garbage is less than filesystem's blocksize, in which
case maxblocks will be zero, and the attempt to find out whether or
not the filesystem has a hole at a given logical block will fail, and
the page cache entry might not get zero'ed out.
Also if the stack garbage in in map_bh->state happens to have the
BH_Mapped bit set, there could be an attempt to call readpage() on a
non-existent page, which could cause nobh_truncate_page() to return an
error when it should not.
Fix this by initializing map_bh->state and map_bh->size.
Fortunately, it's probably fairly unlikely that ext2 and jfs users
mount with nobh these days.
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
Cc: Dave Kleikamp <shaggy@linux.vnet.ibm.com>
Cc: linux-fsdevel@vger.kernel.org
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2009-05-12 05:37:56 -06:00
|
|
|
map_bh.b_size = blocksize;
|
|
|
|
map_bh.b_state = 0;
|
2007-10-16 02:25:25 -06:00
|
|
|
err = get_block(inode, iblock, &map_bh, 0);
|
|
|
|
if (err)
|
|
|
|
goto unlock;
|
|
|
|
/* unmapped? It's a hole - nothing to do */
|
|
|
|
if (!buffer_mapped(&map_bh))
|
|
|
|
goto unlock;
|
|
|
|
|
|
|
|
/* Ok, it's mapped. Make sure it's up-to-date */
|
|
|
|
if (!PageUptodate(page)) {
|
|
|
|
err = mapping->a_ops->readpage(NULL, page);
|
|
|
|
if (err) {
|
|
|
|
page_cache_release(page);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
lock_page(page);
|
|
|
|
if (!PageUptodate(page)) {
|
|
|
|
err = -EIO;
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
if (page_has_buffers(page))
|
|
|
|
goto has_buffers;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2008-02-04 23:28:29 -07:00
|
|
|
zero_user(page, offset, length);
|
2007-10-16 02:25:25 -06:00
|
|
|
set_page_dirty(page);
|
|
|
|
err = 0;
|
|
|
|
|
|
|
|
unlock:
|
2005-04-16 16:20:36 -06:00
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
out:
|
2007-10-16 02:25:25 -06:00
|
|
|
return err;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(nobh_truncate_page);
|
|
|
|
|
|
|
|
int block_truncate_page(struct address_space *mapping,
|
|
|
|
loff_t from, get_block_t *get_block)
|
|
|
|
{
|
|
|
|
pgoff_t index = from >> PAGE_CACHE_SHIFT;
|
|
|
|
unsigned offset = from & (PAGE_CACHE_SIZE-1);
|
|
|
|
unsigned blocksize;
|
2006-01-08 02:03:05 -07:00
|
|
|
sector_t iblock;
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned length, pos;
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
struct page *page;
|
|
|
|
struct buffer_head *bh;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
blocksize = 1 << inode->i_blkbits;
|
|
|
|
length = offset & (blocksize - 1);
|
|
|
|
|
|
|
|
/* Block boundary? Nothing to do */
|
|
|
|
if (!length)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
length = blocksize - length;
|
2006-01-08 02:03:05 -07:00
|
|
|
iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
page = grab_cache_page(mapping, index);
|
|
|
|
err = -ENOMEM;
|
|
|
|
if (!page)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
if (!page_has_buffers(page))
|
|
|
|
create_empty_buffers(page, blocksize, 0);
|
|
|
|
|
|
|
|
/* Find the buffer that contains "offset" */
|
|
|
|
bh = page_buffers(page);
|
|
|
|
pos = blocksize;
|
|
|
|
while (offset >= pos) {
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
iblock++;
|
|
|
|
pos += blocksize;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = 0;
|
|
|
|
if (!buffer_mapped(bh)) {
|
2006-03-26 02:38:00 -07:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-04-16 16:20:36 -06:00
|
|
|
err = get_block(inode, iblock, bh, 0);
|
|
|
|
if (err)
|
|
|
|
goto unlock;
|
|
|
|
/* unmapped? It's a hole - nothing to do */
|
|
|
|
if (!buffer_mapped(bh))
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Ok, it's mapped. Make sure it's up-to-date */
|
|
|
|
if (PageUptodate(page))
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
|
2007-02-12 01:51:41 -07:00
|
|
|
if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
|
2005-04-16 16:20:36 -06:00
|
|
|
err = -EIO;
|
|
|
|
ll_rw_block(READ, 1, &bh);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
/* Uhhuh. Read error. Complain and punt. */
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
2008-02-04 23:28:29 -07:00
|
|
|
zero_user(page, offset, length);
|
2005-04-16 16:20:36 -06:00
|
|
|
mark_buffer_dirty(bh);
|
|
|
|
err = 0;
|
|
|
|
|
|
|
|
unlock:
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(block_truncate_page);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The generic ->writepage function for buffer-backed address_spaces
|
2009-04-15 11:22:38 -06:00
|
|
|
* this form passes in the end_io handler used to finish the IO.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2009-04-15 11:22:38 -06:00
|
|
|
int block_write_full_page_endio(struct page *page, get_block_t *get_block,
|
|
|
|
struct writeback_control *wbc, bh_end_io_t *handler)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct inode * const inode = page->mapping->host;
|
|
|
|
loff_t i_size = i_size_read(inode);
|
|
|
|
const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
|
|
|
|
unsigned offset;
|
|
|
|
|
|
|
|
/* Is the page fully inside i_size? */
|
|
|
|
if (page->index < end_index)
|
2009-04-15 11:22:38 -06:00
|
|
|
return __block_write_full_page(inode, page, get_block, wbc,
|
|
|
|
handler);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/* Is the page fully outside i_size? (truncate in progress) */
|
|
|
|
offset = i_size & (PAGE_CACHE_SIZE-1);
|
|
|
|
if (page->index >= end_index+1 || !offset) {
|
|
|
|
/*
|
|
|
|
* The page may have dirty, unmapped buffers. For example,
|
|
|
|
* they may have been added in ext3_writepage(). Make them
|
|
|
|
* freeable here, so the page does not leak.
|
|
|
|
*/
|
2005-10-30 16:00:16 -07:00
|
|
|
do_invalidatepage(page, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
unlock_page(page);
|
|
|
|
return 0; /* don't care */
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The page straddles i_size. It must be zeroed out on each and every
|
2009-12-11 14:35:40 -07:00
|
|
|
* writepage invocation because it may be mmapped. "A file is mapped
|
2005-04-16 16:20:36 -06:00
|
|
|
* in multiples of the page size. For a file that is not a multiple of
|
|
|
|
* the page size, the remaining memory is zeroed when mapped, and
|
|
|
|
* writes to that region are not written out to the file."
|
|
|
|
*/
|
2008-02-04 23:28:29 -07:00
|
|
|
zero_user_segment(page, offset, PAGE_CACHE_SIZE);
|
2009-04-15 11:22:38 -06:00
|
|
|
return __block_write_full_page(inode, page, get_block, wbc, handler);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(block_write_full_page_endio);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2009-04-15 11:22:38 -06:00
|
|
|
/*
|
|
|
|
* The generic ->writepage function for buffer-backed address_spaces
|
|
|
|
*/
|
|
|
|
int block_write_full_page(struct page *page, get_block_t *get_block,
|
|
|
|
struct writeback_control *wbc)
|
|
|
|
{
|
|
|
|
return block_write_full_page_endio(page, get_block, wbc,
|
|
|
|
end_buffer_async_write);
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(block_write_full_page);
|
2009-04-15 11:22:38 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
|
|
|
|
get_block_t *get_block)
|
|
|
|
{
|
|
|
|
struct buffer_head tmp;
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
tmp.b_state = 0;
|
|
|
|
tmp.b_blocknr = 0;
|
2006-03-26 02:38:00 -07:00
|
|
|
tmp.b_size = 1 << inode->i_blkbits;
|
2005-04-16 16:20:36 -06:00
|
|
|
get_block(inode, block, &tmp, 0);
|
|
|
|
return tmp.b_blocknr;
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(generic_block_bmap);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-09-27 04:47:43 -06:00
|
|
|
static void end_bio_bh_io_sync(struct bio *bio, int err)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = bio->bi_private;
|
|
|
|
|
|
|
|
if (err == -EOPNOTSUPP) {
|
|
|
|
set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
|
|
|
|
}
|
|
|
|
|
block: Supress Buffer I/O errors when SCSI REQ_QUIET flag set
Allow the scsi request REQ_QUIET flag to be propagated to the buffer
file system layer. The basic ideas is to pass the flag from the scsi
request to the bio (block IO) and then to the buffer layer. The buffer
layer can then suppress needless printks.
This patch declutters the kernel log by removed the 40-50 (per lun)
buffer io error messages seen during a boot in my multipath setup . It
is a good chance any real errors will be missed in the "noise" it the
logs without this patch.
During boot I see blocks of messages like
"
__ratelimit: 211 callbacks suppressed
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242847
Buffer I/O error on device sdm, logical block 1
Buffer I/O error on device sdm, logical block 5242878
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242872
"
in my logs.
My disk environment is multipath fiber channel using the SCSI_DH_RDAC
code and multipathd. This topology includes an "active" and "ghost"
path for each lun. IO's to the "ghost" path will never complete and the
SCSI layer, via the scsi device handler rdac code, quick returns the IOs
to theses paths and sets the REQ_QUIET scsi flag to suppress the scsi
layer messages.
I am wanting to extend the QUIET behavior to include the buffer file
system layer to deal with these errors as well. I have been running this
patch for a while now on several boxes without issue. A few runs of
bonnie++ show no noticeable difference in performance in my setup.
Thanks for John Stultz for the quiet_error finalization.
Submitted-by: Keith Mannthey <kmannth@us.ibm.com>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
2008-11-25 02:24:35 -07:00
|
|
|
if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
|
|
|
|
set_bit(BH_Quiet, &bh->b_state);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
|
|
|
|
bio_put(bio);
|
|
|
|
}
|
|
|
|
|
|
|
|
int submit_bh(int rw, struct buffer_head * bh)
|
|
|
|
{
|
|
|
|
struct bio *bio;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
BUG_ON(!buffer_locked(bh));
|
|
|
|
BUG_ON(!buffer_mapped(bh));
|
|
|
|
BUG_ON(!bh->b_end_io);
|
2009-05-12 14:22:37 -06:00
|
|
|
BUG_ON(buffer_delay(bh));
|
|
|
|
BUG_ON(buffer_unwritten(bh));
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
2008-08-22 02:00:36 -06:00
|
|
|
* Only clear out a write error when rewriting
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2008-08-22 02:00:36 -06:00
|
|
|
if (test_set_buffer_req(bh) && (rw & WRITE))
|
2005-04-16 16:20:36 -06:00
|
|
|
clear_buffer_write_io_error(bh);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* from here on down, it's all bio -- do the initial mapping,
|
|
|
|
* submit_bio -> generic_make_request may further map this bio around
|
|
|
|
*/
|
|
|
|
bio = bio_alloc(GFP_NOIO, 1);
|
|
|
|
|
|
|
|
bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
|
|
|
|
bio->bi_bdev = bh->b_bdev;
|
|
|
|
bio->bi_io_vec[0].bv_page = bh->b_page;
|
|
|
|
bio->bi_io_vec[0].bv_len = bh->b_size;
|
|
|
|
bio->bi_io_vec[0].bv_offset = bh_offset(bh);
|
|
|
|
|
|
|
|
bio->bi_vcnt = 1;
|
|
|
|
bio->bi_idx = 0;
|
|
|
|
bio->bi_size = bh->b_size;
|
|
|
|
|
|
|
|
bio->bi_end_io = end_bio_bh_io_sync;
|
|
|
|
bio->bi_private = bh;
|
|
|
|
|
|
|
|
bio_get(bio);
|
|
|
|
submit_bio(rw, bio);
|
|
|
|
|
|
|
|
if (bio_flagged(bio, BIO_EOPNOTSUPP))
|
|
|
|
ret = -EOPNOTSUPP;
|
|
|
|
|
|
|
|
bio_put(bio);
|
|
|
|
return ret;
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(submit_bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/**
|
|
|
|
* ll_rw_block: low-level access to block devices (DEPRECATED)
|
2010-08-11 09:06:24 -06:00
|
|
|
* @rw: whether to %READ or %WRITE or maybe %READA (readahead)
|
2005-04-16 16:20:36 -06:00
|
|
|
* @nr: number of &struct buffer_heads in the array
|
|
|
|
* @bhs: array of pointers to &struct buffer_head
|
|
|
|
*
|
2005-09-06 16:19:10 -06:00
|
|
|
* ll_rw_block() takes an array of pointers to &struct buffer_heads, and
|
|
|
|
* requests an I/O operation on them, either a %READ or a %WRITE. The third
|
2010-08-11 09:06:24 -06:00
|
|
|
* %READA option is described in the documentation for generic_make_request()
|
|
|
|
* which ll_rw_block() calls.
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* This function drops any buffer that it cannot get a lock on (with the
|
2010-08-11 09:06:24 -06:00
|
|
|
* BH_Lock state bit), any buffer that appears to be clean when doing a write
|
|
|
|
* request, and any buffer that appears to be up-to-date when doing read
|
|
|
|
* request. Further it marks as clean buffers that are processed for
|
|
|
|
* writing (the buffer cache won't assume that they are actually clean
|
|
|
|
* until the buffer gets unlocked).
|
2005-04-16 16:20:36 -06:00
|
|
|
*
|
|
|
|
* ll_rw_block sets b_end_io to simple completion handler that marks
|
|
|
|
* the buffer up-to-date (if approriate), unlocks the buffer and wakes
|
|
|
|
* any waiters.
|
|
|
|
*
|
|
|
|
* All of the buffers must be for the same device, and must also be a
|
|
|
|
* multiple of the current approved size for the device.
|
|
|
|
*/
|
|
|
|
void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < nr; i++) {
|
|
|
|
struct buffer_head *bh = bhs[i];
|
|
|
|
|
2010-08-11 09:06:24 -06:00
|
|
|
if (!trylock_buffer(bh))
|
2005-04-16 16:20:36 -06:00
|
|
|
continue;
|
2010-08-11 09:06:24 -06:00
|
|
|
if (rw == WRITE) {
|
2005-04-16 16:20:36 -06:00
|
|
|
if (test_clear_buffer_dirty(bh)) {
|
2005-04-16 16:24:07 -06:00
|
|
|
bh->b_end_io = end_buffer_write_sync;
|
2006-02-03 04:04:43 -07:00
|
|
|
get_bh(bh);
|
2010-08-11 09:06:24 -06:00
|
|
|
submit_bh(WRITE, bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (!buffer_uptodate(bh)) {
|
2005-04-16 16:24:07 -06:00
|
|
|
bh->b_end_io = end_buffer_read_sync;
|
2006-02-03 04:04:43 -07:00
|
|
|
get_bh(bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
submit_bh(rw, bh);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(ll_rw_block);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2010-08-11 09:06:24 -06:00
|
|
|
void write_dirty_buffer(struct buffer_head *bh, int rw)
|
|
|
|
{
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (!test_clear_buffer_dirty(bh)) {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
bh->b_end_io = end_buffer_write_sync;
|
|
|
|
get_bh(bh);
|
|
|
|
submit_bh(rw, bh);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(write_dirty_buffer);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* For a data-integrity writeout, we need to wait upon any in-progress I/O
|
|
|
|
* and then start new I/O and then wait upon it. The caller must have a ref on
|
|
|
|
* the buffer_head.
|
|
|
|
*/
|
2010-08-11 09:05:45 -06:00
|
|
|
int __sync_dirty_buffer(struct buffer_head *bh, int rw)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
WARN_ON(atomic_read(&bh->b_count) < 1);
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (test_clear_buffer_dirty(bh)) {
|
|
|
|
get_bh(bh);
|
|
|
|
bh->b_end_io = end_buffer_write_sync;
|
2010-08-11 09:05:45 -06:00
|
|
|
ret = submit_bh(rw, bh);
|
2005-04-16 16:20:36 -06:00
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!ret && !buffer_uptodate(bh))
|
|
|
|
ret = -EIO;
|
|
|
|
} else {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
2010-08-11 09:05:45 -06:00
|
|
|
EXPORT_SYMBOL(__sync_dirty_buffer);
|
|
|
|
|
|
|
|
int sync_dirty_buffer(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
return __sync_dirty_buffer(bh, WRITE_SYNC);
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(sync_dirty_buffer);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* try_to_free_buffers() checks if all the buffers on this particular page
|
|
|
|
* are unused, and releases them if so.
|
|
|
|
*
|
|
|
|
* Exclusion against try_to_free_buffers may be obtained by either
|
|
|
|
* locking the page or by holding its mapping's private_lock.
|
|
|
|
*
|
|
|
|
* If the page is dirty but all the buffers are clean then we need to
|
|
|
|
* be sure to mark the page clean as well. This is because the page
|
|
|
|
* may be against a block device, and a later reattachment of buffers
|
|
|
|
* to a dirty page will set *all* buffers dirty. Which would corrupt
|
|
|
|
* filesystem data on the same device.
|
|
|
|
*
|
|
|
|
* The same applies to regular filesystem pages: if all the buffers are
|
|
|
|
* clean then we set the page clean and proceed. To do that, we require
|
|
|
|
* total exclusion from __set_page_dirty_buffers(). That is obtained with
|
|
|
|
* private_lock.
|
|
|
|
*
|
|
|
|
* try_to_free_buffers() is non-blocking.
|
|
|
|
*/
|
|
|
|
static inline int buffer_busy(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
return atomic_read(&bh->b_count) |
|
|
|
|
(bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
|
|
|
|
{
|
|
|
|
struct buffer_head *head = page_buffers(page);
|
|
|
|
struct buffer_head *bh;
|
|
|
|
|
|
|
|
bh = head;
|
|
|
|
do {
|
2005-05-01 09:58:39 -06:00
|
|
|
if (buffer_write_io_error(bh) && page->mapping)
|
2005-04-16 16:20:36 -06:00
|
|
|
set_bit(AS_EIO, &page->mapping->flags);
|
|
|
|
if (buffer_busy(bh))
|
|
|
|
goto failed;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
|
2008-02-08 05:21:59 -07:00
|
|
|
if (bh->b_assoc_map)
|
2005-04-16 16:20:36 -06:00
|
|
|
__remove_assoc_queue(bh);
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
|
|
|
*buffers_to_free = head;
|
|
|
|
__clear_page_buffers(page);
|
|
|
|
return 1;
|
|
|
|
failed:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int try_to_free_buffers(struct page *page)
|
|
|
|
{
|
|
|
|
struct address_space * const mapping = page->mapping;
|
|
|
|
struct buffer_head *buffers_to_free = NULL;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
Resurrect 'try_to_free_buffers()' VM hackery
It's not pretty, but it appears that ext3 with data=journal will clean
pages without ever actually telling the VM that they are clean. This,
in turn, will result in the VM (and balance_dirty_pages() in particular)
to never realize that the pages got cleaned, and wait forever for an
event that already happened.
Technically, this seems to be a problem with ext3 itself, but it used to
be hidden by 'try_to_free_buffers()' noticing this situation on its own,
and just working around the filesystem problem.
This commit re-instates that hack, in order to avoid a regression for
the 2.6.20 release. This fixes bugzilla 7844:
http://bugzilla.kernel.org/show_bug.cgi?id=7844
Peter Zijlstra points out that we should probably retain the debugging
code that this removes from cancel_dirty_page(), and I agree, but for
the imminent release we might as well just silence the warning too
(since it's not a new bug: anything that triggers that warning has been
around forever).
Acked-by: Randy Dunlap <rdunlap@xenotime.net>
Acked-by: Jens Axboe <jens.axboe@oracle.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-01-26 13:47:06 -07:00
|
|
|
if (PageWriteback(page))
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (mapping == NULL) { /* can this still happen? */
|
|
|
|
ret = drop_buffers(page, &buffers_to_free);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_lock(&mapping->private_lock);
|
|
|
|
ret = drop_buffers(page, &buffers_to_free);
|
Resurrect 'try_to_free_buffers()' VM hackery
It's not pretty, but it appears that ext3 with data=journal will clean
pages without ever actually telling the VM that they are clean. This,
in turn, will result in the VM (and balance_dirty_pages() in particular)
to never realize that the pages got cleaned, and wait forever for an
event that already happened.
Technically, this seems to be a problem with ext3 itself, but it used to
be hidden by 'try_to_free_buffers()' noticing this situation on its own,
and just working around the filesystem problem.
This commit re-instates that hack, in order to avoid a regression for
the 2.6.20 release. This fixes bugzilla 7844:
http://bugzilla.kernel.org/show_bug.cgi?id=7844
Peter Zijlstra points out that we should probably retain the debugging
code that this removes from cancel_dirty_page(), and I agree, but for
the imminent release we might as well just silence the warning too
(since it's not a new bug: anything that triggers that warning has been
around forever).
Acked-by: Randy Dunlap <rdunlap@xenotime.net>
Acked-by: Jens Axboe <jens.axboe@oracle.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-01-26 13:47:06 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If the filesystem writes its buffers by hand (eg ext3)
|
|
|
|
* then we can have clean buffers against a dirty page. We
|
|
|
|
* clean the page here; otherwise the VM will never notice
|
|
|
|
* that the filesystem did any IO at all.
|
|
|
|
*
|
|
|
|
* Also, during truncate, discard_buffer will have marked all
|
|
|
|
* the page's buffers clean. We discover that here and clean
|
|
|
|
* the page also.
|
2007-01-29 20:36:27 -07:00
|
|
|
*
|
|
|
|
* private_lock must be held over this entire operation in order
|
|
|
|
* to synchronise against __set_page_dirty_buffers and prevent the
|
|
|
|
* dirty bit from being lost.
|
Resurrect 'try_to_free_buffers()' VM hackery
It's not pretty, but it appears that ext3 with data=journal will clean
pages without ever actually telling the VM that they are clean. This,
in turn, will result in the VM (and balance_dirty_pages() in particular)
to never realize that the pages got cleaned, and wait forever for an
event that already happened.
Technically, this seems to be a problem with ext3 itself, but it used to
be hidden by 'try_to_free_buffers()' noticing this situation on its own,
and just working around the filesystem problem.
This commit re-instates that hack, in order to avoid a regression for
the 2.6.20 release. This fixes bugzilla 7844:
http://bugzilla.kernel.org/show_bug.cgi?id=7844
Peter Zijlstra points out that we should probably retain the debugging
code that this removes from cancel_dirty_page(), and I agree, but for
the imminent release we might as well just silence the warning too
(since it's not a new bug: anything that triggers that warning has been
around forever).
Acked-by: Randy Dunlap <rdunlap@xenotime.net>
Acked-by: Jens Axboe <jens.axboe@oracle.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-01-26 13:47:06 -07:00
|
|
|
*/
|
|
|
|
if (ret)
|
|
|
|
cancel_dirty_page(page, PAGE_CACHE_SIZE);
|
2007-01-29 20:36:27 -07:00
|
|
|
spin_unlock(&mapping->private_lock);
|
2005-04-16 16:20:36 -06:00
|
|
|
out:
|
|
|
|
if (buffers_to_free) {
|
|
|
|
struct buffer_head *bh = buffers_to_free;
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
free_buffer_head(bh);
|
|
|
|
bh = next;
|
|
|
|
} while (bh != buffers_to_free);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(try_to_free_buffers);
|
|
|
|
|
2006-03-26 02:37:17 -07:00
|
|
|
void block_sync_page(struct page *page)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
struct address_space *mapping;
|
|
|
|
|
|
|
|
smp_mb();
|
|
|
|
mapping = page_mapping(page);
|
|
|
|
if (mapping)
|
|
|
|
blk_run_backing_dev(mapping->backing_dev_info, page);
|
|
|
|
}
|
2009-09-22 17:43:51 -06:00
|
|
|
EXPORT_SYMBOL(block_sync_page);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* There are no bdflush tunables left. But distributions are
|
|
|
|
* still running obsolete flush daemons, so we terminate them here.
|
|
|
|
*
|
|
|
|
* Use of bdflush() is deprecated and will be removed in a future kernel.
|
2009-09-23 11:37:09 -06:00
|
|
|
* The `flush-X' kernel threads fully replace bdflush daemons and this call.
|
2005-04-16 16:20:36 -06:00
|
|
|
*/
|
2009-01-14 06:14:12 -07:00
|
|
|
SYSCALL_DEFINE2(bdflush, int, func, long, data)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
static int msg_count;
|
|
|
|
|
|
|
|
if (!capable(CAP_SYS_ADMIN))
|
|
|
|
return -EPERM;
|
|
|
|
|
|
|
|
if (msg_count < 5) {
|
|
|
|
msg_count++;
|
|
|
|
printk(KERN_INFO
|
|
|
|
"warning: process `%s' used the obsolete bdflush"
|
|
|
|
" system call\n", current->comm);
|
|
|
|
printk(KERN_INFO "Fix your initscripts?\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
if (func == 1)
|
|
|
|
do_exit(0);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Buffer-head allocation
|
|
|
|
*/
|
2006-12-06 21:33:20 -07:00
|
|
|
static struct kmem_cache *bh_cachep;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Once the number of bh's in the machine exceeds this level, we start
|
|
|
|
* stripping them in writeback.
|
|
|
|
*/
|
|
|
|
static int max_buffer_heads;
|
|
|
|
|
|
|
|
int buffer_heads_over_limit;
|
|
|
|
|
|
|
|
struct bh_accounting {
|
|
|
|
int nr; /* Number of live bh's */
|
|
|
|
int ratelimit; /* Limit cacheline bouncing */
|
|
|
|
};
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
|
|
|
|
|
|
|
|
static void recalc_bh_state(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
int tot = 0;
|
|
|
|
|
2010-12-06 10:40:05 -07:00
|
|
|
if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
|
2005-04-16 16:20:36 -06:00
|
|
|
return;
|
2010-12-06 10:16:28 -07:00
|
|
|
__this_cpu_write(bh_accounting.ratelimit, 0);
|
2006-03-24 04:18:10 -07:00
|
|
|
for_each_online_cpu(i)
|
2005-04-16 16:20:36 -06:00
|
|
|
tot += per_cpu(bh_accounting, i).nr;
|
|
|
|
buffer_heads_over_limit = (tot > max_buffer_heads);
|
|
|
|
}
|
2010-12-06 10:16:28 -07:00
|
|
|
|
2005-10-07 00:46:04 -06:00
|
|
|
struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2010-03-10 16:20:33 -07:00
|
|
|
struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
if (ret) {
|
2007-05-16 23:10:57 -06:00
|
|
|
INIT_LIST_HEAD(&ret->b_assoc_buffers);
|
2010-12-06 10:16:28 -07:00
|
|
|
preempt_disable();
|
|
|
|
__this_cpu_inc(bh_accounting.nr);
|
2005-04-16 16:20:36 -06:00
|
|
|
recalc_bh_state();
|
2010-12-06 10:16:28 -07:00
|
|
|
preempt_enable();
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(alloc_buffer_head);
|
|
|
|
|
|
|
|
void free_buffer_head(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
BUG_ON(!list_empty(&bh->b_assoc_buffers));
|
|
|
|
kmem_cache_free(bh_cachep, bh);
|
2010-12-06 10:16:28 -07:00
|
|
|
preempt_disable();
|
|
|
|
__this_cpu_dec(bh_accounting.nr);
|
2005-04-16 16:20:36 -06:00
|
|
|
recalc_bh_state();
|
2010-12-06 10:16:28 -07:00
|
|
|
preempt_enable();
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(free_buffer_head);
|
|
|
|
|
|
|
|
static void buffer_exit_cpu(int cpu)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct bh_lru *b = &per_cpu(bh_lrus, cpu);
|
|
|
|
|
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
|
|
|
brelse(b->bhs[i]);
|
|
|
|
b->bhs[i] = NULL;
|
|
|
|
}
|
2010-12-06 10:16:28 -07:00
|
|
|
this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
|
2006-03-24 04:18:10 -07:00
|
|
|
per_cpu(bh_accounting, cpu).nr = 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static int buffer_cpu_notify(struct notifier_block *self,
|
|
|
|
unsigned long action, void *hcpu)
|
|
|
|
{
|
2007-05-09 03:35:10 -06:00
|
|
|
if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
|
2005-04-16 16:20:36 -06:00
|
|
|
buffer_exit_cpu((unsigned long)hcpu);
|
|
|
|
return NOTIFY_OK;
|
|
|
|
}
|
|
|
|
|
2008-01-28 21:58:26 -07:00
|
|
|
/**
|
2008-03-19 18:01:00 -06:00
|
|
|
* bh_uptodate_or_lock - Test whether the buffer is uptodate
|
2008-01-28 21:58:26 -07:00
|
|
|
* @bh: struct buffer_head
|
|
|
|
*
|
|
|
|
* Return true if the buffer is up-to-date and false,
|
|
|
|
* with the buffer locked, if not.
|
|
|
|
*/
|
|
|
|
int bh_uptodate_or_lock(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
if (!buffer_uptodate(bh)) {
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
return 0;
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(bh_uptodate_or_lock);
|
|
|
|
|
|
|
|
/**
|
2008-03-19 18:01:00 -06:00
|
|
|
* bh_submit_read - Submit a locked buffer for reading
|
2008-01-28 21:58:26 -07:00
|
|
|
* @bh: struct buffer_head
|
|
|
|
*
|
|
|
|
* Returns zero on success and -EIO on error.
|
|
|
|
*/
|
|
|
|
int bh_submit_read(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
BUG_ON(!buffer_locked(bh));
|
|
|
|
|
|
|
|
if (buffer_uptodate(bh)) {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
get_bh(bh);
|
|
|
|
bh->b_end_io = end_buffer_read_sync;
|
|
|
|
submit_bh(READ, bh);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
return 0;
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(bh_submit_read);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
void __init buffer_init(void)
|
|
|
|
{
|
|
|
|
int nrpages;
|
|
|
|
|
2008-02-04 23:28:36 -07:00
|
|
|
bh_cachep = kmem_cache_create("buffer_head",
|
|
|
|
sizeof(struct buffer_head), 0,
|
|
|
|
(SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
|
|
|
|
SLAB_MEM_SPREAD),
|
2010-03-10 16:20:33 -07:00
|
|
|
NULL);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Limit the bh occupancy to 10% of ZONE_NORMAL
|
|
|
|
*/
|
|
|
|
nrpages = (nr_free_buffer_pages() * 10) / 100;
|
|
|
|
max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
|
|
|
|
hotcpu_notifier(buffer_cpu_notify, 0);
|
|
|
|
}
|