2010-04-28 07:55:08 -06:00
|
|
|
/*
|
|
|
|
* Functions related to generic helpers functions
|
|
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/bio.h>
|
|
|
|
#include <linux/blkdev.h>
|
|
|
|
#include <linux/scatterlist.h>
|
|
|
|
|
|
|
|
#include "blk.h"
|
|
|
|
|
2016-04-16 12:55:27 -06:00
|
|
|
static struct bio *next_bio(struct bio *bio, int rw, unsigned int nr_pages,
|
|
|
|
gfp_t gfp)
|
2010-04-28 07:55:08 -06:00
|
|
|
{
|
2016-04-16 12:55:27 -06:00
|
|
|
struct bio *new = bio_alloc(gfp, nr_pages);
|
|
|
|
|
|
|
|
if (bio) {
|
|
|
|
bio_chain(bio, new);
|
|
|
|
submit_bio(rw, bio);
|
|
|
|
}
|
2011-05-06 19:26:27 -06:00
|
|
|
|
2016-04-16 12:55:27 -06:00
|
|
|
return new;
|
2010-04-28 07:55:08 -06:00
|
|
|
}
|
|
|
|
|
2016-04-16 12:55:28 -06:00
|
|
|
int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
|
|
|
|
sector_t nr_sects, gfp_t gfp_mask, int type, struct bio **biop)
|
2010-04-28 07:55:08 -06:00
|
|
|
{
|
|
|
|
struct request_queue *q = bdev_get_queue(bdev);
|
2016-04-16 12:55:28 -06:00
|
|
|
struct bio *bio = *biop;
|
2015-10-22 10:59:42 -06:00
|
|
|
unsigned int granularity;
|
|
|
|
int alignment;
|
2010-04-28 07:55:08 -06:00
|
|
|
|
|
|
|
if (!q)
|
|
|
|
return -ENXIO;
|
|
|
|
if (!blk_queue_discard(q))
|
|
|
|
return -EOPNOTSUPP;
|
2016-04-16 12:55:28 -06:00
|
|
|
if ((type & REQ_SECURE) && !blk_queue_secdiscard(q))
|
|
|
|
return -EOPNOTSUPP;
|
2010-04-28 07:55:08 -06:00
|
|
|
|
2015-10-22 10:59:42 -06:00
|
|
|
/* Zero-sector (unknown) and one-sector granularities are the same. */
|
|
|
|
granularity = max(q->limits.discard_granularity >> 9, 1U);
|
|
|
|
alignment = (bdev_discard_alignment(bdev) >> 9) % granularity;
|
|
|
|
|
2011-05-06 19:26:27 -06:00
|
|
|
while (nr_sects) {
|
block: split discard into aligned requests
When a disk has large discard_granularity and small max_discard_sectors,
discards are not split with optimal alignment. In the limit case of
discard_granularity == max_discard_sectors, no request could be aligned
correctly, so in fact you might end up with no discarded logical blocks
at all.
Another example that helps showing the condition in the patch is with
discard_granularity == 64, max_discard_sectors == 128. A request that is
submitted for 256 sectors 2..257 will be split in two: 2..129, 130..257.
However, only 2 aligned blocks out of 3 are included in the request;
128..191 may be left intact and not discarded. With this patch, the
first request will be truncated to ensure good alignment of what's left,
and the split will be 2..127, 128..255, 256..257. The patch will also
take into account the discard_alignment.
At most one extra request will be introduced, because the first request
will be reduced by at most granularity-1 sectors, and granularity
must be less than max_discard_sectors. Subsequent requests will run
on round_down(max_discard_sectors, granularity) sectors, as in the
current code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Acked-by: Vivek Goyal <vgoyal@redhat.com>
Tested-by: Mike Snitzer <snitzer@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-02 01:48:50 -06:00
|
|
|
unsigned int req_sects;
|
2015-10-22 10:59:42 -06:00
|
|
|
sector_t end_sect, tmp;
|
block: split discard into aligned requests
When a disk has large discard_granularity and small max_discard_sectors,
discards are not split with optimal alignment. In the limit case of
discard_granularity == max_discard_sectors, no request could be aligned
correctly, so in fact you might end up with no discarded logical blocks
at all.
Another example that helps showing the condition in the patch is with
discard_granularity == 64, max_discard_sectors == 128. A request that is
submitted for 256 sectors 2..257 will be split in two: 2..129, 130..257.
However, only 2 aligned blocks out of 3 are included in the request;
128..191 may be left intact and not discarded. With this patch, the
first request will be truncated to ensure good alignment of what's left,
and the split will be 2..127, 128..255, 256..257. The patch will also
take into account the discard_alignment.
At most one extra request will be introduced, because the first request
will be reduced by at most granularity-1 sectors, and granularity
must be less than max_discard_sectors. Subsequent requests will run
on round_down(max_discard_sectors, granularity) sectors, as in the
current code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Acked-by: Vivek Goyal <vgoyal@redhat.com>
Tested-by: Mike Snitzer <snitzer@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-02 01:48:50 -06:00
|
|
|
|
2015-10-22 10:59:42 -06:00
|
|
|
/* Make sure bi_size doesn't overflow */
|
|
|
|
req_sects = min_t(sector_t, nr_sects, UINT_MAX >> 9);
|
|
|
|
|
2016-04-16 12:55:27 -06:00
|
|
|
/**
|
2015-10-22 10:59:42 -06:00
|
|
|
* If splitting a request, and the next starting sector would be
|
|
|
|
* misaligned, stop the discard at the previous aligned sector.
|
|
|
|
*/
|
block: split discard into aligned requests
When a disk has large discard_granularity and small max_discard_sectors,
discards are not split with optimal alignment. In the limit case of
discard_granularity == max_discard_sectors, no request could be aligned
correctly, so in fact you might end up with no discarded logical blocks
at all.
Another example that helps showing the condition in the patch is with
discard_granularity == 64, max_discard_sectors == 128. A request that is
submitted for 256 sectors 2..257 will be split in two: 2..129, 130..257.
However, only 2 aligned blocks out of 3 are included in the request;
128..191 may be left intact and not discarded. With this patch, the
first request will be truncated to ensure good alignment of what's left,
and the split will be 2..127, 128..255, 256..257. The patch will also
take into account the discard_alignment.
At most one extra request will be introduced, because the first request
will be reduced by at most granularity-1 sectors, and granularity
must be less than max_discard_sectors. Subsequent requests will run
on round_down(max_discard_sectors, granularity) sectors, as in the
current code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Acked-by: Vivek Goyal <vgoyal@redhat.com>
Tested-by: Mike Snitzer <snitzer@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-02 01:48:50 -06:00
|
|
|
end_sect = sector + req_sects;
|
2015-10-22 10:59:42 -06:00
|
|
|
tmp = end_sect;
|
|
|
|
if (req_sects < nr_sects &&
|
|
|
|
sector_div(tmp, granularity) != alignment) {
|
|
|
|
end_sect = end_sect - alignment;
|
|
|
|
sector_div(end_sect, granularity);
|
|
|
|
end_sect = end_sect * granularity + alignment;
|
|
|
|
req_sects = end_sect - sector;
|
|
|
|
}
|
block: split discard into aligned requests
When a disk has large discard_granularity and small max_discard_sectors,
discards are not split with optimal alignment. In the limit case of
discard_granularity == max_discard_sectors, no request could be aligned
correctly, so in fact you might end up with no discarded logical blocks
at all.
Another example that helps showing the condition in the patch is with
discard_granularity == 64, max_discard_sectors == 128. A request that is
submitted for 256 sectors 2..257 will be split in two: 2..129, 130..257.
However, only 2 aligned blocks out of 3 are included in the request;
128..191 may be left intact and not discarded. With this patch, the
first request will be truncated to ensure good alignment of what's left,
and the split will be 2..127, 128..255, 256..257. The patch will also
take into account the discard_alignment.
At most one extra request will be introduced, because the first request
will be reduced by at most granularity-1 sectors, and granularity
must be less than max_discard_sectors. Subsequent requests will run
on round_down(max_discard_sectors, granularity) sectors, as in the
current code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Acked-by: Vivek Goyal <vgoyal@redhat.com>
Tested-by: Mike Snitzer <snitzer@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-02 01:48:50 -06:00
|
|
|
|
2016-04-16 12:55:27 -06:00
|
|
|
bio = next_bio(bio, type, 1, gfp_mask);
|
2013-10-11 16:44:27 -06:00
|
|
|
bio->bi_iter.bi_sector = sector;
|
2010-04-28 07:55:08 -06:00
|
|
|
bio->bi_bdev = bdev;
|
|
|
|
|
2013-10-11 16:44:27 -06:00
|
|
|
bio->bi_iter.bi_size = req_sects << 9;
|
block: split discard into aligned requests
When a disk has large discard_granularity and small max_discard_sectors,
discards are not split with optimal alignment. In the limit case of
discard_granularity == max_discard_sectors, no request could be aligned
correctly, so in fact you might end up with no discarded logical blocks
at all.
Another example that helps showing the condition in the patch is with
discard_granularity == 64, max_discard_sectors == 128. A request that is
submitted for 256 sectors 2..257 will be split in two: 2..129, 130..257.
However, only 2 aligned blocks out of 3 are included in the request;
128..191 may be left intact and not discarded. With this patch, the
first request will be truncated to ensure good alignment of what's left,
and the split will be 2..127, 128..255, 256..257. The patch will also
take into account the discard_alignment.
At most one extra request will be introduced, because the first request
will be reduced by at most granularity-1 sectors, and granularity
must be less than max_discard_sectors. Subsequent requests will run
on round_down(max_discard_sectors, granularity) sectors, as in the
current code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Acked-by: Vivek Goyal <vgoyal@redhat.com>
Tested-by: Mike Snitzer <snitzer@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-02 01:48:50 -06:00
|
|
|
nr_sects -= req_sects;
|
|
|
|
sector = end_sect;
|
2010-04-28 07:55:08 -06:00
|
|
|
|
2014-02-12 09:34:01 -07:00
|
|
|
/*
|
|
|
|
* We can loop for a long time in here, if someone does
|
|
|
|
* full device discards (like mkfs). Be nice and allow
|
|
|
|
* us to schedule out to avoid softlocking if preempt
|
|
|
|
* is disabled.
|
|
|
|
*/
|
|
|
|
cond_resched();
|
2011-05-06 19:26:27 -06:00
|
|
|
}
|
2016-04-16 12:55:28 -06:00
|
|
|
|
|
|
|
*biop = bio;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__blkdev_issue_discard);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* blkdev_issue_discard - queue a discard
|
|
|
|
* @bdev: blockdev to issue discard for
|
|
|
|
* @sector: start sector
|
|
|
|
* @nr_sects: number of sectors to discard
|
|
|
|
* @gfp_mask: memory allocation flags (for bio_alloc)
|
|
|
|
* @flags: BLKDEV_IFL_* flags to control behaviour
|
|
|
|
*
|
|
|
|
* Description:
|
|
|
|
* Issue a discard request for the sectors in question.
|
|
|
|
*/
|
|
|
|
int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
|
|
|
|
sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
|
|
|
|
{
|
|
|
|
int type = REQ_WRITE | REQ_DISCARD;
|
|
|
|
struct bio *bio = NULL;
|
|
|
|
struct blk_plug plug;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (flags & BLKDEV_DISCARD_SECURE)
|
|
|
|
type |= REQ_SECURE;
|
|
|
|
|
|
|
|
blk_start_plug(&plug);
|
|
|
|
ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, type,
|
|
|
|
&bio);
|
2016-05-05 09:54:21 -06:00
|
|
|
if (!ret && bio) {
|
2016-04-16 12:55:27 -06:00
|
|
|
ret = submit_bio_wait(type, bio);
|
2016-05-05 09:54:21 -06:00
|
|
|
if (ret == -EOPNOTSUPP)
|
|
|
|
ret = 0;
|
2016-06-07 10:32:13 -06:00
|
|
|
bio_put(bio);
|
2016-05-05 09:54:21 -06:00
|
|
|
}
|
2012-12-13 20:15:51 -07:00
|
|
|
blk_finish_plug(&plug);
|
2010-04-28 07:55:08 -06:00
|
|
|
|
2016-05-05 09:54:21 -06:00
|
|
|
return ret;
|
2010-04-28 07:55:08 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(blkdev_issue_discard);
|
2010-04-28 07:55:09 -06:00
|
|
|
|
2012-09-18 10:19:27 -06:00
|
|
|
/**
|
|
|
|
* blkdev_issue_write_same - queue a write same operation
|
|
|
|
* @bdev: target blockdev
|
|
|
|
* @sector: start sector
|
|
|
|
* @nr_sects: number of sectors to write
|
|
|
|
* @gfp_mask: memory allocation flags (for bio_alloc)
|
|
|
|
* @page: page containing data to write
|
|
|
|
*
|
|
|
|
* Description:
|
|
|
|
* Issue a write same request for the sectors in question.
|
|
|
|
*/
|
|
|
|
int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
|
|
|
|
sector_t nr_sects, gfp_t gfp_mask,
|
|
|
|
struct page *page)
|
|
|
|
{
|
|
|
|
struct request_queue *q = bdev_get_queue(bdev);
|
|
|
|
unsigned int max_write_same_sectors;
|
2016-04-16 12:55:27 -06:00
|
|
|
struct bio *bio = NULL;
|
2012-09-18 10:19:27 -06:00
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
if (!q)
|
|
|
|
return -ENXIO;
|
|
|
|
|
2015-05-22 01:46:56 -06:00
|
|
|
/* Ensure that max_write_same_sectors doesn't overflow bi_size */
|
|
|
|
max_write_same_sectors = UINT_MAX >> 9;
|
2012-09-18 10:19:27 -06:00
|
|
|
|
|
|
|
while (nr_sects) {
|
2016-04-16 12:55:27 -06:00
|
|
|
bio = next_bio(bio, REQ_WRITE | REQ_WRITE_SAME, 1, gfp_mask);
|
2013-10-11 16:44:27 -06:00
|
|
|
bio->bi_iter.bi_sector = sector;
|
2012-09-18 10:19:27 -06:00
|
|
|
bio->bi_bdev = bdev;
|
|
|
|
bio->bi_vcnt = 1;
|
|
|
|
bio->bi_io_vec->bv_page = page;
|
|
|
|
bio->bi_io_vec->bv_offset = 0;
|
|
|
|
bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev);
|
|
|
|
|
|
|
|
if (nr_sects > max_write_same_sectors) {
|
2013-10-11 16:44:27 -06:00
|
|
|
bio->bi_iter.bi_size = max_write_same_sectors << 9;
|
2012-09-18 10:19:27 -06:00
|
|
|
nr_sects -= max_write_same_sectors;
|
|
|
|
sector += max_write_same_sectors;
|
|
|
|
} else {
|
2013-10-11 16:44:27 -06:00
|
|
|
bio->bi_iter.bi_size = nr_sects << 9;
|
2012-09-18 10:19:27 -06:00
|
|
|
nr_sects = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-07 10:32:13 -06:00
|
|
|
if (bio) {
|
2016-04-16 12:55:27 -06:00
|
|
|
ret = submit_bio_wait(REQ_WRITE | REQ_WRITE_SAME, bio);
|
2016-06-07 10:32:13 -06:00
|
|
|
bio_put(bio);
|
|
|
|
}
|
2016-04-16 12:55:27 -06:00
|
|
|
return ret != -EOPNOTSUPP ? ret : 0;
|
2012-09-18 10:19:27 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(blkdev_issue_write_same);
|
|
|
|
|
2010-04-28 07:55:09 -06:00
|
|
|
/**
|
2011-03-01 11:45:24 -07:00
|
|
|
* blkdev_issue_zeroout - generate number of zero filed write bios
|
2010-04-28 07:55:09 -06:00
|
|
|
* @bdev: blockdev to issue
|
|
|
|
* @sector: start sector
|
|
|
|
* @nr_sects: number of sectors to write
|
|
|
|
* @gfp_mask: memory allocation flags (for bio_alloc)
|
|
|
|
*
|
|
|
|
* Description:
|
|
|
|
* Generate and issue number of bios with zerofiled pages.
|
|
|
|
*/
|
|
|
|
|
2014-05-26 14:19:14 -06:00
|
|
|
static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
|
|
|
|
sector_t nr_sects, gfp_t gfp_mask)
|
2010-04-28 07:55:09 -06:00
|
|
|
{
|
2010-08-06 05:23:25 -06:00
|
|
|
int ret;
|
2016-04-16 12:55:27 -06:00
|
|
|
struct bio *bio = NULL;
|
2011-03-11 02:23:53 -07:00
|
|
|
unsigned int sz;
|
2010-04-28 07:55:09 -06:00
|
|
|
|
|
|
|
while (nr_sects != 0) {
|
2016-04-16 12:55:27 -06:00
|
|
|
bio = next_bio(bio, WRITE,
|
|
|
|
min(nr_sects, (sector_t)BIO_MAX_PAGES),
|
|
|
|
gfp_mask);
|
2013-10-11 16:44:27 -06:00
|
|
|
bio->bi_iter.bi_sector = sector;
|
2010-04-28 07:55:09 -06:00
|
|
|
bio->bi_bdev = bdev;
|
|
|
|
|
2010-04-29 01:28:21 -06:00
|
|
|
while (nr_sects != 0) {
|
|
|
|
sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects);
|
2010-04-28 07:55:09 -06:00
|
|
|
ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0);
|
|
|
|
nr_sects -= ret >> 9;
|
|
|
|
sector += ret >> 9;
|
|
|
|
if (ret < (sz << 9))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-07 10:32:13 -06:00
|
|
|
if (bio) {
|
|
|
|
ret = submit_bio_wait(WRITE, bio);
|
|
|
|
bio_put(bio);
|
|
|
|
return ret;
|
|
|
|
}
|
2016-04-16 12:55:27 -06:00
|
|
|
return 0;
|
2010-04-28 07:55:09 -06:00
|
|
|
}
|
2012-09-18 10:19:28 -06:00
|
|
|
|
|
|
|
/**
|
|
|
|
* blkdev_issue_zeroout - zero-fill a block range
|
|
|
|
* @bdev: blockdev to write
|
|
|
|
* @sector: start sector
|
|
|
|
* @nr_sects: number of sectors to write
|
|
|
|
* @gfp_mask: memory allocation flags (for bio_alloc)
|
2015-01-20 18:06:30 -07:00
|
|
|
* @discard: whether to discard the block range
|
2012-09-18 10:19:28 -06:00
|
|
|
*
|
|
|
|
* Description:
|
2015-01-20 18:06:30 -07:00
|
|
|
* Zero-fill a block range. If the discard flag is set and the block
|
|
|
|
* device guarantees that subsequent READ operations to the block range
|
|
|
|
* in question will return zeroes, the blocks will be discarded. Should
|
|
|
|
* the discard request fail, if the discard flag is not set, or if
|
|
|
|
* discard_zeroes_data is not supported, this function will resort to
|
|
|
|
* zeroing the blocks manually, thus provisioning (allocating,
|
|
|
|
* anchoring) them. If the block device supports the WRITE SAME command
|
|
|
|
* blkdev_issue_zeroout() will use it to optimize the process of
|
|
|
|
* clearing the block range. Otherwise the zeroing will be performed
|
|
|
|
* using regular WRITE calls.
|
2012-09-18 10:19:28 -06:00
|
|
|
*/
|
|
|
|
|
|
|
|
int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
|
2015-01-20 18:06:30 -07:00
|
|
|
sector_t nr_sects, gfp_t gfp_mask, bool discard)
|
2012-09-18 10:19:28 -06:00
|
|
|
{
|
2015-01-20 18:06:30 -07:00
|
|
|
struct request_queue *q = bdev_get_queue(bdev);
|
|
|
|
|
2015-02-05 10:14:54 -07:00
|
|
|
if (discard && blk_queue_discard(q) && q->limits.discard_zeroes_data &&
|
|
|
|
blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, 0) == 0)
|
|
|
|
return 0;
|
2015-01-20 18:06:30 -07:00
|
|
|
|
2015-02-05 10:14:54 -07:00
|
|
|
if (bdev_write_same(bdev) &&
|
|
|
|
blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask,
|
|
|
|
ZERO_PAGE(0)) == 0)
|
|
|
|
return 0;
|
2012-09-18 10:19:28 -06:00
|
|
|
|
|
|
|
return __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask);
|
|
|
|
}
|
2010-04-28 07:55:09 -06:00
|
|
|
EXPORT_SYMBOL(blkdev_issue_zeroout);
|