![Ming Lei](/assets/img/avatar_default.png)
loop is one block device, for any bio submitted to this device, the upper layer does guarantee that pages added to loop's bio won't go away when the bio is in-flight. So mark loop's bvec as ITER_BVEC_FLAG_NO_REF then get_page/put_page can be saved for serving loop's IO. Cc: linux-fsdevel@vger.kernel.org Cc: Christoph Hellwig <hch@infradead.org> Signed-off-by: Ming Lei <ming.lei@redhat.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
2298 lines
56 KiB
C
2298 lines
56 KiB
C
/*
|
|
* linux/drivers/block/loop.c
|
|
*
|
|
* Written by Theodore Ts'o, 3/29/93
|
|
*
|
|
* Copyright 1993 by Theodore Ts'o. Redistribution of this file is
|
|
* permitted under the GNU General Public License.
|
|
*
|
|
* DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993
|
|
* more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996
|
|
*
|
|
* Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994
|
|
* Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996
|
|
*
|
|
* Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997
|
|
*
|
|
* Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998
|
|
*
|
|
* Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998
|
|
*
|
|
* Loadable modules and other fixes by AK, 1998
|
|
*
|
|
* Make real block number available to downstream transfer functions, enables
|
|
* CBC (and relatives) mode encryption requiring unique IVs per data block.
|
|
* Reed H. Petty, rhp@draper.net
|
|
*
|
|
* Maximum number of loop devices now dynamic via max_loop module parameter.
|
|
* Russell Kroll <rkroll@exploits.org> 19990701
|
|
*
|
|
* Maximum number of loop devices when compiled-in now selectable by passing
|
|
* max_loop=<1-255> to the kernel on boot.
|
|
* Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999
|
|
*
|
|
* Completely rewrite request handling to be make_request_fn style and
|
|
* non blocking, pushing work to a helper thread. Lots of fixes from
|
|
* Al Viro too.
|
|
* Jens Axboe <axboe@suse.de>, Nov 2000
|
|
*
|
|
* Support up to 256 loop devices
|
|
* Heinz Mauelshagen <mge@sistina.com>, Feb 2002
|
|
*
|
|
* Support for falling back on the write file operation when the address space
|
|
* operations write_begin is not available on the backing filesystem.
|
|
* Anton Altaparmakov, 16 Feb 2005
|
|
*
|
|
* Still To Fix:
|
|
* - Advisory locking is ignored here.
|
|
* - Should use an own CAP_* category instead of CAP_SYS_ADMIN
|
|
*
|
|
*/
|
|
|
|
#include <linux/module.h>
|
|
#include <linux/moduleparam.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/file.h>
|
|
#include <linux/stat.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/major.h>
|
|
#include <linux/wait.h>
|
|
#include <linux/blkdev.h>
|
|
#include <linux/blkpg.h>
|
|
#include <linux/init.h>
|
|
#include <linux/swap.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/compat.h>
|
|
#include <linux/suspend.h>
|
|
#include <linux/freezer.h>
|
|
#include <linux/mutex.h>
|
|
#include <linux/writeback.h>
|
|
#include <linux/completion.h>
|
|
#include <linux/highmem.h>
|
|
#include <linux/kthread.h>
|
|
#include <linux/splice.h>
|
|
#include <linux/sysfs.h>
|
|
#include <linux/miscdevice.h>
|
|
#include <linux/falloc.h>
|
|
#include <linux/uio.h>
|
|
#include <linux/ioprio.h>
|
|
#include <linux/blk-cgroup.h>
|
|
|
|
#include "loop.h"
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
static DEFINE_IDR(loop_index_idr);
|
|
static DEFINE_MUTEX(loop_ctl_mutex);
|
|
|
|
static int max_part;
|
|
static int part_shift;
|
|
|
|
static int transfer_xor(struct loop_device *lo, int cmd,
|
|
struct page *raw_page, unsigned raw_off,
|
|
struct page *loop_page, unsigned loop_off,
|
|
int size, sector_t real_block)
|
|
{
|
|
char *raw_buf = kmap_atomic(raw_page) + raw_off;
|
|
char *loop_buf = kmap_atomic(loop_page) + loop_off;
|
|
char *in, *out, *key;
|
|
int i, keysize;
|
|
|
|
if (cmd == READ) {
|
|
in = raw_buf;
|
|
out = loop_buf;
|
|
} else {
|
|
in = loop_buf;
|
|
out = raw_buf;
|
|
}
|
|
|
|
key = lo->lo_encrypt_key;
|
|
keysize = lo->lo_encrypt_key_size;
|
|
for (i = 0; i < size; i++)
|
|
*out++ = *in++ ^ key[(i & 511) % keysize];
|
|
|
|
kunmap_atomic(loop_buf);
|
|
kunmap_atomic(raw_buf);
|
|
cond_resched();
|
|
return 0;
|
|
}
|
|
|
|
static int xor_init(struct loop_device *lo, const struct loop_info64 *info)
|
|
{
|
|
if (unlikely(info->lo_encrypt_key_size <= 0))
|
|
return -EINVAL;
|
|
return 0;
|
|
}
|
|
|
|
static struct loop_func_table none_funcs = {
|
|
.number = LO_CRYPT_NONE,
|
|
};
|
|
|
|
static struct loop_func_table xor_funcs = {
|
|
.number = LO_CRYPT_XOR,
|
|
.transfer = transfer_xor,
|
|
.init = xor_init
|
|
};
|
|
|
|
/* xfer_funcs[0] is special - its release function is never called */
|
|
static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = {
|
|
&none_funcs,
|
|
&xor_funcs
|
|
};
|
|
|
|
static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file)
|
|
{
|
|
loff_t loopsize;
|
|
|
|
/* Compute loopsize in bytes */
|
|
loopsize = i_size_read(file->f_mapping->host);
|
|
if (offset > 0)
|
|
loopsize -= offset;
|
|
/* offset is beyond i_size, weird but possible */
|
|
if (loopsize < 0)
|
|
return 0;
|
|
|
|
if (sizelimit > 0 && sizelimit < loopsize)
|
|
loopsize = sizelimit;
|
|
/*
|
|
* Unfortunately, if we want to do I/O on the device,
|
|
* the number of 512-byte sectors has to fit into a sector_t.
|
|
*/
|
|
return loopsize >> 9;
|
|
}
|
|
|
|
static loff_t get_loop_size(struct loop_device *lo, struct file *file)
|
|
{
|
|
return get_size(lo->lo_offset, lo->lo_sizelimit, file);
|
|
}
|
|
|
|
static void __loop_update_dio(struct loop_device *lo, bool dio)
|
|
{
|
|
struct file *file = lo->lo_backing_file;
|
|
struct address_space *mapping = file->f_mapping;
|
|
struct inode *inode = mapping->host;
|
|
unsigned short sb_bsize = 0;
|
|
unsigned dio_align = 0;
|
|
bool use_dio;
|
|
|
|
if (inode->i_sb->s_bdev) {
|
|
sb_bsize = bdev_logical_block_size(inode->i_sb->s_bdev);
|
|
dio_align = sb_bsize - 1;
|
|
}
|
|
|
|
/*
|
|
* We support direct I/O only if lo_offset is aligned with the
|
|
* logical I/O size of backing device, and the logical block
|
|
* size of loop is bigger than the backing device's and the loop
|
|
* needn't transform transfer.
|
|
*
|
|
* TODO: the above condition may be loosed in the future, and
|
|
* direct I/O may be switched runtime at that time because most
|
|
* of requests in sane applications should be PAGE_SIZE aligned
|
|
*/
|
|
if (dio) {
|
|
if (queue_logical_block_size(lo->lo_queue) >= sb_bsize &&
|
|
!(lo->lo_offset & dio_align) &&
|
|
mapping->a_ops->direct_IO &&
|
|
!lo->transfer)
|
|
use_dio = true;
|
|
else
|
|
use_dio = false;
|
|
} else {
|
|
use_dio = false;
|
|
}
|
|
|
|
if (lo->use_dio == use_dio)
|
|
return;
|
|
|
|
/* flush dirty pages before changing direct IO */
|
|
vfs_fsync(file, 0);
|
|
|
|
/*
|
|
* The flag of LO_FLAGS_DIRECT_IO is handled similarly with
|
|
* LO_FLAGS_READ_ONLY, both are set from kernel, and losetup
|
|
* will get updated by ioctl(LOOP_GET_STATUS)
|
|
*/
|
|
blk_mq_freeze_queue(lo->lo_queue);
|
|
lo->use_dio = use_dio;
|
|
if (use_dio) {
|
|
blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, lo->lo_queue);
|
|
lo->lo_flags |= LO_FLAGS_DIRECT_IO;
|
|
} else {
|
|
blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue);
|
|
lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
|
|
}
|
|
blk_mq_unfreeze_queue(lo->lo_queue);
|
|
}
|
|
|
|
static int
|
|
figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit)
|
|
{
|
|
loff_t size = get_size(offset, sizelimit, lo->lo_backing_file);
|
|
sector_t x = (sector_t)size;
|
|
struct block_device *bdev = lo->lo_device;
|
|
|
|
if (unlikely((loff_t)x != size))
|
|
return -EFBIG;
|
|
if (lo->lo_offset != offset)
|
|
lo->lo_offset = offset;
|
|
if (lo->lo_sizelimit != sizelimit)
|
|
lo->lo_sizelimit = sizelimit;
|
|
set_capacity(lo->lo_disk, x);
|
|
bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9);
|
|
/* let user-space know about the new size */
|
|
kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
|
|
return 0;
|
|
}
|
|
|
|
static inline int
|
|
lo_do_transfer(struct loop_device *lo, int cmd,
|
|
struct page *rpage, unsigned roffs,
|
|
struct page *lpage, unsigned loffs,
|
|
int size, sector_t rblock)
|
|
{
|
|
int ret;
|
|
|
|
ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock);
|
|
if (likely(!ret))
|
|
return 0;
|
|
|
|
printk_ratelimited(KERN_ERR
|
|
"loop: Transfer error at byte offset %llu, length %i.\n",
|
|
(unsigned long long)rblock << 9, size);
|
|
return ret;
|
|
}
|
|
|
|
static inline void loop_iov_iter_bvec(struct iov_iter *i,
|
|
unsigned int direction, const struct bio_vec *bvec,
|
|
unsigned long nr_segs, size_t count)
|
|
{
|
|
iov_iter_bvec(i, direction, bvec, nr_segs, count);
|
|
i->type |= ITER_BVEC_FLAG_NO_REF;
|
|
}
|
|
|
|
static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
|
|
{
|
|
struct iov_iter i;
|
|
ssize_t bw;
|
|
|
|
loop_iov_iter_bvec(&i, WRITE, bvec, 1, bvec->bv_len);
|
|
|
|
file_start_write(file);
|
|
bw = vfs_iter_write(file, &i, ppos, 0);
|
|
file_end_write(file);
|
|
|
|
if (likely(bw == bvec->bv_len))
|
|
return 0;
|
|
|
|
printk_ratelimited(KERN_ERR
|
|
"loop: Write error at byte offset %llu, length %i.\n",
|
|
(unsigned long long)*ppos, bvec->bv_len);
|
|
if (bw >= 0)
|
|
bw = -EIO;
|
|
return bw;
|
|
}
|
|
|
|
static int lo_write_simple(struct loop_device *lo, struct request *rq,
|
|
loff_t pos)
|
|
{
|
|
struct bio_vec bvec;
|
|
struct req_iterator iter;
|
|
int ret = 0;
|
|
|
|
rq_for_each_segment(bvec, rq, iter) {
|
|
ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos);
|
|
if (ret < 0)
|
|
break;
|
|
cond_resched();
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* This is the slow, transforming version that needs to double buffer the
|
|
* data as it cannot do the transformations in place without having direct
|
|
* access to the destination pages of the backing file.
|
|
*/
|
|
static int lo_write_transfer(struct loop_device *lo, struct request *rq,
|
|
loff_t pos)
|
|
{
|
|
struct bio_vec bvec, b;
|
|
struct req_iterator iter;
|
|
struct page *page;
|
|
int ret = 0;
|
|
|
|
page = alloc_page(GFP_NOIO);
|
|
if (unlikely(!page))
|
|
return -ENOMEM;
|
|
|
|
rq_for_each_segment(bvec, rq, iter) {
|
|
ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page,
|
|
bvec.bv_offset, bvec.bv_len, pos >> 9);
|
|
if (unlikely(ret))
|
|
break;
|
|
|
|
b.bv_page = page;
|
|
b.bv_offset = 0;
|
|
b.bv_len = bvec.bv_len;
|
|
ret = lo_write_bvec(lo->lo_backing_file, &b, &pos);
|
|
if (ret < 0)
|
|
break;
|
|
}
|
|
|
|
__free_page(page);
|
|
return ret;
|
|
}
|
|
|
|
static int lo_read_simple(struct loop_device *lo, struct request *rq,
|
|
loff_t pos)
|
|
{
|
|
struct bio_vec bvec;
|
|
struct req_iterator iter;
|
|
struct iov_iter i;
|
|
ssize_t len;
|
|
|
|
rq_for_each_segment(bvec, rq, iter) {
|
|
loop_iov_iter_bvec(&i, READ, &bvec, 1, bvec.bv_len);
|
|
len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
|
|
if (len < 0)
|
|
return len;
|
|
|
|
flush_dcache_page(bvec.bv_page);
|
|
|
|
if (len != bvec.bv_len) {
|
|
struct bio *bio;
|
|
|
|
__rq_for_each_bio(bio, rq)
|
|
zero_fill_bio(bio);
|
|
break;
|
|
}
|
|
cond_resched();
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int lo_read_transfer(struct loop_device *lo, struct request *rq,
|
|
loff_t pos)
|
|
{
|
|
struct bio_vec bvec, b;
|
|
struct req_iterator iter;
|
|
struct iov_iter i;
|
|
struct page *page;
|
|
ssize_t len;
|
|
int ret = 0;
|
|
|
|
page = alloc_page(GFP_NOIO);
|
|
if (unlikely(!page))
|
|
return -ENOMEM;
|
|
|
|
rq_for_each_segment(bvec, rq, iter) {
|
|
loff_t offset = pos;
|
|
|
|
b.bv_page = page;
|
|
b.bv_offset = 0;
|
|
b.bv_len = bvec.bv_len;
|
|
|
|
loop_iov_iter_bvec(&i, READ, &b, 1, b.bv_len);
|
|
len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
|
|
if (len < 0) {
|
|
ret = len;
|
|
goto out_free_page;
|
|
}
|
|
|
|
ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page,
|
|
bvec.bv_offset, len, offset >> 9);
|
|
if (ret)
|
|
goto out_free_page;
|
|
|
|
flush_dcache_page(bvec.bv_page);
|
|
|
|
if (len != bvec.bv_len) {
|
|
struct bio *bio;
|
|
|
|
__rq_for_each_bio(bio, rq)
|
|
zero_fill_bio(bio);
|
|
break;
|
|
}
|
|
}
|
|
|
|
ret = 0;
|
|
out_free_page:
|
|
__free_page(page);
|
|
return ret;
|
|
}
|
|
|
|
static int lo_discard(struct loop_device *lo, struct request *rq, loff_t pos)
|
|
{
|
|
/*
|
|
* We use punch hole to reclaim the free space used by the
|
|
* image a.k.a. discard. However we do not support discard if
|
|
* encryption is enabled, because it may give an attacker
|
|
* useful information.
|
|
*/
|
|
struct file *file = lo->lo_backing_file;
|
|
int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
|
|
int ret;
|
|
|
|
if ((!file->f_op->fallocate) || lo->lo_encrypt_key_size) {
|
|
ret = -EOPNOTSUPP;
|
|
goto out;
|
|
}
|
|
|
|
ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
|
|
if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP))
|
|
ret = -EIO;
|
|
out:
|
|
return ret;
|
|
}
|
|
|
|
static int lo_req_flush(struct loop_device *lo, struct request *rq)
|
|
{
|
|
struct file *file = lo->lo_backing_file;
|
|
int ret = vfs_fsync(file, 0);
|
|
if (unlikely(ret && ret != -EINVAL))
|
|
ret = -EIO;
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void lo_complete_rq(struct request *rq)
|
|
{
|
|
struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
|
|
blk_status_t ret = BLK_STS_OK;
|
|
|
|
if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) ||
|
|
req_op(rq) != REQ_OP_READ) {
|
|
if (cmd->ret < 0)
|
|
ret = BLK_STS_IOERR;
|
|
goto end_io;
|
|
}
|
|
|
|
/*
|
|
* Short READ - if we got some data, advance our request and
|
|
* retry it. If we got no data, end the rest with EIO.
|
|
*/
|
|
if (cmd->ret) {
|
|
blk_update_request(rq, BLK_STS_OK, cmd->ret);
|
|
cmd->ret = 0;
|
|
blk_mq_requeue_request(rq, true);
|
|
} else {
|
|
if (cmd->use_aio) {
|
|
struct bio *bio = rq->bio;
|
|
|
|
while (bio) {
|
|
zero_fill_bio(bio);
|
|
bio = bio->bi_next;
|
|
}
|
|
}
|
|
ret = BLK_STS_IOERR;
|
|
end_io:
|
|
blk_mq_end_request(rq, ret);
|
|
}
|
|
}
|
|
|
|
static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
|
|
{
|
|
struct request *rq = blk_mq_rq_from_pdu(cmd);
|
|
|
|
if (!atomic_dec_and_test(&cmd->ref))
|
|
return;
|
|
kfree(cmd->bvec);
|
|
cmd->bvec = NULL;
|
|
blk_mq_complete_request(rq);
|
|
}
|
|
|
|
static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
|
|
{
|
|
struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);
|
|
|
|
if (cmd->css)
|
|
css_put(cmd->css);
|
|
cmd->ret = ret;
|
|
lo_rw_aio_do_completion(cmd);
|
|
}
|
|
|
|
static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
|
|
loff_t pos, bool rw)
|
|
{
|
|
struct iov_iter iter;
|
|
struct req_iterator rq_iter;
|
|
struct bio_vec *bvec;
|
|
struct request *rq = blk_mq_rq_from_pdu(cmd);
|
|
struct bio *bio = rq->bio;
|
|
struct file *file = lo->lo_backing_file;
|
|
struct bio_vec tmp;
|
|
unsigned int offset;
|
|
int nr_bvec = 0;
|
|
int ret;
|
|
|
|
rq_for_each_bvec(tmp, rq, rq_iter)
|
|
nr_bvec++;
|
|
|
|
if (rq->bio != rq->biotail) {
|
|
|
|
bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec),
|
|
GFP_NOIO);
|
|
if (!bvec)
|
|
return -EIO;
|
|
cmd->bvec = bvec;
|
|
|
|
/*
|
|
* The bios of the request may be started from the middle of
|
|
* the 'bvec' because of bio splitting, so we can't directly
|
|
* copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec
|
|
* API will take care of all details for us.
|
|
*/
|
|
rq_for_each_bvec(tmp, rq, rq_iter) {
|
|
*bvec = tmp;
|
|
bvec++;
|
|
}
|
|
bvec = cmd->bvec;
|
|
offset = 0;
|
|
} else {
|
|
/*
|
|
* Same here, this bio may be started from the middle of the
|
|
* 'bvec' because of bio splitting, so offset from the bvec
|
|
* must be passed to iov iterator
|
|
*/
|
|
offset = bio->bi_iter.bi_bvec_done;
|
|
bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
|
|
}
|
|
atomic_set(&cmd->ref, 2);
|
|
|
|
loop_iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq));
|
|
iter.iov_offset = offset;
|
|
|
|
cmd->iocb.ki_pos = pos;
|
|
cmd->iocb.ki_filp = file;
|
|
cmd->iocb.ki_complete = lo_rw_aio_complete;
|
|
cmd->iocb.ki_flags = IOCB_DIRECT;
|
|
cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
|
|
if (cmd->css)
|
|
kthread_associate_blkcg(cmd->css);
|
|
|
|
if (rw == WRITE)
|
|
ret = call_write_iter(file, &cmd->iocb, &iter);
|
|
else
|
|
ret = call_read_iter(file, &cmd->iocb, &iter);
|
|
|
|
lo_rw_aio_do_completion(cmd);
|
|
kthread_associate_blkcg(NULL);
|
|
|
|
if (ret != -EIOCBQUEUED)
|
|
cmd->iocb.ki_complete(&cmd->iocb, ret, 0);
|
|
return 0;
|
|
}
|
|
|
|
static int do_req_filebacked(struct loop_device *lo, struct request *rq)
|
|
{
|
|
struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
|
|
loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
|
|
|
|
/*
|
|
* lo_write_simple and lo_read_simple should have been covered
|
|
* by io submit style function like lo_rw_aio(), one blocker
|
|
* is that lo_read_simple() need to call flush_dcache_page after
|
|
* the page is written from kernel, and it isn't easy to handle
|
|
* this in io submit style function which submits all segments
|
|
* of the req at one time. And direct read IO doesn't need to
|
|
* run flush_dcache_page().
|
|
*/
|
|
switch (req_op(rq)) {
|
|
case REQ_OP_FLUSH:
|
|
return lo_req_flush(lo, rq);
|
|
case REQ_OP_DISCARD:
|
|
case REQ_OP_WRITE_ZEROES:
|
|
return lo_discard(lo, rq, pos);
|
|
case REQ_OP_WRITE:
|
|
if (lo->transfer)
|
|
return lo_write_transfer(lo, rq, pos);
|
|
else if (cmd->use_aio)
|
|
return lo_rw_aio(lo, cmd, pos, WRITE);
|
|
else
|
|
return lo_write_simple(lo, rq, pos);
|
|
case REQ_OP_READ:
|
|
if (lo->transfer)
|
|
return lo_read_transfer(lo, rq, pos);
|
|
else if (cmd->use_aio)
|
|
return lo_rw_aio(lo, cmd, pos, READ);
|
|
else
|
|
return lo_read_simple(lo, rq, pos);
|
|
default:
|
|
WARN_ON_ONCE(1);
|
|
return -EIO;
|
|
}
|
|
}
|
|
|
|
static inline void loop_update_dio(struct loop_device *lo)
|
|
{
|
|
__loop_update_dio(lo, io_is_direct(lo->lo_backing_file) |
|
|
lo->use_dio);
|
|
}
|
|
|
|
static void loop_reread_partitions(struct loop_device *lo,
|
|
struct block_device *bdev)
|
|
{
|
|
int rc;
|
|
|
|
rc = blkdev_reread_part(bdev);
|
|
if (rc)
|
|
pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n",
|
|
__func__, lo->lo_number, lo->lo_file_name, rc);
|
|
}
|
|
|
|
static inline int is_loop_device(struct file *file)
|
|
{
|
|
struct inode *i = file->f_mapping->host;
|
|
|
|
return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR;
|
|
}
|
|
|
|
static int loop_validate_file(struct file *file, struct block_device *bdev)
|
|
{
|
|
struct inode *inode = file->f_mapping->host;
|
|
struct file *f = file;
|
|
|
|
/* Avoid recursion */
|
|
while (is_loop_device(f)) {
|
|
struct loop_device *l;
|
|
|
|
if (f->f_mapping->host->i_bdev == bdev)
|
|
return -EBADF;
|
|
|
|
l = f->f_mapping->host->i_bdev->bd_disk->private_data;
|
|
if (l->lo_state != Lo_bound) {
|
|
return -EINVAL;
|
|
}
|
|
f = l->lo_backing_file;
|
|
}
|
|
if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
|
|
return -EINVAL;
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* loop_change_fd switched the backing store of a loopback device to
|
|
* a new file. This is useful for operating system installers to free up
|
|
* the original file and in High Availability environments to switch to
|
|
* an alternative location for the content in case of server meltdown.
|
|
* This can only work if the loop device is used read-only, and if the
|
|
* new backing store is the same size and type as the old backing store.
|
|
*/
|
|
static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
|
|
unsigned int arg)
|
|
{
|
|
struct file *file = NULL, *old_file;
|
|
int error;
|
|
bool partscan;
|
|
|
|
error = mutex_lock_killable(&loop_ctl_mutex);
|
|
if (error)
|
|
return error;
|
|
error = -ENXIO;
|
|
if (lo->lo_state != Lo_bound)
|
|
goto out_err;
|
|
|
|
/* the loop device has to be read-only */
|
|
error = -EINVAL;
|
|
if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
|
|
goto out_err;
|
|
|
|
error = -EBADF;
|
|
file = fget(arg);
|
|
if (!file)
|
|
goto out_err;
|
|
|
|
error = loop_validate_file(file, bdev);
|
|
if (error)
|
|
goto out_err;
|
|
|
|
old_file = lo->lo_backing_file;
|
|
|
|
error = -EINVAL;
|
|
|
|
/* size of the new backing store needs to be the same */
|
|
if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
|
|
goto out_err;
|
|
|
|
/* and ... switch */
|
|
blk_mq_freeze_queue(lo->lo_queue);
|
|
mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
|
|
lo->lo_backing_file = file;
|
|
lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping);
|
|
mapping_set_gfp_mask(file->f_mapping,
|
|
lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
|
|
loop_update_dio(lo);
|
|
blk_mq_unfreeze_queue(lo->lo_queue);
|
|
partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
/*
|
|
* We must drop file reference outside of loop_ctl_mutex as dropping
|
|
* the file ref can take bd_mutex which creates circular locking
|
|
* dependency.
|
|
*/
|
|
fput(old_file);
|
|
if (partscan)
|
|
loop_reread_partitions(lo, bdev);
|
|
return 0;
|
|
|
|
out_err:
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
if (file)
|
|
fput(file);
|
|
return error;
|
|
}
|
|
|
|
/* loop sysfs attributes */
|
|
|
|
static ssize_t loop_attr_show(struct device *dev, char *page,
|
|
ssize_t (*callback)(struct loop_device *, char *))
|
|
{
|
|
struct gendisk *disk = dev_to_disk(dev);
|
|
struct loop_device *lo = disk->private_data;
|
|
|
|
return callback(lo, page);
|
|
}
|
|
|
|
#define LOOP_ATTR_RO(_name) \
|
|
static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \
|
|
static ssize_t loop_attr_do_show_##_name(struct device *d, \
|
|
struct device_attribute *attr, char *b) \
|
|
{ \
|
|
return loop_attr_show(d, b, loop_attr_##_name##_show); \
|
|
} \
|
|
static struct device_attribute loop_attr_##_name = \
|
|
__ATTR(_name, 0444, loop_attr_do_show_##_name, NULL);
|
|
|
|
static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
|
|
{
|
|
ssize_t ret;
|
|
char *p = NULL;
|
|
|
|
spin_lock_irq(&lo->lo_lock);
|
|
if (lo->lo_backing_file)
|
|
p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1);
|
|
spin_unlock_irq(&lo->lo_lock);
|
|
|
|
if (IS_ERR_OR_NULL(p))
|
|
ret = PTR_ERR(p);
|
|
else {
|
|
ret = strlen(p);
|
|
memmove(buf, p, ret);
|
|
buf[ret++] = '\n';
|
|
buf[ret] = 0;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
|
|
{
|
|
return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset);
|
|
}
|
|
|
|
static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
|
|
{
|
|
return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
|
|
}
|
|
|
|
static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
|
|
{
|
|
int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);
|
|
|
|
return sprintf(buf, "%s\n", autoclear ? "1" : "0");
|
|
}
|
|
|
|
static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
|
|
{
|
|
int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);
|
|
|
|
return sprintf(buf, "%s\n", partscan ? "1" : "0");
|
|
}
|
|
|
|
static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
|
|
{
|
|
int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);
|
|
|
|
return sprintf(buf, "%s\n", dio ? "1" : "0");
|
|
}
|
|
|
|
LOOP_ATTR_RO(backing_file);
|
|
LOOP_ATTR_RO(offset);
|
|
LOOP_ATTR_RO(sizelimit);
|
|
LOOP_ATTR_RO(autoclear);
|
|
LOOP_ATTR_RO(partscan);
|
|
LOOP_ATTR_RO(dio);
|
|
|
|
static struct attribute *loop_attrs[] = {
|
|
&loop_attr_backing_file.attr,
|
|
&loop_attr_offset.attr,
|
|
&loop_attr_sizelimit.attr,
|
|
&loop_attr_autoclear.attr,
|
|
&loop_attr_partscan.attr,
|
|
&loop_attr_dio.attr,
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute_group loop_attribute_group = {
|
|
.name = "loop",
|
|
.attrs= loop_attrs,
|
|
};
|
|
|
|
static void loop_sysfs_init(struct loop_device *lo)
|
|
{
|
|
lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
|
|
&loop_attribute_group);
|
|
}
|
|
|
|
static void loop_sysfs_exit(struct loop_device *lo)
|
|
{
|
|
if (lo->sysfs_inited)
|
|
sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
|
|
&loop_attribute_group);
|
|
}
|
|
|
|
static void loop_config_discard(struct loop_device *lo)
|
|
{
|
|
struct file *file = lo->lo_backing_file;
|
|
struct inode *inode = file->f_mapping->host;
|
|
struct request_queue *q = lo->lo_queue;
|
|
|
|
/*
|
|
* We use punch hole to reclaim the free space used by the
|
|
* image a.k.a. discard. However we do not support discard if
|
|
* encryption is enabled, because it may give an attacker
|
|
* useful information.
|
|
*/
|
|
if ((!file->f_op->fallocate) ||
|
|
lo->lo_encrypt_key_size) {
|
|
q->limits.discard_granularity = 0;
|
|
q->limits.discard_alignment = 0;
|
|
blk_queue_max_discard_sectors(q, 0);
|
|
blk_queue_max_write_zeroes_sectors(q, 0);
|
|
blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q);
|
|
return;
|
|
}
|
|
|
|
q->limits.discard_granularity = inode->i_sb->s_blocksize;
|
|
q->limits.discard_alignment = 0;
|
|
|
|
blk_queue_max_discard_sectors(q, UINT_MAX >> 9);
|
|
blk_queue_max_write_zeroes_sectors(q, UINT_MAX >> 9);
|
|
blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
|
|
}
|
|
|
|
static void loop_unprepare_queue(struct loop_device *lo)
|
|
{
|
|
kthread_flush_worker(&lo->worker);
|
|
kthread_stop(lo->worker_task);
|
|
}
|
|
|
|
static int loop_kthread_worker_fn(void *worker_ptr)
|
|
{
|
|
current->flags |= PF_LESS_THROTTLE;
|
|
return kthread_worker_fn(worker_ptr);
|
|
}
|
|
|
|
static int loop_prepare_queue(struct loop_device *lo)
|
|
{
|
|
kthread_init_worker(&lo->worker);
|
|
lo->worker_task = kthread_run(loop_kthread_worker_fn,
|
|
&lo->worker, "loop%d", lo->lo_number);
|
|
if (IS_ERR(lo->worker_task))
|
|
return -ENOMEM;
|
|
set_user_nice(lo->worker_task, MIN_NICE);
|
|
return 0;
|
|
}
|
|
|
|
static void loop_update_rotational(struct loop_device *lo)
|
|
{
|
|
struct file *file = lo->lo_backing_file;
|
|
struct inode *file_inode = file->f_mapping->host;
|
|
struct block_device *file_bdev = file_inode->i_sb->s_bdev;
|
|
struct request_queue *q = lo->lo_queue;
|
|
bool nonrot = true;
|
|
|
|
/* not all filesystems (e.g. tmpfs) have a sb->s_bdev */
|
|
if (file_bdev)
|
|
nonrot = blk_queue_nonrot(bdev_get_queue(file_bdev));
|
|
|
|
if (nonrot)
|
|
blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
|
|
else
|
|
blk_queue_flag_clear(QUEUE_FLAG_NONROT, q);
|
|
}
|
|
|
|
static int loop_set_fd(struct loop_device *lo, fmode_t mode,
|
|
struct block_device *bdev, unsigned int arg)
|
|
{
|
|
struct file *file;
|
|
struct inode *inode;
|
|
struct address_space *mapping;
|
|
int lo_flags = 0;
|
|
int error;
|
|
loff_t size;
|
|
bool partscan;
|
|
|
|
/* This is safe, since we have a reference from open(). */
|
|
__module_get(THIS_MODULE);
|
|
|
|
error = -EBADF;
|
|
file = fget(arg);
|
|
if (!file)
|
|
goto out;
|
|
|
|
error = mutex_lock_killable(&loop_ctl_mutex);
|
|
if (error)
|
|
goto out_putf;
|
|
|
|
error = -EBUSY;
|
|
if (lo->lo_state != Lo_unbound)
|
|
goto out_unlock;
|
|
|
|
error = loop_validate_file(file, bdev);
|
|
if (error)
|
|
goto out_unlock;
|
|
|
|
mapping = file->f_mapping;
|
|
inode = mapping->host;
|
|
|
|
if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) ||
|
|
!file->f_op->write_iter)
|
|
lo_flags |= LO_FLAGS_READ_ONLY;
|
|
|
|
error = -EFBIG;
|
|
size = get_loop_size(lo, file);
|
|
if ((loff_t)(sector_t)size != size)
|
|
goto out_unlock;
|
|
error = loop_prepare_queue(lo);
|
|
if (error)
|
|
goto out_unlock;
|
|
|
|
error = 0;
|
|
|
|
set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);
|
|
|
|
lo->use_dio = false;
|
|
lo->lo_device = bdev;
|
|
lo->lo_flags = lo_flags;
|
|
lo->lo_backing_file = file;
|
|
lo->transfer = NULL;
|
|
lo->ioctl = NULL;
|
|
lo->lo_sizelimit = 0;
|
|
lo->old_gfp_mask = mapping_gfp_mask(mapping);
|
|
mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
|
|
|
|
if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync)
|
|
blk_queue_write_cache(lo->lo_queue, true, false);
|
|
|
|
loop_update_rotational(lo);
|
|
loop_update_dio(lo);
|
|
set_capacity(lo->lo_disk, size);
|
|
bd_set_size(bdev, size << 9);
|
|
loop_sysfs_init(lo);
|
|
/* let user-space know about the new size */
|
|
kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
|
|
|
|
set_blocksize(bdev, S_ISBLK(inode->i_mode) ?
|
|
block_size(inode->i_bdev) : PAGE_SIZE);
|
|
|
|
lo->lo_state = Lo_bound;
|
|
if (part_shift)
|
|
lo->lo_flags |= LO_FLAGS_PARTSCAN;
|
|
partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
|
|
|
|
/* Grab the block_device to prevent its destruction after we
|
|
* put /dev/loopXX inode. Later in __loop_clr_fd() we bdput(bdev).
|
|
*/
|
|
bdgrab(bdev);
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
if (partscan)
|
|
loop_reread_partitions(lo, bdev);
|
|
return 0;
|
|
|
|
out_unlock:
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
out_putf:
|
|
fput(file);
|
|
out:
|
|
/* This is safe: open() is still holding a reference. */
|
|
module_put(THIS_MODULE);
|
|
return error;
|
|
}
|
|
|
|
static int
|
|
loop_release_xfer(struct loop_device *lo)
|
|
{
|
|
int err = 0;
|
|
struct loop_func_table *xfer = lo->lo_encryption;
|
|
|
|
if (xfer) {
|
|
if (xfer->release)
|
|
err = xfer->release(lo);
|
|
lo->transfer = NULL;
|
|
lo->lo_encryption = NULL;
|
|
module_put(xfer->owner);
|
|
}
|
|
return err;
|
|
}
|
|
|
|
static int
|
|
loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer,
|
|
const struct loop_info64 *i)
|
|
{
|
|
int err = 0;
|
|
|
|
if (xfer) {
|
|
struct module *owner = xfer->owner;
|
|
|
|
if (!try_module_get(owner))
|
|
return -EINVAL;
|
|
if (xfer->init)
|
|
err = xfer->init(lo, i);
|
|
if (err)
|
|
module_put(owner);
|
|
else
|
|
lo->lo_encryption = xfer;
|
|
}
|
|
return err;
|
|
}
|
|
|
|
static int __loop_clr_fd(struct loop_device *lo, bool release)
|
|
{
|
|
struct file *filp = NULL;
|
|
gfp_t gfp = lo->old_gfp_mask;
|
|
struct block_device *bdev = lo->lo_device;
|
|
int err = 0;
|
|
bool partscan = false;
|
|
int lo_number;
|
|
|
|
mutex_lock(&loop_ctl_mutex);
|
|
if (WARN_ON_ONCE(lo->lo_state != Lo_rundown)) {
|
|
err = -ENXIO;
|
|
goto out_unlock;
|
|
}
|
|
|
|
filp = lo->lo_backing_file;
|
|
if (filp == NULL) {
|
|
err = -EINVAL;
|
|
goto out_unlock;
|
|
}
|
|
|
|
/* freeze request queue during the transition */
|
|
blk_mq_freeze_queue(lo->lo_queue);
|
|
|
|
spin_lock_irq(&lo->lo_lock);
|
|
lo->lo_backing_file = NULL;
|
|
spin_unlock_irq(&lo->lo_lock);
|
|
|
|
loop_release_xfer(lo);
|
|
lo->transfer = NULL;
|
|
lo->ioctl = NULL;
|
|
lo->lo_device = NULL;
|
|
lo->lo_encryption = NULL;
|
|
lo->lo_offset = 0;
|
|
lo->lo_sizelimit = 0;
|
|
lo->lo_encrypt_key_size = 0;
|
|
memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
|
|
memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
|
|
memset(lo->lo_file_name, 0, LO_NAME_SIZE);
|
|
blk_queue_logical_block_size(lo->lo_queue, 512);
|
|
blk_queue_physical_block_size(lo->lo_queue, 512);
|
|
blk_queue_io_min(lo->lo_queue, 512);
|
|
if (bdev) {
|
|
bdput(bdev);
|
|
invalidate_bdev(bdev);
|
|
bdev->bd_inode->i_mapping->wb_err = 0;
|
|
}
|
|
set_capacity(lo->lo_disk, 0);
|
|
loop_sysfs_exit(lo);
|
|
if (bdev) {
|
|
bd_set_size(bdev, 0);
|
|
/* let user-space know about this change */
|
|
kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
|
|
}
|
|
mapping_set_gfp_mask(filp->f_mapping, gfp);
|
|
/* This is safe: open() is still holding a reference. */
|
|
module_put(THIS_MODULE);
|
|
blk_mq_unfreeze_queue(lo->lo_queue);
|
|
|
|
partscan = lo->lo_flags & LO_FLAGS_PARTSCAN && bdev;
|
|
lo_number = lo->lo_number;
|
|
loop_unprepare_queue(lo);
|
|
out_unlock:
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
if (partscan) {
|
|
/*
|
|
* bd_mutex has been held already in release path, so don't
|
|
* acquire it if this function is called in such case.
|
|
*
|
|
* If the reread partition isn't from release path, lo_refcnt
|
|
* must be at least one and it can only become zero when the
|
|
* current holder is released.
|
|
*/
|
|
if (release)
|
|
err = __blkdev_reread_part(bdev);
|
|
else
|
|
err = blkdev_reread_part(bdev);
|
|
if (err)
|
|
pr_warn("%s: partition scan of loop%d failed (rc=%d)\n",
|
|
__func__, lo_number, err);
|
|
/* Device is gone, no point in returning error */
|
|
err = 0;
|
|
}
|
|
|
|
/*
|
|
* lo->lo_state is set to Lo_unbound here after above partscan has
|
|
* finished.
|
|
*
|
|
* There cannot be anybody else entering __loop_clr_fd() as
|
|
* lo->lo_backing_file is already cleared and Lo_rundown state
|
|
* protects us from all the other places trying to change the 'lo'
|
|
* device.
|
|
*/
|
|
mutex_lock(&loop_ctl_mutex);
|
|
lo->lo_flags = 0;
|
|
if (!part_shift)
|
|
lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN;
|
|
lo->lo_state = Lo_unbound;
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
|
|
/*
|
|
* Need not hold loop_ctl_mutex to fput backing file.
|
|
* Calling fput holding loop_ctl_mutex triggers a circular
|
|
* lock dependency possibility warning as fput can take
|
|
* bd_mutex which is usually taken before loop_ctl_mutex.
|
|
*/
|
|
if (filp)
|
|
fput(filp);
|
|
return err;
|
|
}
|
|
|
|
static int loop_clr_fd(struct loop_device *lo)
|
|
{
|
|
int err;
|
|
|
|
err = mutex_lock_killable(&loop_ctl_mutex);
|
|
if (err)
|
|
return err;
|
|
if (lo->lo_state != Lo_bound) {
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
return -ENXIO;
|
|
}
|
|
/*
|
|
* If we've explicitly asked to tear down the loop device,
|
|
* and it has an elevated reference count, set it for auto-teardown when
|
|
* the last reference goes away. This stops $!~#$@ udev from
|
|
* preventing teardown because it decided that it needs to run blkid on
|
|
* the loopback device whenever they appear. xfstests is notorious for
|
|
* failing tests because blkid via udev races with a losetup
|
|
* <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d
|
|
* command to fail with EBUSY.
|
|
*/
|
|
if (atomic_read(&lo->lo_refcnt) > 1) {
|
|
lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
return 0;
|
|
}
|
|
lo->lo_state = Lo_rundown;
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
|
|
return __loop_clr_fd(lo, false);
|
|
}
|
|
|
|
static int
|
|
loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
|
|
{
|
|
int err;
|
|
struct loop_func_table *xfer;
|
|
kuid_t uid = current_uid();
|
|
struct block_device *bdev;
|
|
bool partscan = false;
|
|
|
|
err = mutex_lock_killable(&loop_ctl_mutex);
|
|
if (err)
|
|
return err;
|
|
if (lo->lo_encrypt_key_size &&
|
|
!uid_eq(lo->lo_key_owner, uid) &&
|
|
!capable(CAP_SYS_ADMIN)) {
|
|
err = -EPERM;
|
|
goto out_unlock;
|
|
}
|
|
if (lo->lo_state != Lo_bound) {
|
|
err = -ENXIO;
|
|
goto out_unlock;
|
|
}
|
|
if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) {
|
|
err = -EINVAL;
|
|
goto out_unlock;
|
|
}
|
|
|
|
if (lo->lo_offset != info->lo_offset ||
|
|
lo->lo_sizelimit != info->lo_sizelimit) {
|
|
sync_blockdev(lo->lo_device);
|
|
kill_bdev(lo->lo_device);
|
|
}
|
|
|
|
/* I/O need to be drained during transfer transition */
|
|
blk_mq_freeze_queue(lo->lo_queue);
|
|
|
|
err = loop_release_xfer(lo);
|
|
if (err)
|
|
goto out_unfreeze;
|
|
|
|
if (info->lo_encrypt_type) {
|
|
unsigned int type = info->lo_encrypt_type;
|
|
|
|
if (type >= MAX_LO_CRYPT) {
|
|
err = -EINVAL;
|
|
goto out_unfreeze;
|
|
}
|
|
xfer = xfer_funcs[type];
|
|
if (xfer == NULL) {
|
|
err = -EINVAL;
|
|
goto out_unfreeze;
|
|
}
|
|
} else
|
|
xfer = NULL;
|
|
|
|
err = loop_init_xfer(lo, xfer, info);
|
|
if (err)
|
|
goto out_unfreeze;
|
|
|
|
if (lo->lo_offset != info->lo_offset ||
|
|
lo->lo_sizelimit != info->lo_sizelimit) {
|
|
/* kill_bdev should have truncated all the pages */
|
|
if (lo->lo_device->bd_inode->i_mapping->nrpages) {
|
|
err = -EAGAIN;
|
|
pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n",
|
|
__func__, lo->lo_number, lo->lo_file_name,
|
|
lo->lo_device->bd_inode->i_mapping->nrpages);
|
|
goto out_unfreeze;
|
|
}
|
|
if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit)) {
|
|
err = -EFBIG;
|
|
goto out_unfreeze;
|
|
}
|
|
}
|
|
|
|
loop_config_discard(lo);
|
|
|
|
memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
|
|
memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE);
|
|
lo->lo_file_name[LO_NAME_SIZE-1] = 0;
|
|
lo->lo_crypt_name[LO_NAME_SIZE-1] = 0;
|
|
|
|
if (!xfer)
|
|
xfer = &none_funcs;
|
|
lo->transfer = xfer->transfer;
|
|
lo->ioctl = xfer->ioctl;
|
|
|
|
if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) !=
|
|
(info->lo_flags & LO_FLAGS_AUTOCLEAR))
|
|
lo->lo_flags ^= LO_FLAGS_AUTOCLEAR;
|
|
|
|
lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
|
|
lo->lo_init[0] = info->lo_init[0];
|
|
lo->lo_init[1] = info->lo_init[1];
|
|
if (info->lo_encrypt_key_size) {
|
|
memcpy(lo->lo_encrypt_key, info->lo_encrypt_key,
|
|
info->lo_encrypt_key_size);
|
|
lo->lo_key_owner = uid;
|
|
}
|
|
|
|
/* update dio if lo_offset or transfer is changed */
|
|
__loop_update_dio(lo, lo->use_dio);
|
|
|
|
out_unfreeze:
|
|
blk_mq_unfreeze_queue(lo->lo_queue);
|
|
|
|
if (!err && (info->lo_flags & LO_FLAGS_PARTSCAN) &&
|
|
!(lo->lo_flags & LO_FLAGS_PARTSCAN)) {
|
|
lo->lo_flags |= LO_FLAGS_PARTSCAN;
|
|
lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN;
|
|
bdev = lo->lo_device;
|
|
partscan = true;
|
|
}
|
|
out_unlock:
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
if (partscan)
|
|
loop_reread_partitions(lo, bdev);
|
|
|
|
return err;
|
|
}
|
|
|
|
static int
|
|
loop_get_status(struct loop_device *lo, struct loop_info64 *info)
|
|
{
|
|
struct path path;
|
|
struct kstat stat;
|
|
int ret;
|
|
|
|
ret = mutex_lock_killable(&loop_ctl_mutex);
|
|
if (ret)
|
|
return ret;
|
|
if (lo->lo_state != Lo_bound) {
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
return -ENXIO;
|
|
}
|
|
|
|
memset(info, 0, sizeof(*info));
|
|
info->lo_number = lo->lo_number;
|
|
info->lo_offset = lo->lo_offset;
|
|
info->lo_sizelimit = lo->lo_sizelimit;
|
|
info->lo_flags = lo->lo_flags;
|
|
memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
|
|
memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE);
|
|
info->lo_encrypt_type =
|
|
lo->lo_encryption ? lo->lo_encryption->number : 0;
|
|
if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) {
|
|
info->lo_encrypt_key_size = lo->lo_encrypt_key_size;
|
|
memcpy(info->lo_encrypt_key, lo->lo_encrypt_key,
|
|
lo->lo_encrypt_key_size);
|
|
}
|
|
|
|
/* Drop loop_ctl_mutex while we call into the filesystem. */
|
|
path = lo->lo_backing_file->f_path;
|
|
path_get(&path);
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
ret = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT);
|
|
if (!ret) {
|
|
info->lo_device = huge_encode_dev(stat.dev);
|
|
info->lo_inode = stat.ino;
|
|
info->lo_rdevice = huge_encode_dev(stat.rdev);
|
|
}
|
|
path_put(&path);
|
|
return ret;
|
|
}
|
|
|
|
static void
|
|
loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
|
|
{
|
|
memset(info64, 0, sizeof(*info64));
|
|
info64->lo_number = info->lo_number;
|
|
info64->lo_device = info->lo_device;
|
|
info64->lo_inode = info->lo_inode;
|
|
info64->lo_rdevice = info->lo_rdevice;
|
|
info64->lo_offset = info->lo_offset;
|
|
info64->lo_sizelimit = 0;
|
|
info64->lo_encrypt_type = info->lo_encrypt_type;
|
|
info64->lo_encrypt_key_size = info->lo_encrypt_key_size;
|
|
info64->lo_flags = info->lo_flags;
|
|
info64->lo_init[0] = info->lo_init[0];
|
|
info64->lo_init[1] = info->lo_init[1];
|
|
if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
|
|
memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE);
|
|
else
|
|
memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
|
|
memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE);
|
|
}
|
|
|
|
static int
|
|
loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
|
|
{
|
|
memset(info, 0, sizeof(*info));
|
|
info->lo_number = info64->lo_number;
|
|
info->lo_device = info64->lo_device;
|
|
info->lo_inode = info64->lo_inode;
|
|
info->lo_rdevice = info64->lo_rdevice;
|
|
info->lo_offset = info64->lo_offset;
|
|
info->lo_encrypt_type = info64->lo_encrypt_type;
|
|
info->lo_encrypt_key_size = info64->lo_encrypt_key_size;
|
|
info->lo_flags = info64->lo_flags;
|
|
info->lo_init[0] = info64->lo_init[0];
|
|
info->lo_init[1] = info64->lo_init[1];
|
|
if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
|
|
memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
|
|
else
|
|
memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
|
|
memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
|
|
|
|
/* error in case values were truncated */
|
|
if (info->lo_device != info64->lo_device ||
|
|
info->lo_rdevice != info64->lo_rdevice ||
|
|
info->lo_inode != info64->lo_inode ||
|
|
info->lo_offset != info64->lo_offset)
|
|
return -EOVERFLOW;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
|
|
{
|
|
struct loop_info info;
|
|
struct loop_info64 info64;
|
|
|
|
if (copy_from_user(&info, arg, sizeof (struct loop_info)))
|
|
return -EFAULT;
|
|
loop_info64_from_old(&info, &info64);
|
|
return loop_set_status(lo, &info64);
|
|
}
|
|
|
|
static int
|
|
loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
|
|
{
|
|
struct loop_info64 info64;
|
|
|
|
if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
|
|
return -EFAULT;
|
|
return loop_set_status(lo, &info64);
|
|
}
|
|
|
|
static int
|
|
loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
|
|
struct loop_info info;
|
|
struct loop_info64 info64;
|
|
int err;
|
|
|
|
if (!arg)
|
|
return -EINVAL;
|
|
err = loop_get_status(lo, &info64);
|
|
if (!err)
|
|
err = loop_info64_to_old(&info64, &info);
|
|
if (!err && copy_to_user(arg, &info, sizeof(info)))
|
|
err = -EFAULT;
|
|
|
|
return err;
|
|
}
|
|
|
|
static int
|
|
loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
|
|
struct loop_info64 info64;
|
|
int err;
|
|
|
|
if (!arg)
|
|
return -EINVAL;
|
|
err = loop_get_status(lo, &info64);
|
|
if (!err && copy_to_user(arg, &info64, sizeof(info64)))
|
|
err = -EFAULT;
|
|
|
|
return err;
|
|
}
|
|
|
|
static int loop_set_capacity(struct loop_device *lo)
|
|
{
|
|
if (unlikely(lo->lo_state != Lo_bound))
|
|
return -ENXIO;
|
|
|
|
return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit);
|
|
}
|
|
|
|
static int loop_set_dio(struct loop_device *lo, unsigned long arg)
|
|
{
|
|
int error = -ENXIO;
|
|
if (lo->lo_state != Lo_bound)
|
|
goto out;
|
|
|
|
__loop_update_dio(lo, !!arg);
|
|
if (lo->use_dio == !!arg)
|
|
return 0;
|
|
error = -EINVAL;
|
|
out:
|
|
return error;
|
|
}
|
|
|
|
static int loop_set_block_size(struct loop_device *lo, unsigned long arg)
|
|
{
|
|
int err = 0;
|
|
|
|
if (lo->lo_state != Lo_bound)
|
|
return -ENXIO;
|
|
|
|
if (arg < 512 || arg > PAGE_SIZE || !is_power_of_2(arg))
|
|
return -EINVAL;
|
|
|
|
if (lo->lo_queue->limits.logical_block_size != arg) {
|
|
sync_blockdev(lo->lo_device);
|
|
kill_bdev(lo->lo_device);
|
|
}
|
|
|
|
blk_mq_freeze_queue(lo->lo_queue);
|
|
|
|
/* kill_bdev should have truncated all the pages */
|
|
if (lo->lo_queue->limits.logical_block_size != arg &&
|
|
lo->lo_device->bd_inode->i_mapping->nrpages) {
|
|
err = -EAGAIN;
|
|
pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n",
|
|
__func__, lo->lo_number, lo->lo_file_name,
|
|
lo->lo_device->bd_inode->i_mapping->nrpages);
|
|
goto out_unfreeze;
|
|
}
|
|
|
|
blk_queue_logical_block_size(lo->lo_queue, arg);
|
|
blk_queue_physical_block_size(lo->lo_queue, arg);
|
|
blk_queue_io_min(lo->lo_queue, arg);
|
|
loop_update_dio(lo);
|
|
out_unfreeze:
|
|
blk_mq_unfreeze_queue(lo->lo_queue);
|
|
|
|
return err;
|
|
}
|
|
|
|
static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd,
|
|
unsigned long arg)
|
|
{
|
|
int err;
|
|
|
|
err = mutex_lock_killable(&loop_ctl_mutex);
|
|
if (err)
|
|
return err;
|
|
switch (cmd) {
|
|
case LOOP_SET_CAPACITY:
|
|
err = loop_set_capacity(lo);
|
|
break;
|
|
case LOOP_SET_DIRECT_IO:
|
|
err = loop_set_dio(lo, arg);
|
|
break;
|
|
case LOOP_SET_BLOCK_SIZE:
|
|
err = loop_set_block_size(lo, arg);
|
|
break;
|
|
default:
|
|
err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL;
|
|
}
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
return err;
|
|
}
|
|
|
|
static int lo_ioctl(struct block_device *bdev, fmode_t mode,
|
|
unsigned int cmd, unsigned long arg)
|
|
{
|
|
struct loop_device *lo = bdev->bd_disk->private_data;
|
|
int err;
|
|
|
|
switch (cmd) {
|
|
case LOOP_SET_FD:
|
|
return loop_set_fd(lo, mode, bdev, arg);
|
|
case LOOP_CHANGE_FD:
|
|
return loop_change_fd(lo, bdev, arg);
|
|
case LOOP_CLR_FD:
|
|
return loop_clr_fd(lo);
|
|
case LOOP_SET_STATUS:
|
|
err = -EPERM;
|
|
if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) {
|
|
err = loop_set_status_old(lo,
|
|
(struct loop_info __user *)arg);
|
|
}
|
|
break;
|
|
case LOOP_GET_STATUS:
|
|
return loop_get_status_old(lo, (struct loop_info __user *) arg);
|
|
case LOOP_SET_STATUS64:
|
|
err = -EPERM;
|
|
if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) {
|
|
err = loop_set_status64(lo,
|
|
(struct loop_info64 __user *) arg);
|
|
}
|
|
break;
|
|
case LOOP_GET_STATUS64:
|
|
return loop_get_status64(lo, (struct loop_info64 __user *) arg);
|
|
case LOOP_SET_CAPACITY:
|
|
case LOOP_SET_DIRECT_IO:
|
|
case LOOP_SET_BLOCK_SIZE:
|
|
if (!(mode & FMODE_WRITE) && !capable(CAP_SYS_ADMIN))
|
|
return -EPERM;
|
|
/* Fall through */
|
|
default:
|
|
err = lo_simple_ioctl(lo, cmd, arg);
|
|
break;
|
|
}
|
|
|
|
return err;
|
|
}
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
struct compat_loop_info {
|
|
compat_int_t lo_number; /* ioctl r/o */
|
|
compat_dev_t lo_device; /* ioctl r/o */
|
|
compat_ulong_t lo_inode; /* ioctl r/o */
|
|
compat_dev_t lo_rdevice; /* ioctl r/o */
|
|
compat_int_t lo_offset;
|
|
compat_int_t lo_encrypt_type;
|
|
compat_int_t lo_encrypt_key_size; /* ioctl w/o */
|
|
compat_int_t lo_flags; /* ioctl r/o */
|
|
char lo_name[LO_NAME_SIZE];
|
|
unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */
|
|
compat_ulong_t lo_init[2];
|
|
char reserved[4];
|
|
};
|
|
|
|
/*
|
|
* Transfer 32-bit compatibility structure in userspace to 64-bit loop info
|
|
* - noinlined to reduce stack space usage in main part of driver
|
|
*/
|
|
static noinline int
|
|
loop_info64_from_compat(const struct compat_loop_info __user *arg,
|
|
struct loop_info64 *info64)
|
|
{
|
|
struct compat_loop_info info;
|
|
|
|
if (copy_from_user(&info, arg, sizeof(info)))
|
|
return -EFAULT;
|
|
|
|
memset(info64, 0, sizeof(*info64));
|
|
info64->lo_number = info.lo_number;
|
|
info64->lo_device = info.lo_device;
|
|
info64->lo_inode = info.lo_inode;
|
|
info64->lo_rdevice = info.lo_rdevice;
|
|
info64->lo_offset = info.lo_offset;
|
|
info64->lo_sizelimit = 0;
|
|
info64->lo_encrypt_type = info.lo_encrypt_type;
|
|
info64->lo_encrypt_key_size = info.lo_encrypt_key_size;
|
|
info64->lo_flags = info.lo_flags;
|
|
info64->lo_init[0] = info.lo_init[0];
|
|
info64->lo_init[1] = info.lo_init[1];
|
|
if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
|
|
memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE);
|
|
else
|
|
memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE);
|
|
memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Transfer 64-bit loop info to 32-bit compatibility structure in userspace
|
|
* - noinlined to reduce stack space usage in main part of driver
|
|
*/
|
|
static noinline int
|
|
loop_info64_to_compat(const struct loop_info64 *info64,
|
|
struct compat_loop_info __user *arg)
|
|
{
|
|
struct compat_loop_info info;
|
|
|
|
memset(&info, 0, sizeof(info));
|
|
info.lo_number = info64->lo_number;
|
|
info.lo_device = info64->lo_device;
|
|
info.lo_inode = info64->lo_inode;
|
|
info.lo_rdevice = info64->lo_rdevice;
|
|
info.lo_offset = info64->lo_offset;
|
|
info.lo_encrypt_type = info64->lo_encrypt_type;
|
|
info.lo_encrypt_key_size = info64->lo_encrypt_key_size;
|
|
info.lo_flags = info64->lo_flags;
|
|
info.lo_init[0] = info64->lo_init[0];
|
|
info.lo_init[1] = info64->lo_init[1];
|
|
if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
|
|
memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
|
|
else
|
|
memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE);
|
|
memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
|
|
|
|
/* error in case values were truncated */
|
|
if (info.lo_device != info64->lo_device ||
|
|
info.lo_rdevice != info64->lo_rdevice ||
|
|
info.lo_inode != info64->lo_inode ||
|
|
info.lo_offset != info64->lo_offset ||
|
|
info.lo_init[0] != info64->lo_init[0] ||
|
|
info.lo_init[1] != info64->lo_init[1])
|
|
return -EOVERFLOW;
|
|
|
|
if (copy_to_user(arg, &info, sizeof(info)))
|
|
return -EFAULT;
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
loop_set_status_compat(struct loop_device *lo,
|
|
const struct compat_loop_info __user *arg)
|
|
{
|
|
struct loop_info64 info64;
|
|
int ret;
|
|
|
|
ret = loop_info64_from_compat(arg, &info64);
|
|
if (ret < 0)
|
|
return ret;
|
|
return loop_set_status(lo, &info64);
|
|
}
|
|
|
|
static int
|
|
loop_get_status_compat(struct loop_device *lo,
|
|
struct compat_loop_info __user *arg)
|
|
{
|
|
struct loop_info64 info64;
|
|
int err;
|
|
|
|
if (!arg)
|
|
return -EINVAL;
|
|
err = loop_get_status(lo, &info64);
|
|
if (!err)
|
|
err = loop_info64_to_compat(&info64, arg);
|
|
return err;
|
|
}
|
|
|
|
static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode,
|
|
unsigned int cmd, unsigned long arg)
|
|
{
|
|
struct loop_device *lo = bdev->bd_disk->private_data;
|
|
int err;
|
|
|
|
switch(cmd) {
|
|
case LOOP_SET_STATUS:
|
|
err = loop_set_status_compat(lo,
|
|
(const struct compat_loop_info __user *)arg);
|
|
break;
|
|
case LOOP_GET_STATUS:
|
|
err = loop_get_status_compat(lo,
|
|
(struct compat_loop_info __user *)arg);
|
|
break;
|
|
case LOOP_SET_CAPACITY:
|
|
case LOOP_CLR_FD:
|
|
case LOOP_GET_STATUS64:
|
|
case LOOP_SET_STATUS64:
|
|
arg = (unsigned long) compat_ptr(arg);
|
|
/* fall through */
|
|
case LOOP_SET_FD:
|
|
case LOOP_CHANGE_FD:
|
|
case LOOP_SET_BLOCK_SIZE:
|
|
err = lo_ioctl(bdev, mode, cmd, arg);
|
|
break;
|
|
default:
|
|
err = -ENOIOCTLCMD;
|
|
break;
|
|
}
|
|
return err;
|
|
}
|
|
#endif
|
|
|
|
static int lo_open(struct block_device *bdev, fmode_t mode)
|
|
{
|
|
struct loop_device *lo;
|
|
int err;
|
|
|
|
err = mutex_lock_killable(&loop_ctl_mutex);
|
|
if (err)
|
|
return err;
|
|
lo = bdev->bd_disk->private_data;
|
|
if (!lo) {
|
|
err = -ENXIO;
|
|
goto out;
|
|
}
|
|
|
|
atomic_inc(&lo->lo_refcnt);
|
|
out:
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
return err;
|
|
}
|
|
|
|
static void lo_release(struct gendisk *disk, fmode_t mode)
|
|
{
|
|
struct loop_device *lo;
|
|
|
|
mutex_lock(&loop_ctl_mutex);
|
|
lo = disk->private_data;
|
|
if (atomic_dec_return(&lo->lo_refcnt))
|
|
goto out_unlock;
|
|
|
|
if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) {
|
|
if (lo->lo_state != Lo_bound)
|
|
goto out_unlock;
|
|
lo->lo_state = Lo_rundown;
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
/*
|
|
* In autoclear mode, stop the loop thread
|
|
* and remove configuration after last close.
|
|
*/
|
|
__loop_clr_fd(lo, true);
|
|
return;
|
|
} else if (lo->lo_state == Lo_bound) {
|
|
/*
|
|
* Otherwise keep thread (if running) and config,
|
|
* but flush possible ongoing bios in thread.
|
|
*/
|
|
blk_mq_freeze_queue(lo->lo_queue);
|
|
blk_mq_unfreeze_queue(lo->lo_queue);
|
|
}
|
|
|
|
out_unlock:
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
}
|
|
|
|
static const struct block_device_operations lo_fops = {
|
|
.owner = THIS_MODULE,
|
|
.open = lo_open,
|
|
.release = lo_release,
|
|
.ioctl = lo_ioctl,
|
|
#ifdef CONFIG_COMPAT
|
|
.compat_ioctl = lo_compat_ioctl,
|
|
#endif
|
|
};
|
|
|
|
/*
|
|
* And now the modules code and kernel interface.
|
|
*/
|
|
static int max_loop;
|
|
module_param(max_loop, int, 0444);
|
|
MODULE_PARM_DESC(max_loop, "Maximum number of loop devices");
|
|
module_param(max_part, int, 0444);
|
|
MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device");
|
|
MODULE_LICENSE("GPL");
|
|
MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR);
|
|
|
|
int loop_register_transfer(struct loop_func_table *funcs)
|
|
{
|
|
unsigned int n = funcs->number;
|
|
|
|
if (n >= MAX_LO_CRYPT || xfer_funcs[n])
|
|
return -EINVAL;
|
|
xfer_funcs[n] = funcs;
|
|
return 0;
|
|
}
|
|
|
|
static int unregister_transfer_cb(int id, void *ptr, void *data)
|
|
{
|
|
struct loop_device *lo = ptr;
|
|
struct loop_func_table *xfer = data;
|
|
|
|
mutex_lock(&loop_ctl_mutex);
|
|
if (lo->lo_encryption == xfer)
|
|
loop_release_xfer(lo);
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
return 0;
|
|
}
|
|
|
|
int loop_unregister_transfer(int number)
|
|
{
|
|
unsigned int n = number;
|
|
struct loop_func_table *xfer;
|
|
|
|
if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL)
|
|
return -EINVAL;
|
|
|
|
xfer_funcs[n] = NULL;
|
|
idr_for_each(&loop_index_idr, &unregister_transfer_cb, xfer);
|
|
return 0;
|
|
}
|
|
|
|
EXPORT_SYMBOL(loop_register_transfer);
|
|
EXPORT_SYMBOL(loop_unregister_transfer);
|
|
|
|
static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx,
|
|
const struct blk_mq_queue_data *bd)
|
|
{
|
|
struct request *rq = bd->rq;
|
|
struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
|
|
struct loop_device *lo = rq->q->queuedata;
|
|
|
|
blk_mq_start_request(rq);
|
|
|
|
if (lo->lo_state != Lo_bound)
|
|
return BLK_STS_IOERR;
|
|
|
|
switch (req_op(rq)) {
|
|
case REQ_OP_FLUSH:
|
|
case REQ_OP_DISCARD:
|
|
case REQ_OP_WRITE_ZEROES:
|
|
cmd->use_aio = false;
|
|
break;
|
|
default:
|
|
cmd->use_aio = lo->use_dio;
|
|
break;
|
|
}
|
|
|
|
/* always use the first bio's css */
|
|
#ifdef CONFIG_BLK_CGROUP
|
|
if (cmd->use_aio && rq->bio && rq->bio->bi_blkg) {
|
|
cmd->css = &bio_blkcg(rq->bio)->css;
|
|
css_get(cmd->css);
|
|
} else
|
|
#endif
|
|
cmd->css = NULL;
|
|
kthread_queue_work(&lo->worker, &cmd->work);
|
|
|
|
return BLK_STS_OK;
|
|
}
|
|
|
|
static void loop_handle_cmd(struct loop_cmd *cmd)
|
|
{
|
|
struct request *rq = blk_mq_rq_from_pdu(cmd);
|
|
const bool write = op_is_write(req_op(rq));
|
|
struct loop_device *lo = rq->q->queuedata;
|
|
int ret = 0;
|
|
|
|
if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) {
|
|
ret = -EIO;
|
|
goto failed;
|
|
}
|
|
|
|
ret = do_req_filebacked(lo, rq);
|
|
failed:
|
|
/* complete non-aio request */
|
|
if (!cmd->use_aio || ret) {
|
|
cmd->ret = ret ? -EIO : 0;
|
|
blk_mq_complete_request(rq);
|
|
}
|
|
}
|
|
|
|
static void loop_queue_work(struct kthread_work *work)
|
|
{
|
|
struct loop_cmd *cmd =
|
|
container_of(work, struct loop_cmd, work);
|
|
|
|
loop_handle_cmd(cmd);
|
|
}
|
|
|
|
static int loop_init_request(struct blk_mq_tag_set *set, struct request *rq,
|
|
unsigned int hctx_idx, unsigned int numa_node)
|
|
{
|
|
struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
|
|
|
|
kthread_init_work(&cmd->work, loop_queue_work);
|
|
return 0;
|
|
}
|
|
|
|
static const struct blk_mq_ops loop_mq_ops = {
|
|
.queue_rq = loop_queue_rq,
|
|
.init_request = loop_init_request,
|
|
.complete = lo_complete_rq,
|
|
};
|
|
|
|
static int loop_add(struct loop_device **l, int i)
|
|
{
|
|
struct loop_device *lo;
|
|
struct gendisk *disk;
|
|
int err;
|
|
|
|
err = -ENOMEM;
|
|
lo = kzalloc(sizeof(*lo), GFP_KERNEL);
|
|
if (!lo)
|
|
goto out;
|
|
|
|
lo->lo_state = Lo_unbound;
|
|
|
|
/* allocate id, if @id >= 0, we're requesting that specific id */
|
|
if (i >= 0) {
|
|
err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL);
|
|
if (err == -ENOSPC)
|
|
err = -EEXIST;
|
|
} else {
|
|
err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL);
|
|
}
|
|
if (err < 0)
|
|
goto out_free_dev;
|
|
i = err;
|
|
|
|
err = -ENOMEM;
|
|
lo->tag_set.ops = &loop_mq_ops;
|
|
lo->tag_set.nr_hw_queues = 1;
|
|
lo->tag_set.queue_depth = 128;
|
|
lo->tag_set.numa_node = NUMA_NO_NODE;
|
|
lo->tag_set.cmd_size = sizeof(struct loop_cmd);
|
|
lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
|
|
lo->tag_set.driver_data = lo;
|
|
|
|
err = blk_mq_alloc_tag_set(&lo->tag_set);
|
|
if (err)
|
|
goto out_free_idr;
|
|
|
|
lo->lo_queue = blk_mq_init_queue(&lo->tag_set);
|
|
if (IS_ERR(lo->lo_queue)) {
|
|
err = PTR_ERR(lo->lo_queue);
|
|
goto out_cleanup_tags;
|
|
}
|
|
lo->lo_queue->queuedata = lo;
|
|
|
|
blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS);
|
|
|
|
/*
|
|
* By default, we do buffer IO, so it doesn't make sense to enable
|
|
* merge because the I/O submitted to backing file is handled page by
|
|
* page. For directio mode, merge does help to dispatch bigger request
|
|
* to underlayer disk. We will enable merge once directio is enabled.
|
|
*/
|
|
blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue);
|
|
|
|
err = -ENOMEM;
|
|
disk = lo->lo_disk = alloc_disk(1 << part_shift);
|
|
if (!disk)
|
|
goto out_free_queue;
|
|
|
|
/*
|
|
* Disable partition scanning by default. The in-kernel partition
|
|
* scanning can be requested individually per-device during its
|
|
* setup. Userspace can always add and remove partitions from all
|
|
* devices. The needed partition minors are allocated from the
|
|
* extended minor space, the main loop device numbers will continue
|
|
* to match the loop minors, regardless of the number of partitions
|
|
* used.
|
|
*
|
|
* If max_part is given, partition scanning is globally enabled for
|
|
* all loop devices. The minors for the main loop devices will be
|
|
* multiples of max_part.
|
|
*
|
|
* Note: Global-for-all-devices, set-only-at-init, read-only module
|
|
* parameteters like 'max_loop' and 'max_part' make things needlessly
|
|
* complicated, are too static, inflexible and may surprise
|
|
* userspace tools. Parameters like this in general should be avoided.
|
|
*/
|
|
if (!part_shift)
|
|
disk->flags |= GENHD_FL_NO_PART_SCAN;
|
|
disk->flags |= GENHD_FL_EXT_DEVT;
|
|
atomic_set(&lo->lo_refcnt, 0);
|
|
lo->lo_number = i;
|
|
spin_lock_init(&lo->lo_lock);
|
|
disk->major = LOOP_MAJOR;
|
|
disk->first_minor = i << part_shift;
|
|
disk->fops = &lo_fops;
|
|
disk->private_data = lo;
|
|
disk->queue = lo->lo_queue;
|
|
sprintf(disk->disk_name, "loop%d", i);
|
|
add_disk(disk);
|
|
*l = lo;
|
|
return lo->lo_number;
|
|
|
|
out_free_queue:
|
|
blk_cleanup_queue(lo->lo_queue);
|
|
out_cleanup_tags:
|
|
blk_mq_free_tag_set(&lo->tag_set);
|
|
out_free_idr:
|
|
idr_remove(&loop_index_idr, i);
|
|
out_free_dev:
|
|
kfree(lo);
|
|
out:
|
|
return err;
|
|
}
|
|
|
|
static void loop_remove(struct loop_device *lo)
|
|
{
|
|
del_gendisk(lo->lo_disk);
|
|
blk_cleanup_queue(lo->lo_queue);
|
|
blk_mq_free_tag_set(&lo->tag_set);
|
|
put_disk(lo->lo_disk);
|
|
kfree(lo);
|
|
}
|
|
|
|
static int find_free_cb(int id, void *ptr, void *data)
|
|
{
|
|
struct loop_device *lo = ptr;
|
|
struct loop_device **l = data;
|
|
|
|
if (lo->lo_state == Lo_unbound) {
|
|
*l = lo;
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int loop_lookup(struct loop_device **l, int i)
|
|
{
|
|
struct loop_device *lo;
|
|
int ret = -ENODEV;
|
|
|
|
if (i < 0) {
|
|
int err;
|
|
|
|
err = idr_for_each(&loop_index_idr, &find_free_cb, &lo);
|
|
if (err == 1) {
|
|
*l = lo;
|
|
ret = lo->lo_number;
|
|
}
|
|
goto out;
|
|
}
|
|
|
|
/* lookup and return a specific i */
|
|
lo = idr_find(&loop_index_idr, i);
|
|
if (lo) {
|
|
*l = lo;
|
|
ret = lo->lo_number;
|
|
}
|
|
out:
|
|
return ret;
|
|
}
|
|
|
|
static struct kobject *loop_probe(dev_t dev, int *part, void *data)
|
|
{
|
|
struct loop_device *lo;
|
|
struct kobject *kobj;
|
|
int err;
|
|
|
|
mutex_lock(&loop_ctl_mutex);
|
|
err = loop_lookup(&lo, MINOR(dev) >> part_shift);
|
|
if (err < 0)
|
|
err = loop_add(&lo, MINOR(dev) >> part_shift);
|
|
if (err < 0)
|
|
kobj = NULL;
|
|
else
|
|
kobj = get_disk_and_module(lo->lo_disk);
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
|
|
*part = 0;
|
|
return kobj;
|
|
}
|
|
|
|
static long loop_control_ioctl(struct file *file, unsigned int cmd,
|
|
unsigned long parm)
|
|
{
|
|
struct loop_device *lo;
|
|
int ret;
|
|
|
|
ret = mutex_lock_killable(&loop_ctl_mutex);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = -ENOSYS;
|
|
switch (cmd) {
|
|
case LOOP_CTL_ADD:
|
|
ret = loop_lookup(&lo, parm);
|
|
if (ret >= 0) {
|
|
ret = -EEXIST;
|
|
break;
|
|
}
|
|
ret = loop_add(&lo, parm);
|
|
break;
|
|
case LOOP_CTL_REMOVE:
|
|
ret = loop_lookup(&lo, parm);
|
|
if (ret < 0)
|
|
break;
|
|
if (lo->lo_state != Lo_unbound) {
|
|
ret = -EBUSY;
|
|
break;
|
|
}
|
|
if (atomic_read(&lo->lo_refcnt) > 0) {
|
|
ret = -EBUSY;
|
|
break;
|
|
}
|
|
lo->lo_disk->private_data = NULL;
|
|
idr_remove(&loop_index_idr, lo->lo_number);
|
|
loop_remove(lo);
|
|
break;
|
|
case LOOP_CTL_GET_FREE:
|
|
ret = loop_lookup(&lo, -1);
|
|
if (ret >= 0)
|
|
break;
|
|
ret = loop_add(&lo, -1);
|
|
}
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static const struct file_operations loop_ctl_fops = {
|
|
.open = nonseekable_open,
|
|
.unlocked_ioctl = loop_control_ioctl,
|
|
.compat_ioctl = loop_control_ioctl,
|
|
.owner = THIS_MODULE,
|
|
.llseek = noop_llseek,
|
|
};
|
|
|
|
static struct miscdevice loop_misc = {
|
|
.minor = LOOP_CTRL_MINOR,
|
|
.name = "loop-control",
|
|
.fops = &loop_ctl_fops,
|
|
};
|
|
|
|
MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR);
|
|
MODULE_ALIAS("devname:loop-control");
|
|
|
|
static int __init loop_init(void)
|
|
{
|
|
int i, nr;
|
|
unsigned long range;
|
|
struct loop_device *lo;
|
|
int err;
|
|
|
|
part_shift = 0;
|
|
if (max_part > 0) {
|
|
part_shift = fls(max_part);
|
|
|
|
/*
|
|
* Adjust max_part according to part_shift as it is exported
|
|
* to user space so that user can decide correct minor number
|
|
* if [s]he want to create more devices.
|
|
*
|
|
* Note that -1 is required because partition 0 is reserved
|
|
* for the whole disk.
|
|
*/
|
|
max_part = (1UL << part_shift) - 1;
|
|
}
|
|
|
|
if ((1UL << part_shift) > DISK_MAX_PARTS) {
|
|
err = -EINVAL;
|
|
goto err_out;
|
|
}
|
|
|
|
if (max_loop > 1UL << (MINORBITS - part_shift)) {
|
|
err = -EINVAL;
|
|
goto err_out;
|
|
}
|
|
|
|
/*
|
|
* If max_loop is specified, create that many devices upfront.
|
|
* This also becomes a hard limit. If max_loop is not specified,
|
|
* create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module
|
|
* init time. Loop devices can be requested on-demand with the
|
|
* /dev/loop-control interface, or be instantiated by accessing
|
|
* a 'dead' device node.
|
|
*/
|
|
if (max_loop) {
|
|
nr = max_loop;
|
|
range = max_loop << part_shift;
|
|
} else {
|
|
nr = CONFIG_BLK_DEV_LOOP_MIN_COUNT;
|
|
range = 1UL << MINORBITS;
|
|
}
|
|
|
|
err = misc_register(&loop_misc);
|
|
if (err < 0)
|
|
goto err_out;
|
|
|
|
|
|
if (register_blkdev(LOOP_MAJOR, "loop")) {
|
|
err = -EIO;
|
|
goto misc_out;
|
|
}
|
|
|
|
blk_register_region(MKDEV(LOOP_MAJOR, 0), range,
|
|
THIS_MODULE, loop_probe, NULL, NULL);
|
|
|
|
/* pre-create number of devices given by config or max_loop */
|
|
mutex_lock(&loop_ctl_mutex);
|
|
for (i = 0; i < nr; i++)
|
|
loop_add(&lo, i);
|
|
mutex_unlock(&loop_ctl_mutex);
|
|
|
|
printk(KERN_INFO "loop: module loaded\n");
|
|
return 0;
|
|
|
|
misc_out:
|
|
misc_deregister(&loop_misc);
|
|
err_out:
|
|
return err;
|
|
}
|
|
|
|
static int loop_exit_cb(int id, void *ptr, void *data)
|
|
{
|
|
struct loop_device *lo = ptr;
|
|
|
|
loop_remove(lo);
|
|
return 0;
|
|
}
|
|
|
|
static void __exit loop_exit(void)
|
|
{
|
|
unsigned long range;
|
|
|
|
range = max_loop ? max_loop << part_shift : 1UL << MINORBITS;
|
|
|
|
idr_for_each(&loop_index_idr, &loop_exit_cb, NULL);
|
|
idr_destroy(&loop_index_idr);
|
|
|
|
blk_unregister_region(MKDEV(LOOP_MAJOR, 0), range);
|
|
unregister_blkdev(LOOP_MAJOR, "loop");
|
|
|
|
misc_deregister(&loop_misc);
|
|
}
|
|
|
|
module_init(loop_init);
|
|
module_exit(loop_exit);
|
|
|
|
#ifndef MODULE
|
|
static int __init max_loop_setup(char *str)
|
|
{
|
|
max_loop = simple_strtol(str, NULL, 0);
|
|
return 1;
|
|
}
|
|
|
|
__setup("max_loop=", max_loop_setup);
|
|
#endif
|