![Dave Jiang](/assets/img/avatar_default.png)
Re-enable deep flush so that users always have a way to be sure that a
write makes it all the way out to media. Writes from the PMEM driver
always arrive at the NVDIMM since movnt is used to bypass the cache, and
the driver relies on the ADR (Asynchronous DRAM Refresh) mechanism to
flush write buffers on power failure. The Deep Flush mechanism is there
to explicitly write buffers to protect against (rare) ADR failure. This
change prevents a regression in deep flush behavior so that applications
can continue to depend on fsync() as a mechanism to trigger deep flush
in the filesystem-DAX case.
Fixes: 06e8ccdab1
("acpi: nfit: Add support for detect platform CPU cache...")
Reviewed-by: Jeff Moyer <jmoyer@redhat.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
564 lines
14 KiB
C
564 lines
14 KiB
C
/*
|
|
* Persistent Memory Driver
|
|
*
|
|
* Copyright (c) 2014-2015, Intel Corporation.
|
|
* Copyright (c) 2015, Christoph Hellwig <hch@lst.de>.
|
|
* Copyright (c) 2015, Boaz Harrosh <boaz@plexistor.com>.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
* under the terms and conditions of the GNU General Public License,
|
|
* version 2, as published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
* more details.
|
|
*/
|
|
|
|
#include <asm/cacheflush.h>
|
|
#include <linux/blkdev.h>
|
|
#include <linux/hdreg.h>
|
|
#include <linux/init.h>
|
|
#include <linux/platform_device.h>
|
|
#include <linux/module.h>
|
|
#include <linux/moduleparam.h>
|
|
#include <linux/badblocks.h>
|
|
#include <linux/memremap.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/blk-mq.h>
|
|
#include <linux/pfn_t.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/uio.h>
|
|
#include <linux/dax.h>
|
|
#include <linux/nd.h>
|
|
#include <linux/backing-dev.h>
|
|
#include "pmem.h"
|
|
#include "pfn.h"
|
|
#include "nd.h"
|
|
#include "nd-core.h"
|
|
|
|
static struct device *to_dev(struct pmem_device *pmem)
|
|
{
|
|
/*
|
|
* nvdimm bus services need a 'dev' parameter, and we record the device
|
|
* at init in bb.dev.
|
|
*/
|
|
return pmem->bb.dev;
|
|
}
|
|
|
|
static struct nd_region *to_region(struct pmem_device *pmem)
|
|
{
|
|
return to_nd_region(to_dev(pmem)->parent);
|
|
}
|
|
|
|
static blk_status_t pmem_clear_poison(struct pmem_device *pmem,
|
|
phys_addr_t offset, unsigned int len)
|
|
{
|
|
struct device *dev = to_dev(pmem);
|
|
sector_t sector;
|
|
long cleared;
|
|
blk_status_t rc = BLK_STS_OK;
|
|
|
|
sector = (offset - pmem->data_offset) / 512;
|
|
|
|
cleared = nvdimm_clear_poison(dev, pmem->phys_addr + offset, len);
|
|
if (cleared < len)
|
|
rc = BLK_STS_IOERR;
|
|
if (cleared > 0 && cleared / 512) {
|
|
cleared /= 512;
|
|
dev_dbg(dev, "%s: %#llx clear %ld sector%s\n", __func__,
|
|
(unsigned long long) sector, cleared,
|
|
cleared > 1 ? "s" : "");
|
|
badblocks_clear(&pmem->bb, sector, cleared);
|
|
if (pmem->bb_state)
|
|
sysfs_notify_dirent(pmem->bb_state);
|
|
}
|
|
|
|
arch_invalidate_pmem(pmem->virt_addr + offset, len);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static void write_pmem(void *pmem_addr, struct page *page,
|
|
unsigned int off, unsigned int len)
|
|
{
|
|
unsigned int chunk;
|
|
void *mem;
|
|
|
|
while (len) {
|
|
mem = kmap_atomic(page);
|
|
chunk = min_t(unsigned int, len, PAGE_SIZE);
|
|
memcpy_flushcache(pmem_addr, mem + off, chunk);
|
|
kunmap_atomic(mem);
|
|
len -= chunk;
|
|
off = 0;
|
|
page++;
|
|
pmem_addr += PAGE_SIZE;
|
|
}
|
|
}
|
|
|
|
static blk_status_t read_pmem(struct page *page, unsigned int off,
|
|
void *pmem_addr, unsigned int len)
|
|
{
|
|
unsigned int chunk;
|
|
int rc;
|
|
void *mem;
|
|
|
|
while (len) {
|
|
mem = kmap_atomic(page);
|
|
chunk = min_t(unsigned int, len, PAGE_SIZE);
|
|
rc = memcpy_mcsafe(mem + off, pmem_addr, chunk);
|
|
kunmap_atomic(mem);
|
|
if (rc)
|
|
return BLK_STS_IOERR;
|
|
len -= chunk;
|
|
off = 0;
|
|
page++;
|
|
pmem_addr += PAGE_SIZE;
|
|
}
|
|
return BLK_STS_OK;
|
|
}
|
|
|
|
static blk_status_t pmem_do_bvec(struct pmem_device *pmem, struct page *page,
|
|
unsigned int len, unsigned int off, bool is_write,
|
|
sector_t sector)
|
|
{
|
|
blk_status_t rc = BLK_STS_OK;
|
|
bool bad_pmem = false;
|
|
phys_addr_t pmem_off = sector * 512 + pmem->data_offset;
|
|
void *pmem_addr = pmem->virt_addr + pmem_off;
|
|
|
|
if (unlikely(is_bad_pmem(&pmem->bb, sector, len)))
|
|
bad_pmem = true;
|
|
|
|
if (!is_write) {
|
|
if (unlikely(bad_pmem))
|
|
rc = BLK_STS_IOERR;
|
|
else {
|
|
rc = read_pmem(page, off, pmem_addr, len);
|
|
flush_dcache_page(page);
|
|
}
|
|
} else {
|
|
/*
|
|
* Note that we write the data both before and after
|
|
* clearing poison. The write before clear poison
|
|
* handles situations where the latest written data is
|
|
* preserved and the clear poison operation simply marks
|
|
* the address range as valid without changing the data.
|
|
* In this case application software can assume that an
|
|
* interrupted write will either return the new good
|
|
* data or an error.
|
|
*
|
|
* However, if pmem_clear_poison() leaves the data in an
|
|
* indeterminate state we need to perform the write
|
|
* after clear poison.
|
|
*/
|
|
flush_dcache_page(page);
|
|
write_pmem(pmem_addr, page, off, len);
|
|
if (unlikely(bad_pmem)) {
|
|
rc = pmem_clear_poison(pmem, pmem_off, len);
|
|
write_pmem(pmem_addr, page, off, len);
|
|
}
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
/* account for REQ_FLUSH rename, replace with REQ_PREFLUSH after v4.8-rc1 */
|
|
#ifndef REQ_FLUSH
|
|
#define REQ_FLUSH REQ_PREFLUSH
|
|
#endif
|
|
|
|
static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio)
|
|
{
|
|
blk_status_t rc = 0;
|
|
bool do_acct;
|
|
unsigned long start;
|
|
struct bio_vec bvec;
|
|
struct bvec_iter iter;
|
|
struct pmem_device *pmem = q->queuedata;
|
|
struct nd_region *nd_region = to_region(pmem);
|
|
|
|
if (bio->bi_opf & REQ_FLUSH)
|
|
nvdimm_flush(nd_region);
|
|
|
|
do_acct = nd_iostat_start(bio, &start);
|
|
bio_for_each_segment(bvec, bio, iter) {
|
|
rc = pmem_do_bvec(pmem, bvec.bv_page, bvec.bv_len,
|
|
bvec.bv_offset, op_is_write(bio_op(bio)),
|
|
iter.bi_sector);
|
|
if (rc) {
|
|
bio->bi_status = rc;
|
|
break;
|
|
}
|
|
}
|
|
if (do_acct)
|
|
nd_iostat_end(bio, start);
|
|
|
|
if (bio->bi_opf & REQ_FUA)
|
|
nvdimm_flush(nd_region);
|
|
|
|
bio_endio(bio);
|
|
return BLK_QC_T_NONE;
|
|
}
|
|
|
|
static int pmem_rw_page(struct block_device *bdev, sector_t sector,
|
|
struct page *page, bool is_write)
|
|
{
|
|
struct pmem_device *pmem = bdev->bd_queue->queuedata;
|
|
blk_status_t rc;
|
|
|
|
rc = pmem_do_bvec(pmem, page, hpage_nr_pages(page) * PAGE_SIZE,
|
|
0, is_write, sector);
|
|
|
|
/*
|
|
* The ->rw_page interface is subtle and tricky. The core
|
|
* retries on any error, so we can only invoke page_endio() in
|
|
* the successful completion case. Otherwise, we'll see crashes
|
|
* caused by double completion.
|
|
*/
|
|
if (rc == 0)
|
|
page_endio(page, is_write, 0);
|
|
|
|
return blk_status_to_errno(rc);
|
|
}
|
|
|
|
/* see "strong" declaration in tools/testing/nvdimm/pmem-dax.c */
|
|
__weak long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
|
|
long nr_pages, void **kaddr, pfn_t *pfn)
|
|
{
|
|
resource_size_t offset = PFN_PHYS(pgoff) + pmem->data_offset;
|
|
|
|
if (unlikely(is_bad_pmem(&pmem->bb, PFN_PHYS(pgoff) / 512,
|
|
PFN_PHYS(nr_pages))))
|
|
return -EIO;
|
|
*kaddr = pmem->virt_addr + offset;
|
|
*pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);
|
|
|
|
/*
|
|
* If badblocks are present, limit known good range to the
|
|
* requested range.
|
|
*/
|
|
if (unlikely(pmem->bb.count))
|
|
return nr_pages;
|
|
return PHYS_PFN(pmem->size - pmem->pfn_pad - offset);
|
|
}
|
|
|
|
static const struct block_device_operations pmem_fops = {
|
|
.owner = THIS_MODULE,
|
|
.rw_page = pmem_rw_page,
|
|
.revalidate_disk = nvdimm_revalidate_disk,
|
|
};
|
|
|
|
static long pmem_dax_direct_access(struct dax_device *dax_dev,
|
|
pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn)
|
|
{
|
|
struct pmem_device *pmem = dax_get_private(dax_dev);
|
|
|
|
return __pmem_direct_access(pmem, pgoff, nr_pages, kaddr, pfn);
|
|
}
|
|
|
|
static size_t pmem_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
|
|
void *addr, size_t bytes, struct iov_iter *i)
|
|
{
|
|
return copy_from_iter_flushcache(addr, bytes, i);
|
|
}
|
|
|
|
static const struct dax_operations pmem_dax_ops = {
|
|
.direct_access = pmem_dax_direct_access,
|
|
.copy_from_iter = pmem_copy_from_iter,
|
|
};
|
|
|
|
static const struct attribute_group *pmem_attribute_groups[] = {
|
|
&dax_attribute_group,
|
|
NULL,
|
|
};
|
|
|
|
static void pmem_release_queue(void *q)
|
|
{
|
|
blk_cleanup_queue(q);
|
|
}
|
|
|
|
static void pmem_freeze_queue(void *q)
|
|
{
|
|
blk_freeze_queue_start(q);
|
|
}
|
|
|
|
static void pmem_release_disk(void *__pmem)
|
|
{
|
|
struct pmem_device *pmem = __pmem;
|
|
|
|
kill_dax(pmem->dax_dev);
|
|
put_dax(pmem->dax_dev);
|
|
del_gendisk(pmem->disk);
|
|
put_disk(pmem->disk);
|
|
}
|
|
|
|
static int pmem_attach_disk(struct device *dev,
|
|
struct nd_namespace_common *ndns)
|
|
{
|
|
struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev);
|
|
struct nd_region *nd_region = to_nd_region(dev->parent);
|
|
int nid = dev_to_node(dev), fua, wbc;
|
|
struct resource *res = &nsio->res;
|
|
struct resource bb_res;
|
|
struct nd_pfn *nd_pfn = NULL;
|
|
struct dax_device *dax_dev;
|
|
struct nd_pfn_sb *pfn_sb;
|
|
struct pmem_device *pmem;
|
|
struct request_queue *q;
|
|
struct device *gendev;
|
|
struct gendisk *disk;
|
|
void *addr;
|
|
int rc;
|
|
|
|
pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);
|
|
if (!pmem)
|
|
return -ENOMEM;
|
|
|
|
/* while nsio_rw_bytes is active, parse a pfn info block if present */
|
|
if (is_nd_pfn(dev)) {
|
|
nd_pfn = to_nd_pfn(dev);
|
|
rc = nvdimm_setup_pfn(nd_pfn, &pmem->pgmap);
|
|
if (rc)
|
|
return rc;
|
|
}
|
|
|
|
/* we're attaching a block device, disable raw namespace access */
|
|
devm_nsio_disable(dev, nsio);
|
|
|
|
dev_set_drvdata(dev, pmem);
|
|
pmem->phys_addr = res->start;
|
|
pmem->size = resource_size(res);
|
|
fua = nvdimm_has_flush(nd_region);
|
|
if (!IS_ENABLED(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) || fua < 0) {
|
|
dev_warn(dev, "unable to guarantee persistence of writes\n");
|
|
fua = 0;
|
|
}
|
|
wbc = nvdimm_has_cache(nd_region);
|
|
|
|
if (!devm_request_mem_region(dev, res->start, resource_size(res),
|
|
dev_name(&ndns->dev))) {
|
|
dev_warn(dev, "could not reserve region %pR\n", res);
|
|
return -EBUSY;
|
|
}
|
|
|
|
q = blk_alloc_queue_node(GFP_KERNEL, dev_to_node(dev));
|
|
if (!q)
|
|
return -ENOMEM;
|
|
|
|
if (devm_add_action_or_reset(dev, pmem_release_queue, q))
|
|
return -ENOMEM;
|
|
|
|
pmem->pfn_flags = PFN_DEV;
|
|
pmem->pgmap.ref = &q->q_usage_counter;
|
|
if (is_nd_pfn(dev)) {
|
|
addr = devm_memremap_pages(dev, &pmem->pgmap);
|
|
pfn_sb = nd_pfn->pfn_sb;
|
|
pmem->data_offset = le64_to_cpu(pfn_sb->dataoff);
|
|
pmem->pfn_pad = resource_size(res) -
|
|
resource_size(&pmem->pgmap.res);
|
|
pmem->pfn_flags |= PFN_MAP;
|
|
memcpy(&bb_res, &pmem->pgmap.res, sizeof(bb_res));
|
|
bb_res.start += pmem->data_offset;
|
|
} else if (pmem_should_map_pages(dev)) {
|
|
memcpy(&pmem->pgmap.res, &nsio->res, sizeof(pmem->pgmap.res));
|
|
pmem->pgmap.altmap_valid = false;
|
|
addr = devm_memremap_pages(dev, &pmem->pgmap);
|
|
pmem->pfn_flags |= PFN_MAP;
|
|
memcpy(&bb_res, &pmem->pgmap.res, sizeof(bb_res));
|
|
} else
|
|
addr = devm_memremap(dev, pmem->phys_addr,
|
|
pmem->size, ARCH_MEMREMAP_PMEM);
|
|
|
|
/*
|
|
* At release time the queue must be frozen before
|
|
* devm_memremap_pages is unwound
|
|
*/
|
|
if (devm_add_action_or_reset(dev, pmem_freeze_queue, q))
|
|
return -ENOMEM;
|
|
|
|
if (IS_ERR(addr))
|
|
return PTR_ERR(addr);
|
|
pmem->virt_addr = addr;
|
|
|
|
blk_queue_write_cache(q, wbc, fua);
|
|
blk_queue_make_request(q, pmem_make_request);
|
|
blk_queue_physical_block_size(q, PAGE_SIZE);
|
|
blk_queue_logical_block_size(q, pmem_sector_size(ndns));
|
|
blk_queue_max_hw_sectors(q, UINT_MAX);
|
|
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
|
|
queue_flag_set_unlocked(QUEUE_FLAG_DAX, q);
|
|
q->queuedata = pmem;
|
|
|
|
disk = alloc_disk_node(0, nid);
|
|
if (!disk)
|
|
return -ENOMEM;
|
|
pmem->disk = disk;
|
|
|
|
disk->fops = &pmem_fops;
|
|
disk->queue = q;
|
|
disk->flags = GENHD_FL_EXT_DEVT;
|
|
disk->queue->backing_dev_info->capabilities |= BDI_CAP_SYNCHRONOUS_IO;
|
|
nvdimm_namespace_disk_name(ndns, disk->disk_name);
|
|
set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset)
|
|
/ 512);
|
|
if (devm_init_badblocks(dev, &pmem->bb))
|
|
return -ENOMEM;
|
|
nvdimm_badblocks_populate(nd_region, &pmem->bb, &bb_res);
|
|
disk->bb = &pmem->bb;
|
|
|
|
dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops);
|
|
if (!dax_dev) {
|
|
put_disk(disk);
|
|
return -ENOMEM;
|
|
}
|
|
dax_write_cache(dax_dev, wbc);
|
|
pmem->dax_dev = dax_dev;
|
|
|
|
gendev = disk_to_dev(disk);
|
|
gendev->groups = pmem_attribute_groups;
|
|
|
|
device_add_disk(dev, disk);
|
|
if (devm_add_action_or_reset(dev, pmem_release_disk, pmem))
|
|
return -ENOMEM;
|
|
|
|
revalidate_disk(disk);
|
|
|
|
pmem->bb_state = sysfs_get_dirent(disk_to_dev(disk)->kobj.sd,
|
|
"badblocks");
|
|
if (!pmem->bb_state)
|
|
dev_warn(dev, "'badblocks' notification disabled\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int nd_pmem_probe(struct device *dev)
|
|
{
|
|
struct nd_namespace_common *ndns;
|
|
|
|
ndns = nvdimm_namespace_common_probe(dev);
|
|
if (IS_ERR(ndns))
|
|
return PTR_ERR(ndns);
|
|
|
|
if (devm_nsio_enable(dev, to_nd_namespace_io(&ndns->dev)))
|
|
return -ENXIO;
|
|
|
|
if (is_nd_btt(dev))
|
|
return nvdimm_namespace_attach_btt(ndns);
|
|
|
|
if (is_nd_pfn(dev))
|
|
return pmem_attach_disk(dev, ndns);
|
|
|
|
/* if we find a valid info-block we'll come back as that personality */
|
|
if (nd_btt_probe(dev, ndns) == 0 || nd_pfn_probe(dev, ndns) == 0
|
|
|| nd_dax_probe(dev, ndns) == 0)
|
|
return -ENXIO;
|
|
|
|
/* ...otherwise we're just a raw pmem device */
|
|
return pmem_attach_disk(dev, ndns);
|
|
}
|
|
|
|
static int nd_pmem_remove(struct device *dev)
|
|
{
|
|
struct pmem_device *pmem = dev_get_drvdata(dev);
|
|
|
|
if (is_nd_btt(dev))
|
|
nvdimm_namespace_detach_btt(to_nd_btt(dev));
|
|
else {
|
|
/*
|
|
* Note, this assumes device_lock() context to not race
|
|
* nd_pmem_notify()
|
|
*/
|
|
sysfs_put(pmem->bb_state);
|
|
pmem->bb_state = NULL;
|
|
}
|
|
nvdimm_flush(to_nd_region(dev->parent));
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void nd_pmem_shutdown(struct device *dev)
|
|
{
|
|
nvdimm_flush(to_nd_region(dev->parent));
|
|
}
|
|
|
|
static void nd_pmem_notify(struct device *dev, enum nvdimm_event event)
|
|
{
|
|
struct nd_region *nd_region;
|
|
resource_size_t offset = 0, end_trunc = 0;
|
|
struct nd_namespace_common *ndns;
|
|
struct nd_namespace_io *nsio;
|
|
struct resource res;
|
|
struct badblocks *bb;
|
|
struct kernfs_node *bb_state;
|
|
|
|
if (event != NVDIMM_REVALIDATE_POISON)
|
|
return;
|
|
|
|
if (is_nd_btt(dev)) {
|
|
struct nd_btt *nd_btt = to_nd_btt(dev);
|
|
|
|
ndns = nd_btt->ndns;
|
|
nd_region = to_nd_region(ndns->dev.parent);
|
|
nsio = to_nd_namespace_io(&ndns->dev);
|
|
bb = &nsio->bb;
|
|
bb_state = NULL;
|
|
} else {
|
|
struct pmem_device *pmem = dev_get_drvdata(dev);
|
|
|
|
nd_region = to_region(pmem);
|
|
bb = &pmem->bb;
|
|
bb_state = pmem->bb_state;
|
|
|
|
if (is_nd_pfn(dev)) {
|
|
struct nd_pfn *nd_pfn = to_nd_pfn(dev);
|
|
struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb;
|
|
|
|
ndns = nd_pfn->ndns;
|
|
offset = pmem->data_offset +
|
|
__le32_to_cpu(pfn_sb->start_pad);
|
|
end_trunc = __le32_to_cpu(pfn_sb->end_trunc);
|
|
} else {
|
|
ndns = to_ndns(dev);
|
|
}
|
|
|
|
nsio = to_nd_namespace_io(&ndns->dev);
|
|
}
|
|
|
|
res.start = nsio->res.start + offset;
|
|
res.end = nsio->res.end - end_trunc;
|
|
nvdimm_badblocks_populate(nd_region, bb, &res);
|
|
if (bb_state)
|
|
sysfs_notify_dirent(bb_state);
|
|
}
|
|
|
|
MODULE_ALIAS("pmem");
|
|
MODULE_ALIAS_ND_DEVICE(ND_DEVICE_NAMESPACE_IO);
|
|
MODULE_ALIAS_ND_DEVICE(ND_DEVICE_NAMESPACE_PMEM);
|
|
static struct nd_device_driver nd_pmem_driver = {
|
|
.probe = nd_pmem_probe,
|
|
.remove = nd_pmem_remove,
|
|
.notify = nd_pmem_notify,
|
|
.shutdown = nd_pmem_shutdown,
|
|
.drv = {
|
|
.name = "nd_pmem",
|
|
},
|
|
.type = ND_DRIVER_NAMESPACE_IO | ND_DRIVER_NAMESPACE_PMEM,
|
|
};
|
|
|
|
static int __init pmem_init(void)
|
|
{
|
|
return nd_driver_register(&nd_pmem_driver);
|
|
}
|
|
module_init(pmem_init);
|
|
|
|
static void pmem_exit(void)
|
|
{
|
|
driver_unregister(&nd_pmem_driver.drv);
|
|
}
|
|
module_exit(pmem_exit);
|
|
|
|
MODULE_AUTHOR("Ross Zwisler <ross.zwisler@linux.intel.com>");
|
|
MODULE_LICENSE("GPL v2");
|