virtio: Add virtio-over-ivshmem transport driver
This provides a virtio transport driver over the Inter-VM shared memory device as found in QEMU and the Jailhouse hypervisor. ... Note: Specification work for both ivshmem and the virtio transport is ongoing, so details may still change. Acked-by: Ye Li <ye.li@nxp.com> Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>5.4-rM2-2.2.x-imx-squashed
Jan Kiszka
Peng Fan
parent
92de0e9899
commit
3ea2de8648
|
|
@ -95,4 +95,14 @@ config VIRTIO_MMIO_CMDLINE_DEVICES
|
|||
|
||||
If unsure, say 'N'.
|
||||
|
||||
config VIRTIO_IVSHMEM
|
||||
tristate "Driver for ivshmem-based virtio front-end devices"
|
||||
depends on PCI && !HIGHMEM
|
||||
select VIRTIO
|
||||
---help---
|
||||
This provides virtio front-end devices via ivshmem shared memory
|
||||
devices.
|
||||
|
||||
If unsure, say 'N'.
|
||||
|
||||
endif # VIRTIO_MENU
|
||||
|
|
|
|||
|
|
@ -4,5 +4,6 @@ obj-$(CONFIG_VIRTIO_MMIO) += virtio_mmio.o
|
|||
obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
|
||||
virtio_pci-y := virtio_pci_modern.o virtio_pci_common.o
|
||||
virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o
|
||||
obj-$(CONFIG_VIRTIO_IVSHMEM) += virtio_ivshmem.o
|
||||
obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o
|
||||
obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o
|
||||
|
|
|
|||
|
|
@ -0,0 +1,949 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Virtio over ivshmem front-end device driver
|
||||
*
|
||||
* Copyright (c) Siemens AG, 2019
|
||||
*/
|
||||
|
||||
#include <linux/delay.h>
|
||||
#include <linux/ivshmem.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/virtio.h>
|
||||
#include <linux/virtio_config.h>
|
||||
#include <linux/virtio_ring.h>
|
||||
|
||||
#define DRV_NAME "virtio-ivshmem"
|
||||
|
||||
#define VIRTIO_IVSHMEM_PREFERRED_ALLOC_CHUNKS 4096
|
||||
|
||||
#define VIRTIO_STATE_READY cpu_to_le32(1)
|
||||
|
||||
struct virtio_ivshmem_header {
|
||||
__le32 revision;
|
||||
__le32 size;
|
||||
|
||||
__le32 write_transaction;
|
||||
|
||||
__le32 device_features;
|
||||
__le32 device_features_sel;
|
||||
__le32 driver_features;
|
||||
__le32 driver_features_sel;
|
||||
|
||||
__le32 queue_sel;
|
||||
|
||||
__le16 queue_size;
|
||||
__le16 queue_device_vector;
|
||||
__le16 queue_driver_vector;
|
||||
__le16 queue_enable;
|
||||
__le64 queue_desc;
|
||||
__le64 queue_driver;
|
||||
__le64 queue_device;
|
||||
|
||||
__u8 config_event;
|
||||
__u8 queue_event;
|
||||
__u8 __reserved[2];
|
||||
__le32 device_status;
|
||||
|
||||
__le32 config_generation;
|
||||
__u8 config[];
|
||||
};
|
||||
|
||||
#define VI_REG_OFFSET(reg) offsetof(struct virtio_ivshmem_header, reg)
|
||||
|
||||
struct virtio_ivshmem_device {
|
||||
struct virtio_device vdev;
|
||||
struct pci_dev *pci_dev;
|
||||
|
||||
struct ivshm_regs __iomem *ivshm_regs;
|
||||
|
||||
unsigned int num_vectors;
|
||||
bool per_vq_vector;
|
||||
char *config_irq_name;
|
||||
char *queues_irq_name;
|
||||
|
||||
u32 peer_id;
|
||||
u32 *peer_state;
|
||||
|
||||
void *shmem;
|
||||
resource_size_t shmem_sz;
|
||||
struct virtio_ivshmem_header *virtio_header;
|
||||
|
||||
spinlock_t alloc_lock;
|
||||
unsigned long *alloc_bitmap;
|
||||
unsigned int alloc_shift;
|
||||
void **map_src_addr;
|
||||
|
||||
/* a list of queues so we can dispatch IRQs */
|
||||
spinlock_t virtqueues_lock;
|
||||
struct list_head virtqueues;
|
||||
};
|
||||
|
||||
struct virtio_ivshmem_vq_info {
|
||||
/* the actual virtqueue */
|
||||
struct virtqueue *vq;
|
||||
|
||||
/* vector to use for signaling the device */
|
||||
unsigned int device_vector;
|
||||
/* vector used by the device for signaling the driver */
|
||||
unsigned int driver_vector;
|
||||
|
||||
char *irq_name;
|
||||
|
||||
/* the list node for the virtqueues list */
|
||||
struct list_head node;
|
||||
};
|
||||
|
||||
static inline unsigned int get_custom_order(unsigned long size,
|
||||
unsigned int shift)
|
||||
{
|
||||
size--;
|
||||
size >>= shift;
|
||||
#if BITS_PER_LONG == 32
|
||||
return fls(size);
|
||||
#else
|
||||
return fls64(size);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline struct virtio_ivshmem_device *
|
||||
to_virtio_ivshmem_device(struct virtio_device *vdev)
|
||||
{
|
||||
return container_of(vdev, struct virtio_ivshmem_device, vdev);
|
||||
}
|
||||
|
||||
static bool vi_synchronize_reg_write(struct virtio_ivshmem_device *vi_dev)
|
||||
{
|
||||
while (READ_ONCE(vi_dev->virtio_header->write_transaction)) {
|
||||
if (READ_ONCE(*vi_dev->peer_state) != VIRTIO_STATE_READY) {
|
||||
dev_err_ratelimited(&vi_dev->pci_dev->dev,
|
||||
"backend failed!");
|
||||
return false;
|
||||
}
|
||||
cpu_relax();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool vi_reg_write(struct virtio_ivshmem_device *vi_dev, unsigned int reg,
|
||||
u64 value, unsigned int size)
|
||||
{
|
||||
u8 *reg_area = (u8 *)vi_dev->virtio_header;
|
||||
|
||||
if (!vi_synchronize_reg_write(vi_dev))
|
||||
return false;
|
||||
|
||||
if (size == 1)
|
||||
*(u8 *)(reg_area + reg) = (u8)value;
|
||||
else if (size == 2)
|
||||
*(u16 *)(reg_area + reg) = cpu_to_le16((u16)value);
|
||||
else if (size == 4)
|
||||
*(u32 *)(reg_area + reg) = cpu_to_le32((u32)value);
|
||||
else if (size == 8)
|
||||
*(u64 *)(reg_area + reg) = cpu_to_le64(value);
|
||||
else
|
||||
BUG();
|
||||
virt_wmb();
|
||||
|
||||
vi_dev->virtio_header->write_transaction = cpu_to_le32(reg);
|
||||
virt_wmb();
|
||||
|
||||
writel((vi_dev->peer_id << 16), &vi_dev->ivshm_regs->doorbell);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool vi_reg_write16(struct virtio_ivshmem_device *vi_dev,
|
||||
unsigned int reg, u32 value)
|
||||
{
|
||||
return vi_reg_write(vi_dev, reg, value, 2);
|
||||
}
|
||||
|
||||
static bool vi_reg_write32(struct virtio_ivshmem_device *vi_dev,
|
||||
unsigned int reg, u32 value)
|
||||
{
|
||||
return vi_reg_write(vi_dev, reg, value, 4);
|
||||
}
|
||||
|
||||
static bool vi_reg_write64(struct virtio_ivshmem_device *vi_dev,
|
||||
unsigned int reg, u64 value)
|
||||
{
|
||||
return vi_reg_write(vi_dev, reg, value, 8);
|
||||
}
|
||||
|
||||
static void vi_get(struct virtio_device *vdev, unsigned offset,
|
||||
void *buf, unsigned len)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
__le16 w;
|
||||
__le32 l;
|
||||
__le64 q;
|
||||
|
||||
switch (len) {
|
||||
case 1:
|
||||
*(u8 *)buf = *(u8 *)(vi_dev->virtio_header->config + offset);
|
||||
break;
|
||||
case 2:
|
||||
w = *(u16 *)(vi_dev->virtio_header->config + offset);
|
||||
*(u16 *)buf = le16_to_cpu(w);
|
||||
break;
|
||||
case 4:
|
||||
l = *(u32 *)(vi_dev->virtio_header->config + offset);
|
||||
*(u32 *)buf = le32_to_cpu(l);
|
||||
break;
|
||||
case 8:
|
||||
q = *(u64 *)(vi_dev->virtio_header->config + offset);
|
||||
*(u64 *)buf = le64_to_cpu(q);
|
||||
break;
|
||||
default:
|
||||
BUG();
|
||||
}
|
||||
}
|
||||
|
||||
static void vi_set(struct virtio_device *vdev, unsigned offset,
|
||||
const void *buf, unsigned len)
|
||||
{
|
||||
u64 value;
|
||||
|
||||
switch (len) {
|
||||
case 1:
|
||||
value = *(u8 *)buf;
|
||||
break;
|
||||
case 2:
|
||||
value = *(u16 *)buf;
|
||||
break;
|
||||
case 4:
|
||||
value = *(u32 *)buf;
|
||||
break;
|
||||
case 8:
|
||||
value = *(u64 *)buf;
|
||||
break;
|
||||
default:
|
||||
BUG();
|
||||
}
|
||||
vi_reg_write(to_virtio_ivshmem_device(vdev),
|
||||
offsetof(struct virtio_ivshmem_header, config) + offset,
|
||||
value, len);
|
||||
}
|
||||
|
||||
static u32 vi_generation(struct virtio_device *vdev)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
u32 gen = READ_ONCE(vi_dev->virtio_header->config_generation);
|
||||
|
||||
while (gen & 1) {
|
||||
if (READ_ONCE(*vi_dev->peer_state) != VIRTIO_STATE_READY) {
|
||||
dev_err_ratelimited(&vi_dev->pci_dev->dev,
|
||||
"backend failed!");
|
||||
return 0;
|
||||
}
|
||||
cpu_relax();
|
||||
|
||||
gen = READ_ONCE(vi_dev->virtio_header->config_generation);
|
||||
}
|
||||
return gen;
|
||||
}
|
||||
|
||||
static u8 vi_get_status(struct virtio_device *vdev)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
|
||||
return le32_to_cpu(vi_dev->virtio_header->device_status) & 0xff;
|
||||
}
|
||||
|
||||
static void vi_set_status(struct virtio_device *vdev, u8 status)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
|
||||
/* We should never be setting status to 0. */
|
||||
BUG_ON(status == 0);
|
||||
|
||||
vi_reg_write32(vi_dev, VI_REG_OFFSET(device_status), status);
|
||||
}
|
||||
|
||||
static void vi_reset(struct virtio_device *vdev)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
|
||||
/* 0 status means a reset. */
|
||||
vi_reg_write32(vi_dev, VI_REG_OFFSET(device_status), 0);
|
||||
}
|
||||
|
||||
static u64 vi_get_features(struct virtio_device *vdev)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
u64 features;
|
||||
|
||||
if (!vi_reg_write32(vi_dev, VI_REG_OFFSET(device_features_sel), 1) ||
|
||||
!vi_synchronize_reg_write(vi_dev))
|
||||
return 0;
|
||||
features = le32_to_cpu(vi_dev->virtio_header->device_features);
|
||||
features <<= 32;
|
||||
|
||||
if (!vi_reg_write32(vi_dev, VI_REG_OFFSET(device_features_sel), 0) ||
|
||||
!vi_synchronize_reg_write(vi_dev))
|
||||
return 0;
|
||||
features |= le32_to_cpu(vi_dev->virtio_header->device_features);
|
||||
|
||||
return features;
|
||||
}
|
||||
|
||||
static int vi_finalize_features(struct virtio_device *vdev)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
|
||||
/* Give virtio_ring a chance to accept features. */
|
||||
vring_transport_features(vdev);
|
||||
|
||||
if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) {
|
||||
dev_err(&vdev->dev,
|
||||
"virtio: device does not have VIRTIO_F_VERSION_1\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (!vi_reg_write32(vi_dev, VI_REG_OFFSET(driver_features_sel), 1) ||
|
||||
!vi_reg_write32(vi_dev, VI_REG_OFFSET(driver_features),
|
||||
(u32)(vdev->features >> 32)))
|
||||
return -ENODEV;
|
||||
|
||||
if (!vi_reg_write32(vi_dev, VI_REG_OFFSET(driver_features_sel), 0) ||
|
||||
!vi_reg_write32(vi_dev, VI_REG_OFFSET(driver_features),
|
||||
(u32)vdev->features))
|
||||
return -ENODEV;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* the notify function used when creating a virt queue */
|
||||
static bool vi_notify(struct virtqueue *vq)
|
||||
{
|
||||
struct virtio_ivshmem_vq_info *info = vq->priv;
|
||||
struct virtio_ivshmem_device *vi_dev =
|
||||
to_virtio_ivshmem_device(vq->vdev);
|
||||
|
||||
virt_wmb();
|
||||
writel((vi_dev->peer_id << 16) | info->device_vector,
|
||||
&vi_dev->ivshm_regs->doorbell);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static irqreturn_t vi_config_interrupt(int irq, void *opaque)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = opaque;
|
||||
|
||||
if (unlikely(READ_ONCE(*vi_dev->peer_state) != VIRTIO_STATE_READY)) {
|
||||
virtio_break_device(&vi_dev->vdev);
|
||||
vi_dev->virtio_header->config_event = 0;
|
||||
vi_dev->virtio_header->queue_event = 0;
|
||||
dev_err(&vi_dev->pci_dev->dev, "backend failed!");
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
if (unlikely(READ_ONCE(vi_dev->virtio_header->config_event) & 1)) {
|
||||
vi_dev->virtio_header->config_event = 0;
|
||||
virt_wmb();
|
||||
virtio_config_changed(&vi_dev->vdev);
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
return IRQ_NONE;
|
||||
}
|
||||
|
||||
static irqreturn_t vi_queues_interrupt(int irq, void *opaque)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = opaque;
|
||||
struct virtio_ivshmem_vq_info *info;
|
||||
irqreturn_t ret = IRQ_NONE;
|
||||
|
||||
if (likely(READ_ONCE(vi_dev->virtio_header->queue_event) & 1)) {
|
||||
vi_dev->virtio_header->queue_event = 0;
|
||||
virt_wmb();
|
||||
spin_lock(&vi_dev->virtqueues_lock);
|
||||
list_for_each_entry(info, &vi_dev->virtqueues, node)
|
||||
ret |= vring_interrupt(irq, info->vq);
|
||||
spin_unlock(&vi_dev->virtqueues_lock);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static irqreturn_t vi_interrupt(int irq, void *opaque)
|
||||
{
|
||||
return vi_config_interrupt(irq, opaque) |
|
||||
vi_queues_interrupt(irq, opaque);
|
||||
}
|
||||
|
||||
static struct virtqueue *vi_setup_vq(struct virtio_device *vdev,
|
||||
unsigned int index,
|
||||
void (*callback)(struct virtqueue *vq),
|
||||
const char *name, bool ctx,
|
||||
unsigned int irq_vector)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
struct virtio_ivshmem_vq_info *info;
|
||||
struct virtqueue *vq;
|
||||
unsigned long flags;
|
||||
unsigned int size;
|
||||
int irq, err;
|
||||
|
||||
/* Select the queue we're interested in */
|
||||
if (!vi_reg_write32(vi_dev, VI_REG_OFFSET(queue_sel), index) ||
|
||||
!vi_synchronize_reg_write(vi_dev))
|
||||
return ERR_PTR(-ENODEV);
|
||||
|
||||
/* Queue shouldn't already be set up. */
|
||||
if (vi_dev->virtio_header->queue_enable)
|
||||
return ERR_PTR(-ENOENT);
|
||||
|
||||
/* Allocate and fill out our active queue description */
|
||||
info = kzalloc(sizeof(*info), GFP_KERNEL);
|
||||
if (!info)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
size = vi_dev->virtio_header->queue_size;
|
||||
if (size == 0) {
|
||||
err = -ENOENT;
|
||||
goto error_new_virtqueue;
|
||||
}
|
||||
|
||||
info->device_vector = vi_dev->virtio_header->queue_device_vector;
|
||||
info->driver_vector = irq_vector;
|
||||
|
||||
/* Create the vring */
|
||||
vq = vring_create_virtqueue(index, size, SMP_CACHE_BYTES, vdev, true,
|
||||
true, ctx, vi_notify, callback, name);
|
||||
if (!vq) {
|
||||
err = -ENOMEM;
|
||||
goto error_new_virtqueue;
|
||||
}
|
||||
|
||||
if (callback && vi_dev->per_vq_vector) {
|
||||
irq = pci_irq_vector(vi_dev->pci_dev, info->driver_vector);
|
||||
info->irq_name = kasprintf(GFP_KERNEL, "%s-%s",
|
||||
dev_name(&vdev->dev), name);
|
||||
if (!info->irq_name) {
|
||||
err = -ENOMEM;
|
||||
goto error_setup_virtqueue;
|
||||
}
|
||||
|
||||
err = request_irq(irq, vring_interrupt, 0, info->irq_name, vq);
|
||||
if (err)
|
||||
goto error_setup_virtqueue;
|
||||
}
|
||||
|
||||
/* Activate the queue */
|
||||
if (!vi_reg_write16(vi_dev, VI_REG_OFFSET(queue_size),
|
||||
virtqueue_get_vring_size(vq)) ||
|
||||
!vi_reg_write16(vi_dev, VI_REG_OFFSET(queue_driver_vector),
|
||||
info->driver_vector) ||
|
||||
!vi_reg_write64(vi_dev, VI_REG_OFFSET(queue_desc),
|
||||
virtqueue_get_desc_addr(vq)) ||
|
||||
!vi_reg_write64(vi_dev, VI_REG_OFFSET(queue_driver),
|
||||
virtqueue_get_avail_addr(vq)) ||
|
||||
!vi_reg_write64(vi_dev, VI_REG_OFFSET(queue_device),
|
||||
virtqueue_get_used_addr(vq)) ||
|
||||
!vi_reg_write16(vi_dev, VI_REG_OFFSET(queue_enable), 1)) {
|
||||
err = -ENODEV;
|
||||
goto error_setup_virtqueue;
|
||||
}
|
||||
|
||||
vq->priv = info;
|
||||
info->vq = vq;
|
||||
|
||||
spin_lock_irqsave(&vi_dev->virtqueues_lock, flags);
|
||||
list_add(&info->node, &vi_dev->virtqueues);
|
||||
spin_unlock_irqrestore(&vi_dev->virtqueues_lock, flags);
|
||||
|
||||
return vq;
|
||||
|
||||
error_setup_virtqueue:
|
||||
vring_del_virtqueue(vq);
|
||||
|
||||
error_new_virtqueue:
|
||||
vi_reg_write32(vi_dev, VI_REG_OFFSET(queue_enable), 0);
|
||||
kfree(info);
|
||||
return ERR_PTR(err);
|
||||
}
|
||||
|
||||
static void vi_del_vq(struct virtqueue *vq)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev =
|
||||
to_virtio_ivshmem_device(vq->vdev);
|
||||
struct virtio_ivshmem_vq_info *info = vq->priv;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&vi_dev->virtqueues_lock, flags);
|
||||
list_del(&info->node);
|
||||
spin_unlock_irqrestore(&vi_dev->virtqueues_lock, flags);
|
||||
|
||||
/* Select and deactivate the queue */
|
||||
vi_reg_write32(vi_dev, VI_REG_OFFSET(queue_sel), vq->index);
|
||||
vi_reg_write32(vi_dev, VI_REG_OFFSET(queue_enable), 0);
|
||||
|
||||
vring_del_virtqueue(vq);
|
||||
|
||||
if (info->driver_vector) {
|
||||
free_irq(pci_irq_vector(vi_dev->pci_dev, info->driver_vector),
|
||||
vq);
|
||||
kfree(info->irq_name);
|
||||
}
|
||||
|
||||
kfree(info);
|
||||
}
|
||||
|
||||
static void vi_del_vqs(struct virtio_device *vdev)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
struct virtqueue *vq, *n;
|
||||
|
||||
list_for_each_entry_safe(vq, n, &vdev->vqs, list)
|
||||
vi_del_vq(vq);
|
||||
|
||||
free_irq(pci_irq_vector(vi_dev->pci_dev, 0), vi_dev);
|
||||
if (!vi_dev->per_vq_vector && vi_dev->num_vectors > 1)
|
||||
free_irq(pci_irq_vector(vi_dev->pci_dev, 1), vi_dev);
|
||||
pci_free_irq_vectors(vi_dev->pci_dev);
|
||||
|
||||
kfree(vi_dev->config_irq_name);
|
||||
vi_dev->config_irq_name = NULL;
|
||||
kfree(vi_dev->queues_irq_name);
|
||||
vi_dev->queues_irq_name = NULL;
|
||||
}
|
||||
|
||||
static int vi_find_vqs(struct virtio_device *vdev, unsigned nvqs,
|
||||
struct virtqueue *vqs[],
|
||||
vq_callback_t *callbacks[],
|
||||
const char * const names[],
|
||||
const bool *ctx,
|
||||
struct irq_affinity *desc)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
unsigned int vq_vector, desired_vectors;
|
||||
int err, vectors, i, queue_idx = 0;
|
||||
|
||||
desired_vectors = 1; /* one for config events */
|
||||
for (i = 0; i < nvqs; i++)
|
||||
if (callbacks[i])
|
||||
desired_vectors++;
|
||||
|
||||
vectors = pci_alloc_irq_vectors(vi_dev->pci_dev, desired_vectors,
|
||||
desired_vectors, PCI_IRQ_MSIX);
|
||||
if (vectors != desired_vectors) {
|
||||
vectors = pci_alloc_irq_vectors(vi_dev->pci_dev, 1, 2,
|
||||
PCI_IRQ_LEGACY | PCI_IRQ_MSIX);
|
||||
if (vectors < 0)
|
||||
return vectors;
|
||||
}
|
||||
|
||||
vi_dev->num_vectors = vectors;
|
||||
vi_dev->per_vq_vector = vectors == desired_vectors;
|
||||
|
||||
if (vectors == 1) {
|
||||
vq_vector = 0;
|
||||
err = request_irq(pci_irq_vector(vi_dev->pci_dev, 0),
|
||||
vi_interrupt, IRQF_SHARED,
|
||||
dev_name(&vdev->dev), vi_dev);
|
||||
if (err)
|
||||
goto error_common_irq;
|
||||
} else {
|
||||
vq_vector = 1;
|
||||
vi_dev->config_irq_name = kasprintf(GFP_KERNEL, "%s-config",
|
||||
dev_name(&vdev->dev));
|
||||
if (!vi_dev->config_irq_name) {
|
||||
err = -ENOMEM;
|
||||
goto error_common_irq;
|
||||
}
|
||||
|
||||
err = request_irq(pci_irq_vector(vi_dev->pci_dev, 0),
|
||||
vi_config_interrupt, 0,
|
||||
vi_dev->config_irq_name, vi_dev);
|
||||
if (err)
|
||||
goto error_common_irq;
|
||||
}
|
||||
|
||||
if (!vi_dev->per_vq_vector && vectors > 1) {
|
||||
vi_dev->queues_irq_name = kasprintf(GFP_KERNEL, "%s-virtqueues",
|
||||
dev_name(&vdev->dev));
|
||||
if (!vi_dev->queues_irq_name) {
|
||||
err = -ENOMEM;
|
||||
goto error_queues_irq;
|
||||
}
|
||||
|
||||
err = request_irq(pci_irq_vector(vi_dev->pci_dev, 1),
|
||||
vi_queues_interrupt, 0,
|
||||
vi_dev->queues_irq_name, vi_dev);
|
||||
if (err)
|
||||
goto error_queues_irq;
|
||||
}
|
||||
|
||||
for (i = 0; i < nvqs; ++i) {
|
||||
if (!names[i]) {
|
||||
vqs[i] = NULL;
|
||||
continue;
|
||||
}
|
||||
|
||||
vqs[i] = vi_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
|
||||
ctx ? ctx[i] : false, vq_vector);
|
||||
if (IS_ERR(vqs[i])) {
|
||||
vi_del_vqs(vdev);
|
||||
return PTR_ERR(vqs[i]);
|
||||
}
|
||||
|
||||
if (vi_dev->per_vq_vector)
|
||||
vq_vector++;
|
||||
}
|
||||
|
||||
writel(IVSHM_INT_ENABLE, &vi_dev->ivshm_regs->int_control);
|
||||
|
||||
return 0;
|
||||
|
||||
error_queues_irq:
|
||||
free_irq(pci_irq_vector(vi_dev->pci_dev, 0), vi_dev);
|
||||
kfree(vi_dev->config_irq_name);
|
||||
vi_dev->config_irq_name = NULL;
|
||||
|
||||
error_common_irq:
|
||||
kfree(vi_dev->queues_irq_name);
|
||||
vi_dev->queues_irq_name = NULL;
|
||||
pci_free_irq_vectors(vi_dev->pci_dev);
|
||||
return err;
|
||||
}
|
||||
|
||||
static const char *vi_bus_name(struct virtio_device *vdev)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
|
||||
return pci_name(vi_dev->pci_dev);
|
||||
}
|
||||
|
||||
static const struct virtio_config_ops virtio_ivshmem_config_ops = {
|
||||
.get = vi_get,
|
||||
.set = vi_set,
|
||||
.generation = vi_generation,
|
||||
.get_status = vi_get_status,
|
||||
.set_status = vi_set_status,
|
||||
.reset = vi_reset,
|
||||
.find_vqs = vi_find_vqs,
|
||||
.del_vqs = vi_del_vqs,
|
||||
.get_features = vi_get_features,
|
||||
.finalize_features = vi_finalize_features,
|
||||
.bus_name = vi_bus_name,
|
||||
};
|
||||
|
||||
static void virtio_ivshmem_release_dev(struct device *_d)
|
||||
{
|
||||
struct virtio_device *vdev = dev_to_virtio(_d);
|
||||
struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
|
||||
|
||||
devm_kfree(&vi_dev->pci_dev->dev, vi_dev);
|
||||
}
|
||||
|
||||
static u64 get_config_qword(struct pci_dev *pci_dev, unsigned int pos)
|
||||
{
|
||||
u32 lo, hi;
|
||||
|
||||
pci_read_config_dword(pci_dev, pos, &lo);
|
||||
pci_read_config_dword(pci_dev, pos + 4, &hi);
|
||||
return lo | ((u64)hi << 32);
|
||||
}
|
||||
|
||||
static void *vi_dma_alloc(struct device *dev, size_t size,
|
||||
dma_addr_t *dma_handle, gfp_t flag,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct pci_dev *pci_dev = to_pci_dev(dev);
|
||||
struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
|
||||
int order = get_custom_order(size, vi_dev->alloc_shift);
|
||||
int chunk = -ENOMEM;
|
||||
unsigned long flags;
|
||||
void *addr;
|
||||
|
||||
spin_lock_irqsave(&vi_dev->alloc_lock, flags);
|
||||
chunk = bitmap_find_free_region(vi_dev->alloc_bitmap,
|
||||
vi_dev->shmem_sz >> vi_dev->alloc_shift,
|
||||
order);
|
||||
spin_unlock_irqrestore(&vi_dev->alloc_lock, flags);
|
||||
|
||||
if (chunk < 0) {
|
||||
if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit())
|
||||
dev_warn(dev,
|
||||
"shared memory is full (size: %zd bytes)\n",
|
||||
size);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
*dma_handle = chunk << vi_dev->alloc_shift;
|
||||
addr = vi_dev->shmem + *dma_handle;
|
||||
memset(addr, 0, size);
|
||||
|
||||
return addr;
|
||||
}
|
||||
|
||||
static void vi_dma_free(struct device *dev, size_t size, void *vaddr,
|
||||
dma_addr_t dma_handle, unsigned long attrs)
|
||||
{
|
||||
struct pci_dev *pci_dev = to_pci_dev(dev);
|
||||
struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
|
||||
int order = get_custom_order(size, vi_dev->alloc_shift);
|
||||
int chunk = (int)(dma_handle >> vi_dev->alloc_shift);
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&vi_dev->alloc_lock, flags);
|
||||
bitmap_release_region(vi_dev->alloc_bitmap, chunk, order);
|
||||
spin_unlock_irqrestore(&vi_dev->alloc_lock, flags);
|
||||
}
|
||||
|
||||
static dma_addr_t vi_dma_map_page(struct device *dev, struct page *page,
|
||||
unsigned long offset, size_t size,
|
||||
enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct pci_dev *pci_dev = to_pci_dev(dev);
|
||||
struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
|
||||
void *buffer, *orig_addr;
|
||||
dma_addr_t dma_addr;
|
||||
|
||||
buffer = vi_dma_alloc(dev, size, &dma_addr, 0, attrs);
|
||||
if (!buffer)
|
||||
return DMA_MAPPING_ERROR;
|
||||
|
||||
orig_addr = page_address(page) + offset;
|
||||
vi_dev->map_src_addr[dma_addr >> vi_dev->alloc_shift] = orig_addr;
|
||||
|
||||
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
|
||||
(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
|
||||
memcpy(buffer, orig_addr, size);
|
||||
|
||||
return dma_addr;
|
||||
}
|
||||
|
||||
static void vi_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
|
||||
size_t size, enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct pci_dev *pci_dev = to_pci_dev(dev);
|
||||
struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
|
||||
void *orig_addr = vi_dev->map_src_addr[dma_addr >> vi_dev->alloc_shift];
|
||||
void *buffer = vi_dev->shmem + dma_addr;
|
||||
|
||||
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
|
||||
((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
|
||||
memcpy(orig_addr, buffer, size);
|
||||
|
||||
vi_dma_free(dev, size, buffer, dma_addr, attrs);
|
||||
}
|
||||
|
||||
static void
|
||||
vi_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr,
|
||||
size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
struct pci_dev *pci_dev = to_pci_dev(dev);
|
||||
struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
|
||||
void *orig_addr = vi_dev->map_src_addr[dma_addr >> vi_dev->alloc_shift];
|
||||
void *buffer = vi_dev->shmem + dma_addr;
|
||||
|
||||
memcpy(orig_addr, buffer, size);
|
||||
}
|
||||
|
||||
static void
|
||||
vi_dma_sync_single_for_device(struct device *dev, dma_addr_t dma_addr,
|
||||
size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
struct pci_dev *pci_dev = to_pci_dev(dev);
|
||||
struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
|
||||
void *orig_addr = vi_dev->map_src_addr[dma_addr >> vi_dev->alloc_shift];
|
||||
void *buffer = vi_dev->shmem + dma_addr;
|
||||
|
||||
memcpy(buffer, orig_addr, size);
|
||||
}
|
||||
|
||||
static const struct dma_map_ops virtio_ivshmem_dma_ops = {
|
||||
.alloc = vi_dma_alloc,
|
||||
.free = vi_dma_free,
|
||||
.map_page = vi_dma_map_page,
|
||||
.unmap_page = vi_dma_unmap_page,
|
||||
.sync_single_for_cpu = vi_dma_sync_single_for_cpu,
|
||||
.sync_single_for_device = vi_dma_sync_single_for_device,
|
||||
};
|
||||
|
||||
static int virtio_ivshmem_probe(struct pci_dev *pci_dev,
|
||||
const struct pci_device_id *pci_id)
|
||||
{
|
||||
unsigned int chunks, chunk_size, bitmap_size;
|
||||
struct virtio_ivshmem_device *vi_dev;
|
||||
resource_size_t section_sz;
|
||||
phys_addr_t section_addr;
|
||||
unsigned int cap_pos;
|
||||
u32 *state_table;
|
||||
int vendor_cap;
|
||||
u32 id, dword;
|
||||
int ret;
|
||||
|
||||
vi_dev = devm_kzalloc(&pci_dev->dev, sizeof(*vi_dev), GFP_KERNEL);
|
||||
if (!vi_dev)
|
||||
return -ENOMEM;
|
||||
|
||||
pci_set_drvdata(pci_dev, vi_dev);
|
||||
vi_dev->vdev.dev.parent = &pci_dev->dev;
|
||||
vi_dev->vdev.dev.release = virtio_ivshmem_release_dev;
|
||||
vi_dev->vdev.config = &virtio_ivshmem_config_ops;
|
||||
vi_dev->vdev.id.device = pci_dev->class & IVSHM_PROTO_VIRTIO_DEVID_MASK;
|
||||
vi_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
|
||||
vi_dev->pci_dev = pci_dev;
|
||||
|
||||
spin_lock_init(&vi_dev->virtqueues_lock);
|
||||
INIT_LIST_HEAD(&vi_dev->virtqueues);
|
||||
|
||||
ret = pcim_enable_device(pci_dev);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ret = pcim_iomap_regions(pci_dev, BIT(0), DRV_NAME);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
vi_dev->ivshm_regs = pcim_iomap_table(pci_dev)[0];
|
||||
|
||||
id = readl(&vi_dev->ivshm_regs->id);
|
||||
if (id > 1) {
|
||||
dev_err(&pci_dev->dev, "invalid ID %d\n", id);
|
||||
return -EINVAL;
|
||||
}
|
||||
if (readl(&vi_dev->ivshm_regs->max_peers) != 2) {
|
||||
dev_err(&pci_dev->dev, "number of peers must be 2\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
vi_dev->peer_id = !id;
|
||||
|
||||
vendor_cap = pci_find_capability(pci_dev, PCI_CAP_ID_VNDR);
|
||||
if (vendor_cap < 0) {
|
||||
dev_err(&pci_dev->dev, "missing vendor capability\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (pci_resource_len(pci_dev, 2) > 0) {
|
||||
section_addr = pci_resource_start(pci_dev, 2);
|
||||
} else {
|
||||
cap_pos = vendor_cap + IVSHM_CFG_ADDRESS;
|
||||
section_addr = get_config_qword(pci_dev, cap_pos);
|
||||
}
|
||||
|
||||
cap_pos = vendor_cap + IVSHM_CFG_STATE_TAB_SZ;
|
||||
pci_read_config_dword(pci_dev, cap_pos, &dword);
|
||||
section_sz = dword;
|
||||
|
||||
if (!devm_request_mem_region(&pci_dev->dev, section_addr, section_sz,
|
||||
DRV_NAME))
|
||||
return -EBUSY;
|
||||
|
||||
state_table = devm_memremap(&pci_dev->dev, section_addr, section_sz,
|
||||
MEMREMAP_WB);
|
||||
if (!state_table)
|
||||
return -ENOMEM;
|
||||
|
||||
vi_dev->peer_state = &state_table[vi_dev->peer_id];
|
||||
|
||||
section_addr += section_sz;
|
||||
|
||||
cap_pos = vendor_cap + IVSHM_CFG_RW_SECTION_SZ;
|
||||
section_sz = get_config_qword(pci_dev, cap_pos);
|
||||
if (section_sz < 2 * PAGE_SIZE) {
|
||||
dev_err(&pci_dev->dev, "R/W section too small\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
vi_dev->shmem_sz = section_sz;
|
||||
vi_dev->shmem = devm_memremap(&pci_dev->dev, section_addr, section_sz,
|
||||
MEMREMAP_WB);
|
||||
if (!vi_dev->shmem)
|
||||
return -ENOMEM;
|
||||
|
||||
vi_dev->virtio_header = vi_dev->shmem;
|
||||
if (vi_dev->virtio_header->revision < 1) {
|
||||
dev_err(&pci_dev->dev, "invalid virtio-ivshmem revision\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
spin_lock_init(&vi_dev->alloc_lock);
|
||||
|
||||
chunk_size = vi_dev->shmem_sz / VIRTIO_IVSHMEM_PREFERRED_ALLOC_CHUNKS;
|
||||
if (chunk_size < SMP_CACHE_BYTES)
|
||||
chunk_size = SMP_CACHE_BYTES;
|
||||
if (chunk_size > PAGE_SIZE)
|
||||
chunk_size = PAGE_SIZE;
|
||||
vi_dev->alloc_shift = get_custom_order(chunk_size, 0);
|
||||
|
||||
chunks = vi_dev->shmem_sz >> vi_dev->alloc_shift;
|
||||
bitmap_size = BITS_TO_LONGS(chunks) * sizeof(long);
|
||||
vi_dev->alloc_bitmap = devm_kzalloc(&pci_dev->dev,
|
||||
bitmap_size,
|
||||
GFP_KERNEL);
|
||||
if (!vi_dev->alloc_bitmap)
|
||||
return -ENOMEM;
|
||||
|
||||
/* mark the header chunks used */
|
||||
bitmap_set(vi_dev->alloc_bitmap, 0,
|
||||
1 << get_custom_order(vi_dev->virtio_header->size,
|
||||
vi_dev->alloc_shift));
|
||||
|
||||
vi_dev->map_src_addr = devm_kzalloc(&pci_dev->dev,
|
||||
chunks * sizeof(void *),
|
||||
GFP_KERNEL);
|
||||
if (!vi_dev->map_src_addr)
|
||||
return -ENOMEM;
|
||||
|
||||
set_dma_ops(&pci_dev->dev, &virtio_ivshmem_dma_ops);
|
||||
|
||||
if (*vi_dev->peer_state != VIRTIO_STATE_READY) {
|
||||
dev_err(&pci_dev->dev, "backend not ready\n");
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
pci_set_master(pci_dev);
|
||||
pci_write_config_byte(pci_dev, vendor_cap + IVSHM_CFG_PRIV_CNTL, 0);
|
||||
|
||||
writel(VIRTIO_STATE_READY, &vi_dev->ivshm_regs->state);
|
||||
|
||||
ret = register_virtio_device(&vi_dev->vdev);
|
||||
if (ret) {
|
||||
dev_err(&pci_dev->dev, "failed to register device\n");
|
||||
writel(0, &vi_dev->ivshm_regs->state);
|
||||
put_device(&vi_dev->vdev.dev);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void virtio_ivshmem_remove(struct pci_dev *pci_dev)
|
||||
{
|
||||
struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
|
||||
|
||||
writel(0, &vi_dev->ivshm_regs->state);
|
||||
writel(0, &vi_dev->ivshm_regs->int_control);
|
||||
|
||||
unregister_virtio_device(&vi_dev->vdev);
|
||||
}
|
||||
|
||||
static const struct pci_device_id virtio_ivshmem_id_table[] = {
|
||||
{ PCI_DEVICE(PCI_VENDOR_ID_SIEMENS, PCI_DEVICE_ID_IVSHMEM),
|
||||
(PCI_CLASS_OTHERS << 16) | IVSHM_PROTO_VIRTIO_FRONT, 0xffff00 },
|
||||
{ 0 }
|
||||
};
|
||||
|
||||
MODULE_DEVICE_TABLE(pci, virtio_ivshmem_id_table);
|
||||
|
||||
static struct pci_driver virtio_ivshmem_driver = {
|
||||
.name = DRV_NAME,
|
||||
.id_table = virtio_ivshmem_id_table,
|
||||
.probe = virtio_ivshmem_probe,
|
||||
.remove = virtio_ivshmem_remove,
|
||||
};
|
||||
|
||||
module_pci_driver(virtio_ivshmem_driver);
|
||||
|
||||
MODULE_AUTHOR("Jan Kiszka <jan.kiszka@siemens.com>");
|
||||
MODULE_DESCRIPTION("Driver for ivshmem-based virtio front-end devices");
|
||||
MODULE_LICENSE("GPL v2");
|
||||
Loading…
Reference in New Issue