From de9c7602ca25f52bbfeb52e7d85765fe70b92dce Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Mon, 10 Jun 2013 16:40:56 -0600 Subject: [PATCH 1/3] vfio: Don't overreact to DEL_DEVICE BUS_NOTIFY_DEL_DEVICE triggers IOMMU drivers to remove devices from their iommu group, but there's really nothing we can do about it at this point. If the device is in use, then the vfio sub-driver will block the device_del from completing until it's released. If the device is not in use or not owned by a vfio sub-driver, then we really don't care that it's being removed. The current code can be triggered just by unloading an sr-iov driver (ex. igb) while the VFs are attached to vfio-pci because it makes an incorrect assumption about the ordering of driver remove callbacks vs the DEL_DEVICE notification. Signed-off-by: Alex Williamson --- drivers/vfio/vfio.c | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index c488da5db7c7..6d18c3cafdd4 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -494,27 +494,6 @@ static int vfio_group_nb_add_dev(struct vfio_group *group, struct device *dev) return 0; } -static int vfio_group_nb_del_dev(struct vfio_group *group, struct device *dev) -{ - struct vfio_device *device; - - /* - * Expect to fall out here. If a device was in use, it would - * have been bound to a vfio sub-driver, which would have blocked - * in .remove at vfio_del_group_dev. Sanity check that we no - * longer track the device, so it's safe to remove. - */ - device = vfio_group_get_device(group, dev); - if (likely(!device)) - return 0; - - WARN("Device %s removed from live group %d!\n", dev_name(dev), - iommu_group_id(group->iommu_group)); - - vfio_device_put(device); - return 0; -} - static int vfio_group_nb_verify(struct vfio_group *group, struct device *dev) { /* We don't care what happens when the group isn't in use */ @@ -545,7 +524,13 @@ static int vfio_iommu_group_notifier(struct notifier_block *nb, vfio_group_nb_add_dev(group, dev); break; case IOMMU_GROUP_NOTIFY_DEL_DEVICE: - vfio_group_nb_del_dev(group, dev); + /* + * Nothing to do here. If the device is in use, then the + * vfio sub-driver should block the remove callback until + * it is unused. If the device is unused or attached to a + * stub driver, then it should be released and we don't + * care that it will be going away. + */ break; case IOMMU_GROUP_NOTIFY_BIND_DRIVER: pr_debug("%s: Device %s, group %d binding to driver\n", From c64019302bbb0b445484d870e674ab34a19a18a1 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Mon, 10 Jun 2013 16:40:56 -0600 Subject: [PATCH 2/3] vfio: Ignore sprurious notifies Remove debugging WARN_ON if we get a spurious notify for a group that no longer exists. No reports of anyone hitting this, but it would likely be a race and not a bug if they did. Signed-off-by: Alex Williamson --- drivers/vfio/vfio.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 6d18c3cafdd4..842f4507883e 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -510,13 +510,11 @@ static int vfio_iommu_group_notifier(struct notifier_block *nb, struct device *dev = data; /* - * Need to go through a group_lock lookup to get a reference or - * we risk racing a group being removed. Leave a WARN_ON for - * debuging, but if the group no longer exists, a spurious notify - * is harmless. + * Need to go through a group_lock lookup to get a reference or we + * risk racing a group being removed. Ignore spurious notifies. */ group = vfio_group_try_get(group); - if (WARN_ON(!group)) + if (!group) return NOTIFY_OK; switch (action) { From d24cdbfd28b7e0ffecb1e281d73e73c03a57f734 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Mon, 10 Jun 2013 16:40:57 -0600 Subject: [PATCH 3/3] vfio-pci: Avoid deadlock on remove If an attempt is made to unbind a device from vfio-pci while that device is in use, the request is blocked until the device becomes unused. Unfortunately, that unbind path still grabs the device_lock, which certain things like __pci_reset_function() also want to take. This means we need to try to acquire the locks ourselves and use the pre-locked version, __pci_reset_function_locked(). Signed-off-by: Alex Williamson --- drivers/vfio/pci/vfio_pci.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index c5179e269df6..cef6002acbd4 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -137,8 +137,27 @@ static void vfio_pci_disable(struct vfio_pci_device *vdev) */ pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); - if (vdev->reset_works) - __pci_reset_function(pdev); + /* + * Careful, device_lock may already be held. This is the case if + * a driver unbind is blocked. Try to get the locks ourselves to + * prevent a deadlock. + */ + if (vdev->reset_works) { + bool reset_done = false; + + if (pci_cfg_access_trylock(pdev)) { + if (device_trylock(&pdev->dev)) { + __pci_reset_function_locked(pdev); + reset_done = true; + device_unlock(&pdev->dev); + } + pci_cfg_access_unlock(pdev); + } + + if (!reset_done) + pr_warn("%s: Unable to acquire locks for reset of %s\n", + __func__, dev_name(&pdev->dev)); + } pci_restore_state(pdev); }