diff options
Diffstat (limited to 'vfio-pci-block-user-access-to-disabled-device-MMIO.patch')
| -rw-r--r-- | vfio-pci-block-user-access-to-disabled-device-MMIO.patch | 857 |
1 files changed, 857 insertions, 0 deletions
diff --git a/vfio-pci-block-user-access-to-disabled-device-MMIO.patch b/vfio-pci-block-user-access-to-disabled-device-MMIO.patch new file mode 100644 index 000000000..f289b448f --- /dev/null +++ b/vfio-pci-block-user-access-to-disabled-device-MMIO.patch @@ -0,0 +1,857 @@ +From MAILER-DAEMON Wed May 20 15:47:40 2020 +Subject: [PATCH v2 1/3] vfio/type1: Support faulting PFNMAP vmas +From: Alex Williamson <alex.williamson@redhat.com> +To: kvm@vger.kernel.org +Cc: linux-kernel@vger.kernel.org, cohuck@redhat.com, jgg@ziepe.ca +Date: Tue, 05 May 2020 15:54:44 -0600 +Message-ID: <158871568480.15589.17339878308143043906.stgit@gimli.home> +In-Reply-To: <158871401328.15589.17598154478222071285.stgit@gimli.home> +References: <158871401328.15589.17598154478222071285.stgit@gimli.home> +Sender: kvm-owner@vger.kernel.org +List-ID: <kvm.vger.kernel.org> +X-Mailing-List: kvm@vger.kernel.org +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: 7bit + +With conversion to follow_pfn(), DMA mapping a PFNMAP range depends on +the range being faulted into the vma. Add support to manually provide +that, in the same way as done on KVM with hva_to_pfn_remapped(). + +Signed-off-by: Alex Williamson <alex.williamson@redhat.com> +Reviewed-by: Peter Xu <peterx@redhat.com> +--- + drivers/vfio/vfio_iommu_type1.c | 36 +++++++++++++++++++++++++++++++++--- + 1 file changed, 33 insertions(+), 3 deletions(-) + +diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c +index cc1d64765ce7..4a4cb7cd86b2 100644 +--- a/drivers/vfio/vfio_iommu_type1.c ++++ b/drivers/vfio/vfio_iommu_type1.c +@@ -317,6 +317,32 @@ static int put_pfn(unsigned long pfn, int prot) + return 0; + } + ++static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, ++ unsigned long vaddr, unsigned long *pfn, ++ bool write_fault) ++{ ++ int ret; ++ ++ ret = follow_pfn(vma, vaddr, pfn); ++ if (ret) { ++ bool unlocked = false; ++ ++ ret = fixup_user_fault(NULL, mm, vaddr, ++ FAULT_FLAG_REMOTE | ++ (write_fault ? FAULT_FLAG_WRITE : 0), ++ &unlocked); ++ if (unlocked) ++ return -EAGAIN; ++ ++ if (ret) ++ return ret; ++ ++ ret = follow_pfn(vma, vaddr, pfn); ++ } ++ ++ return ret; ++} ++ + static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, + int prot, unsigned long *pfn) + { +@@ -339,12 +365,16 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, + + vaddr = untagged_addr(vaddr); + ++retry: + vma = find_vma_intersection(mm, vaddr, vaddr + 1); + + if (vma && vma->vm_flags & VM_PFNMAP) { +- if (!follow_pfn(vma, vaddr, pfn) && +- is_invalid_reserved_pfn(*pfn)) +- ret = 0; ++ ret = follow_fault_pfn(vma, mm, vaddr, pfn, prot & IOMMU_WRITE); ++ if (ret == -EAGAIN) ++ goto retry; ++ ++ if (!ret && !is_invalid_reserved_pfn(*pfn)) ++ ret = -EFAULT; + } + done: + up_read(&mm->mmap_sem); + + +From MAILER-DAEMON Wed May 20 15:47:40 2020 +Subject: [PATCH v2 2/3] vfio-pci: Fault mmaps to enable vma tracking +From: Alex Williamson <alex.williamson@redhat.com> +To: kvm@vger.kernel.org +Cc: linux-kernel@vger.kernel.org, cohuck@redhat.com, jgg@ziepe.ca +Date: Tue, 05 May 2020 15:54:53 -0600 +Message-ID: <158871569380.15589.16950418949340311053.stgit@gimli.home> +In-Reply-To: <158871401328.15589.17598154478222071285.stgit@gimli.home> +References: <158871401328.15589.17598154478222071285.stgit@gimli.home> +Sender: kvm-owner@vger.kernel.org +List-ID: <kvm.vger.kernel.org> +X-Mailing-List: kvm@vger.kernel.org +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: 7bit + +Rather than calling remap_pfn_range() when a region is mmap'd, setup +a vm_ops handler to support dynamic faulting of the range on access. +This allows us to manage a list of vmas actively mapping the area that +we can later use to invalidate those mappings. The open callback +invalidates the vma range so that all tracking is inserted in the +fault handler and removed in the close handler. + +Signed-off-by: Alex Williamson <alex.williamson@redhat.com> +Reviewed-by: Peter Xu <peterx@redhat.com> +--- + drivers/vfio/pci/vfio_pci.c | 76 ++++++++++++++++++++++++++++++++++- + drivers/vfio/pci/vfio_pci_private.h | 7 +++ + 2 files changed, 81 insertions(+), 2 deletions(-) + +diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c +index 6c6b37b5c04e..66a545a01f8f 100644 +--- a/drivers/vfio/pci/vfio_pci.c ++++ b/drivers/vfio/pci/vfio_pci.c +@@ -1299,6 +1299,70 @@ static ssize_t vfio_pci_write(void *device_data, const char __user *buf, + return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true); + } + ++static int vfio_pci_add_vma(struct vfio_pci_device *vdev, ++ struct vm_area_struct *vma) ++{ ++ struct vfio_pci_mmap_vma *mmap_vma; ++ ++ mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL); ++ if (!mmap_vma) ++ return -ENOMEM; ++ ++ mmap_vma->vma = vma; ++ ++ mutex_lock(&vdev->vma_lock); ++ list_add(&mmap_vma->vma_next, &vdev->vma_list); ++ mutex_unlock(&vdev->vma_lock); ++ ++ return 0; ++} ++ ++/* ++ * Zap mmaps on open so that we can fault them in on access and therefore ++ * our vma_list only tracks mappings accessed since last zap. ++ */ ++static void vfio_pci_mmap_open(struct vm_area_struct *vma) ++{ ++ zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); ++} ++ ++static void vfio_pci_mmap_close(struct vm_area_struct *vma) ++{ ++ struct vfio_pci_device *vdev = vma->vm_private_data; ++ struct vfio_pci_mmap_vma *mmap_vma; ++ ++ mutex_lock(&vdev->vma_lock); ++ list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) { ++ if (mmap_vma->vma == vma) { ++ list_del(&mmap_vma->vma_next); ++ kfree(mmap_vma); ++ break; ++ } ++ } ++ mutex_unlock(&vdev->vma_lock); ++} ++ ++static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf) ++{ ++ struct vm_area_struct *vma = vmf->vma; ++ struct vfio_pci_device *vdev = vma->vm_private_data; ++ ++ if (vfio_pci_add_vma(vdev, vma)) ++ return VM_FAULT_OOM; ++ ++ if (remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, ++ vma->vm_end - vma->vm_start, vma->vm_page_prot)) ++ return VM_FAULT_SIGBUS; ++ ++ return VM_FAULT_NOPAGE; ++} ++ ++static const struct vm_operations_struct vfio_pci_mmap_ops = { ++ .open = vfio_pci_mmap_open, ++ .close = vfio_pci_mmap_close, ++ .fault = vfio_pci_mmap_fault, ++}; ++ + static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) + { + struct vfio_pci_device *vdev = device_data; +@@ -1357,8 +1421,14 @@ static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff; + +- return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, +- req_len, vma->vm_page_prot); ++ /* ++ * See remap_pfn_range(), called from vfio_pci_fault() but we can't ++ * change vm_flags within the fault handler. Set them now. ++ */ ++ vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; ++ vma->vm_ops = &vfio_pci_mmap_ops; ++ ++ return 0; + } + + static void vfio_pci_request(void *device_data, unsigned int count) +@@ -1608,6 +1678,8 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) + spin_lock_init(&vdev->irqlock); + mutex_init(&vdev->ioeventfds_lock); + INIT_LIST_HEAD(&vdev->ioeventfds_list); ++ mutex_init(&vdev->vma_lock); ++ INIT_LIST_HEAD(&vdev->vma_list); + + ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); + if (ret) { +diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h +index 36ec69081ecd..9b25f9f6ce1d 100644 +--- a/drivers/vfio/pci/vfio_pci_private.h ++++ b/drivers/vfio/pci/vfio_pci_private.h +@@ -92,6 +92,11 @@ struct vfio_pci_vf_token { + struct mutex lock; + }; + ++struct vfio_pci_mmap_vma { ++ struct vm_area_struct *vma; ++ struct list_head vma_next; ++}; ++ + struct vfio_pci_device { + struct pci_dev *pdev; + void __iomem *barmap[PCI_STD_NUM_BARS]; +@@ -132,6 +137,8 @@ struct vfio_pci_device { + struct list_head dummy_resources_list; + struct mutex ioeventfds_lock; + struct list_head ioeventfds_list; ++ struct mutex vma_lock; ++ struct list_head vma_list; + }; + + #define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) + + +From MAILER-DAEMON Wed May 20 15:47:40 2020 +Subject: [PATCH v2 3/3] vfio-pci: Invalidate mmaps and block MMIO access on disabled memory +From: Alex Williamson <alex.williamson@redhat.com> +To: kvm@vger.kernel.org +Cc: linux-kernel@vger.kernel.org, cohuck@redhat.com, jgg@ziepe.ca +Date: Tue, 05 May 2020 15:55:02 -0600 +Message-ID: <158871570274.15589.10563806532874116326.stgit@gimli.home> +In-Reply-To: <158871401328.15589.17598154478222071285.stgit@gimli.home> +References: <158871401328.15589.17598154478222071285.stgit@gimli.home> +Sender: kvm-owner@vger.kernel.org +List-ID: <kvm.vger.kernel.org> +X-Mailing-List: kvm@vger.kernel.org +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: 7bit + +Accessing the disabled memory space of a PCI device would typically +result in a master abort response on conventional PCI, or an +unsupported request on PCI express. The user would generally see +these as a -1 response for the read return data and the write would be +silently discarded, possibly with an uncorrected, non-fatal AER error +triggered on the host. Some systems however take it upon themselves +to bring down the entire system when they see something that might +indicate a loss of data, such as this discarded write to a disabled +memory space. + +To avoid this, we want to try to block the user from accessing memory +spaces while they're disabled. We start with a semaphore around the +memory enable bit, where writers modify the memory enable state and +must be serialized, while readers make use of the memory region and +can access in parallel. Writers include both direct manipulation via +the command register, as well as any reset path where the internal +mechanics of the reset may both explicitly and implicitly disable +memory access, and manipulation of the MSI-X configuration, where the +MSI-X vector table resides in MMIO space of the device. Readers +include the read and write file ops to access the vfio device fd +offsets as well as memory mapped access. In the latter case, we make +use of our new vma list support to zap, or invalidate, those memory +mappings in order to force them to be faulted back in on access. + +Our semaphore usage will stall user access to MMIO spaces across +internal operations like reset, but the user might experience new +behavior when trying to access the MMIO space while disabled via the +PCI command register. Access via read or write while disabled will +return -EIO and access via memory maps will result in a SIGBUS. This +is expected to be compatible with known use cases and potentially +provides better error handling capabilities than present in the +hardware, while avoiding the more readily accessible and severe +platform error responses that might otherwise occur. + +Signed-off-by: Alex Williamson <alex.williamson@redhat.com> +--- + drivers/vfio/pci/vfio_pci.c | 263 +++++++++++++++++++++++++++++++---- + drivers/vfio/pci/vfio_pci_config.c | 36 ++++- + drivers/vfio/pci/vfio_pci_intrs.c | 18 ++ + drivers/vfio/pci/vfio_pci_private.h | 5 + + drivers/vfio/pci/vfio_pci_rdwr.c | 12 ++ + 5 files changed, 300 insertions(+), 34 deletions(-) + +diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c +index 66a545a01f8f..49ae9faa6099 100644 +--- a/drivers/vfio/pci/vfio_pci.c ++++ b/drivers/vfio/pci/vfio_pci.c +@@ -26,6 +26,7 @@ + #include <linux/vfio.h> + #include <linux/vgaarb.h> + #include <linux/nospec.h> ++#include <linux/sched/mm.h> + + #include "vfio_pci_private.h" + +@@ -184,6 +185,7 @@ static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev) + + static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev); + static void vfio_pci_disable(struct vfio_pci_device *vdev); ++static int vfio_pci_try_zap_and_vma_lock_cb(struct pci_dev *pdev, void *data); + + /* + * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND +@@ -736,6 +738,12 @@ int vfio_pci_register_dev_region(struct vfio_pci_device *vdev, + return 0; + } + ++struct vfio_devices { ++ struct vfio_device **devices; ++ int cur_index; ++ int max_index; ++}; ++ + static long vfio_pci_ioctl(void *device_data, + unsigned int cmd, unsigned long arg) + { +@@ -984,8 +992,16 @@ static long vfio_pci_ioctl(void *device_data, + return ret; + + } else if (cmd == VFIO_DEVICE_RESET) { +- return vdev->reset_works ? +- pci_try_reset_function(vdev->pdev) : -EINVAL; ++ int ret; ++ ++ if (!vdev->reset_works) ++ return -EINVAL; ++ ++ vfio_pci_zap_and_down_write_memory_lock(vdev); ++ ret = pci_try_reset_function(vdev->pdev); ++ up_write(&vdev->memory_lock); ++ ++ return ret; + + } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) { + struct vfio_pci_hot_reset_info hdr; +@@ -1065,8 +1081,9 @@ static long vfio_pci_ioctl(void *device_data, + int32_t *group_fds; + struct vfio_pci_group_entry *groups; + struct vfio_pci_group_info info; ++ struct vfio_devices devs = { .cur_index = 0 }; + bool slot = false; +- int i, count = 0, ret = 0; ++ int i, group_idx, mem_idx = 0, count = 0, ret = 0; + + minsz = offsetofend(struct vfio_pci_hot_reset, count); + +@@ -1118,9 +1135,9 @@ static long vfio_pci_ioctl(void *device_data, + * user interface and store the group and iommu ID. This + * ensures the group is held across the reset. + */ +- for (i = 0; i < hdr.count; i++) { ++ for (group_idx = 0; group_idx < hdr.count; group_idx++) { + struct vfio_group *group; +- struct fd f = fdget(group_fds[i]); ++ struct fd f = fdget(group_fds[group_idx]); + if (!f.file) { + ret = -EBADF; + break; +@@ -1133,8 +1150,9 @@ static long vfio_pci_ioctl(void *device_data, + break; + } + +- groups[i].group = group; +- groups[i].id = vfio_external_user_iommu_id(group); ++ groups[group_idx].group = group; ++ groups[group_idx].id = ++ vfio_external_user_iommu_id(group); + } + + kfree(group_fds); +@@ -1153,13 +1171,63 @@ static long vfio_pci_ioctl(void *device_data, + ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, + vfio_pci_validate_devs, + &info, slot); +- if (!ret) +- /* User has access, do the reset */ +- ret = pci_reset_bus(vdev->pdev); ++ if (ret) ++ goto hot_reset_release; ++ ++ devs.max_index = count; ++ devs.devices = kcalloc(count, sizeof(struct vfio_device *), ++ GFP_KERNEL); ++ if (!devs.devices) { ++ ret = -ENOMEM; ++ goto hot_reset_release; ++ } ++ ++ /* ++ * We need to get memory_lock for each device, but devices ++ * can share mmap_sem, therefore we need to zap and hold ++ * the vma_lock for each device, and only then get each ++ * memory_lock. ++ */ ++ ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, ++ vfio_pci_try_zap_and_vma_lock_cb, ++ &devs, slot); ++ if (ret) ++ goto hot_reset_release; ++ ++ for (; mem_idx < devs.cur_index; mem_idx++) { ++ struct vfio_pci_device *tmp; ++ ++ tmp = vfio_device_data(devs.devices[mem_idx]); ++ ++ ret = down_write_trylock(&tmp->memory_lock); ++ if (!ret) { ++ ret = -EBUSY; ++ goto hot_reset_release; ++ } ++ mutex_unlock(&tmp->vma_lock); ++ } ++ ++ /* User has access, do the reset */ ++ ret = pci_reset_bus(vdev->pdev); + + hot_reset_release: +- for (i--; i >= 0; i--) +- vfio_group_put_external_user(groups[i].group); ++ for (i = 0; i < devs.cur_index; i++) { ++ struct vfio_device *device; ++ struct vfio_pci_device *tmp; ++ ++ device = devs.devices[i]; ++ tmp = vfio_device_data(device); ++ ++ if (i < mem_idx) ++ up_write(&tmp->memory_lock); ++ else ++ mutex_unlock(&tmp->vma_lock); ++ vfio_device_put(device); ++ } ++ kfree(devs.devices); ++ ++ for (group_idx--; group_idx >= 0; group_idx--) ++ vfio_group_put_external_user(groups[group_idx].group); + + kfree(groups); + return ret; +@@ -1299,8 +1367,107 @@ static ssize_t vfio_pci_write(void *device_data, const char __user *buf, + return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true); + } + +-static int vfio_pci_add_vma(struct vfio_pci_device *vdev, +- struct vm_area_struct *vma) ++/* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */ ++static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try) ++{ ++ struct vfio_pci_mmap_vma *mmap_vma, *tmp; ++ ++ /* ++ * Lock ordering: ++ * vma_lock is nested under mmap_sem for vm_ops callback paths. ++ * The memory_lock semaphore is used by both code paths calling ++ * into this function to zap vmas and the vm_ops.fault callback ++ * to protect the memory enable state of the device. ++ * ++ * When zapping vmas we need to maintain the mmap_sem => vma_lock ++ * ordering, which requires using vma_lock to walk vma_list to ++ * acquire an mm, then dropping vma_lock to get the mmap_sem and ++ * reacquiring vma_lock. This logic is derived from similar ++ * requirements in uverbs_user_mmap_disassociate(). ++ * ++ * mmap_sem must always be the top-level lock when it is taken. ++ * Therefore we can only hold the memory_lock write lock when ++ * vma_list is empty, as we'd need to take mmap_sem to clear ++ * entries. vma_list can only be guaranteed empty when holding ++ * vma_lock, thus memory_lock is nested under vma_lock. ++ * ++ * This enables the vm_ops.fault callback to acquire vma_lock, ++ * followed by memory_lock read lock, while already holding ++ * mmap_sem without risk of deadlock. ++ */ ++ while (1) { ++ struct mm_struct *mm = NULL; ++ ++ if (try) { ++ if (!mutex_trylock(&vdev->vma_lock)) ++ return 0; ++ } else { ++ mutex_lock(&vdev->vma_lock); ++ } ++ while (!list_empty(&vdev->vma_list)) { ++ mmap_vma = list_first_entry(&vdev->vma_list, ++ struct vfio_pci_mmap_vma, ++ vma_next); ++ mm = mmap_vma->vma->vm_mm; ++ if (mmget_not_zero(mm)) ++ break; ++ ++ list_del(&mmap_vma->vma_next); ++ kfree(mmap_vma); ++ mm = NULL; ++ } ++ if (!mm) ++ return 1; ++ mutex_unlock(&vdev->vma_lock); ++ ++ if (try) { ++ if (!down_read_trylock(&mm->mmap_sem)) { ++ mmput(mm); ++ return 0; ++ } ++ } else { ++ down_read(&mm->mmap_sem); ++ } ++ if (mmget_still_valid(mm)) { ++ if (try) { ++ if (!mutex_trylock(&vdev->vma_lock)) { ++ up_read(&mm->mmap_sem); ++ mmput(mm); ++ return 0; ++ } ++ } else { ++ mutex_lock(&vdev->vma_lock); ++ } ++ list_for_each_entry_safe(mmap_vma, tmp, ++ &vdev->vma_list, vma_next) { ++ struct vm_area_struct *vma = mmap_vma->vma; ++ ++ if (vma->vm_mm != mm) ++ continue; ++ ++ list_del(&mmap_vma->vma_next); ++ kfree(mmap_vma); ++ ++ zap_vma_ptes(vma, vma->vm_start, ++ vma->vm_end - vma->vm_start); ++ } ++ mutex_unlock(&vdev->vma_lock); ++ } ++ up_read(&mm->mmap_sem); ++ mmput(mm); ++ } ++} ++ ++void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device *vdev) ++{ ++ vfio_pci_zap_and_vma_lock(vdev, false); ++ down_write(&vdev->memory_lock); ++ mutex_unlock(&vdev->vma_lock); ++} ++ ++/* Caller holds vma_lock */ ++static int __vfio_pci_add_vma(struct vfio_pci_device *vdev, ++ struct vm_area_struct *vma) + { + struct vfio_pci_mmap_vma *mmap_vma; + +@@ -1309,10 +1476,7 @@ static int vfio_pci_add_vma(struct vfio_pci_device *vdev, + return -ENOMEM; + + mmap_vma->vma = vma; +- +- mutex_lock(&vdev->vma_lock); + list_add(&mmap_vma->vma_next, &vdev->vma_list); +- mutex_unlock(&vdev->vma_lock); + + return 0; + } +@@ -1346,15 +1510,32 @@ static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf) + { + struct vm_area_struct *vma = vmf->vma; + struct vfio_pci_device *vdev = vma->vm_private_data; ++ vm_fault_t ret = VM_FAULT_NOPAGE; + +- if (vfio_pci_add_vma(vdev, vma)) +- return VM_FAULT_OOM; ++ mutex_lock(&vdev->vma_lock); ++ down_read(&vdev->memory_lock); ++ ++ if (!__vfio_pci_memory_enabled(vdev)) { ++ ret = VM_FAULT_SIGBUS; ++ mutex_unlock(&vdev->vma_lock); ++ goto up_out; ++ } ++ ++ if (__vfio_pci_add_vma(vdev, vma)) { ++ ret = VM_FAULT_OOM; ++ mutex_unlock(&vdev->vma_lock); ++ goto up_out; ++ } ++ ++ mutex_unlock(&vdev->vma_lock); + + if (remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + vma->vm_end - vma->vm_start, vma->vm_page_prot)) +- return VM_FAULT_SIGBUS; ++ ret = VM_FAULT_SIGBUS; + +- return VM_FAULT_NOPAGE; ++up_out: ++ up_read(&vdev->memory_lock); ++ return ret; + } + + static const struct vm_operations_struct vfio_pci_mmap_ops = { +@@ -1680,6 +1861,7 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) + INIT_LIST_HEAD(&vdev->ioeventfds_list); + mutex_init(&vdev->vma_lock); + INIT_LIST_HEAD(&vdev->vma_list); ++ init_rwsem(&vdev->memory_lock); + + ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); + if (ret) { +@@ -1933,12 +2115,6 @@ static void vfio_pci_reflck_put(struct vfio_pci_reflck *reflck) + kref_put_mutex(&reflck->kref, vfio_pci_reflck_release, &reflck_lock); + } + +-struct vfio_devices { +- struct vfio_device **devices; +- int cur_index; +- int max_index; +-}; +- + static int vfio_pci_get_unused_devs(struct pci_dev *pdev, void *data) + { + struct vfio_devices *devs = data; +@@ -1969,6 +2145,39 @@ static int vfio_pci_get_unused_devs(struct pci_dev *pdev, void *data) + return 0; + } + ++static int vfio_pci_try_zap_and_vma_lock_cb(struct pci_dev *pdev, void *data) ++{ ++ struct vfio_devices *devs = data; ++ struct vfio_device *device; ++ struct vfio_pci_device *vdev; ++ ++ if (devs->cur_index == devs->max_index) ++ return -ENOSPC; ++ ++ device = vfio_device_get_from_dev(&pdev->dev); ++ if (!device) ++ return -EINVAL; ++ ++ if (pci_dev_driver(pdev) != &vfio_pci_driver) { ++ vfio_device_put(device); ++ return -EBUSY; ++ } ++ ++ vdev = vfio_device_data(device); ++ ++ /* ++ * Locking multiple devices is prone to deadlock, runaway and ++ * unwind if we hit contention. ++ */ ++ if (!vfio_pci_zap_and_vma_lock(vdev, true)) { ++ vfio_device_put(device); ++ return -EBUSY; ++ } ++ ++ devs->devices[devs->cur_index++] = device; ++ return 0; ++} ++ + /* + * If a bus or slot reset is available for the provided device and: + * - All of the devices affected by that bus or slot reset are unused +diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c +index 90c0b80f8acf..3dcddbd572e6 100644 +--- a/drivers/vfio/pci/vfio_pci_config.c ++++ b/drivers/vfio/pci/vfio_pci_config.c +@@ -395,6 +395,14 @@ static inline void p_setd(struct perm_bits *p, int off, u32 virt, u32 write) + *(__le32 *)(&p->write[off]) = cpu_to_le32(write); + } + ++/* Caller should hold memory_lock semaphore */ ++bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev) ++{ ++ u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]); ++ ++ return cmd & PCI_COMMAND_MEMORY; ++} ++ + /* + * Restore the *real* BARs after we detect a FLR or backdoor reset. + * (backdoor = some device specific technique that we didn't catch) +@@ -556,13 +564,18 @@ static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos, + + new_cmd = le32_to_cpu(val); + ++ phys_io = !!(phys_cmd & PCI_COMMAND_IO); ++ virt_io = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_IO); ++ new_io = !!(new_cmd & PCI_COMMAND_IO); ++ + phys_mem = !!(phys_cmd & PCI_COMMAND_MEMORY); + virt_mem = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_MEMORY); + new_mem = !!(new_cmd & PCI_COMMAND_MEMORY); + +- phys_io = !!(phys_cmd & PCI_COMMAND_IO); +- virt_io = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_IO); +- new_io = !!(new_cmd & PCI_COMMAND_IO); ++ if (!new_mem) ++ vfio_pci_zap_and_down_write_memory_lock(vdev); ++ else ++ down_write(&vdev->memory_lock); + + /* + * If the user is writing mem/io enable (new_mem/io) and we +@@ -579,8 +592,11 @@ static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos, + } + + count = vfio_default_config_write(vdev, pos, count, perm, offset, val); +- if (count < 0) ++ if (count < 0) { ++ if (offset == PCI_COMMAND) ++ up_write(&vdev->memory_lock); + return count; ++ } + + /* + * Save current memory/io enable bits in vconfig to allow for +@@ -591,6 +607,8 @@ static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos, + + *virt_cmd &= cpu_to_le16(~mask); + *virt_cmd |= cpu_to_le16(new_cmd & mask); ++ ++ up_write(&vdev->memory_lock); + } + + /* Emulate INTx disable */ +@@ -828,8 +846,11 @@ static int vfio_exp_config_write(struct vfio_pci_device *vdev, int pos, + pos - offset + PCI_EXP_DEVCAP, + &cap); + +- if (!ret && (cap & PCI_EXP_DEVCAP_FLR)) ++ if (!ret && (cap & PCI_EXP_DEVCAP_FLR)) { ++ vfio_pci_zap_and_down_write_memory_lock(vdev); + pci_try_reset_function(vdev->pdev); ++ up_write(&vdev->memory_lock); ++ } + } + + /* +@@ -907,8 +928,11 @@ static int vfio_af_config_write(struct vfio_pci_device *vdev, int pos, + pos - offset + PCI_AF_CAP, + &cap); + +- if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP)) ++ if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP)) { ++ vfio_pci_zap_and_down_write_memory_lock(vdev); + pci_try_reset_function(vdev->pdev); ++ up_write(&vdev->memory_lock); ++ } + } + + return count; +diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c +index 2056f3f85f59..54102a7eb9d3 100644 +--- a/drivers/vfio/pci/vfio_pci_intrs.c ++++ b/drivers/vfio/pci/vfio_pci_intrs.c +@@ -626,6 +626,8 @@ int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev, uint32_t flags, + int (*func)(struct vfio_pci_device *vdev, unsigned index, + unsigned start, unsigned count, uint32_t flags, + void *data) = NULL; ++ int ret; ++ u16 cmd; + + switch (index) { + case VFIO_PCI_INTX_IRQ_INDEX: +@@ -673,5 +675,19 @@ int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev, uint32_t flags, + if (!func) + return -ENOTTY; + +- return func(vdev, index, start, count, flags, data); ++ if (index == VFIO_PCI_MSIX_IRQ_INDEX) { ++ down_write(&vdev->memory_lock); ++ pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd); ++ pci_write_config_word(vdev->pdev, PCI_COMMAND, ++ cmd | PCI_COMMAND_MEMORY); ++ } ++ ++ ret = func(vdev, index, start, count, flags, data); ++ ++ if (index == VFIO_PCI_MSIX_IRQ_INDEX) { ++ pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd); ++ up_write(&vdev->memory_lock); ++ } ++ ++ return ret; + } +diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h +index 9b25f9f6ce1d..c4f25f1e80d7 100644 +--- a/drivers/vfio/pci/vfio_pci_private.h ++++ b/drivers/vfio/pci/vfio_pci_private.h +@@ -139,6 +139,7 @@ struct vfio_pci_device { + struct list_head ioeventfds_list; + struct mutex vma_lock; + struct list_head vma_list; ++ struct rw_semaphore memory_lock; + }; + + #define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) +@@ -181,6 +182,10 @@ extern int vfio_pci_register_dev_region(struct vfio_pci_device *vdev, + extern int vfio_pci_set_power_state(struct vfio_pci_device *vdev, + pci_power_t state); + ++extern bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev); ++extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device ++ *vdev); ++ + #ifdef CONFIG_VFIO_PCI_IGD + extern int vfio_pci_igd_init(struct vfio_pci_device *vdev); + #else +diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c +index a87992892a9f..f58c45308682 100644 +--- a/drivers/vfio/pci/vfio_pci_rdwr.c ++++ b/drivers/vfio/pci/vfio_pci_rdwr.c +@@ -162,6 +162,7 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf, + size_t x_start = 0, x_end = 0; + resource_size_t end; + void __iomem *io; ++ struct resource *res = &vdev->pdev->resource[bar]; + ssize_t done; + + if (pci_resource_start(pdev, bar)) +@@ -200,8 +201,19 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf, + x_end = vdev->msix_offset + vdev->msix_size; + } + ++ if (res->flags & IORESOURCE_MEM) { ++ down_read(&vdev->memory_lock); ++ if (!__vfio_pci_memory_enabled(vdev)) { ++ up_read(&vdev->memory_lock); ++ return -EIO; ++ } ++ } ++ + done = do_io_rw(io, buf, pos, count, x_start, x_end, iswrite); + ++ if (res->flags & IORESOURCE_MEM) ++ up_read(&vdev->memory_lock); ++ + if (done >= 0) + *ppos += done; + + + |