Re: [PATCH v3 2/2] PCI/IOV: Fix race between SR-IOV enable/disable and hotplug

[Benjamin Block]

On Tue, Mar 17, 2026 at 10:46:56AM +0100, Benjamin Block wrote:
> On Tue, Mar 17, 2026 at 10:01:49AM +0100, Benjamin Block wrote:
> > One quick idea: can we somehow unbind the device from any device driver
> > in remove_store() before calling
> > pci_stop_and_remove_bus_device_locked()?  That way we would not have any
> > SR-IOV functions attached anymore at the point where we remove the PF,
> > since the DD are expected to clean them up.
> 
> Very quick-and-dirty:
> 
> diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
> index 16eaaf749ba9..490b6c1f07d3 100644
> --- a/drivers/pci/pci-sysfs.c
> +++ b/drivers/pci/pci-sysfs.c
> @@ -33,6 +33,8 @@
>  #include <linux/unaligned.h>
>  #include "pci.h"
>  
> +extern void device_driver_detach(struct device *dev);
> +
>  #ifndef ARCH_PCI_DEV_GROUPS
>  #define ARCH_PCI_DEV_GROUPS
>  #endif
> @@ -518,8 +520,10 @@ static ssize_t remove_store(struct device *dev, struct device_attribute *attr,
>  	if (kstrtoul(buf, 0, &val) < 0)
>  		return -EINVAL;
>  
> -	if (val && device_remove_file_self(dev, attr))
> +	if (val && device_remove_file_self(dev, attr)) {
> +		device_driver_detach(dev);

Hmm, thinking on it, this leaves open the possibility, that something re-binds
the DD in between these two calls. Although unlikely, it might happen.

>  		pci_stop_and_remove_bus_device_locked(to_pci_dev(dev));
> +	}
>  	return count;
>  }
>  static DEVICE_ATTR_IGNORE_LOCKDEP(remove, 0220, NULL,

I guess, I dig the hole a bit deeper.

diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index 16eaaf749ba9..d2877b805d6e 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -33,6 +33,8 @@
 #include <linux/unaligned.h>
 #include "pci.h"
 
+extern void device_driver_detach(struct device *dev);
+
 #ifndef ARCH_PCI_DEV_GROUPS
 #define ARCH_PCI_DEV_GROUPS
 #endif
@@ -518,8 +520,13 @@ static ssize_t remove_store(struct device *dev, struct device_attribute *attr,
 	if (kstrtoul(buf, 0, &val) < 0)
 		return -EINVAL;
 
-	if (val && device_remove_file_self(dev, attr))
+	if (val && device_remove_file_self(dev, attr)) {
+		device_lock(dev);
+		kill_device(dev);
+		device_unlock(dev);
+		device_driver_detach(dev);
 		pci_stop_and_remove_bus_device_locked(to_pci_dev(dev));
+	}
 	return count;
 }
 static DEVICE_ATTR_IGNORE_LOCKDEP(remove, 0220, NULL,

Marking the device dead prevents re-binding after the unbind.

Same sniff tests as before:

  root@b315lp04:~# cd /sys/bus/pci/
  root@b315lp04:/sys/bus/pci# lspci
  0103:00:00.0 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
  1100:00:00.0 Processing accelerators: IBM Spyre Accelerator (rev 02)
  1100:00:00.1 Processing accelerators: IBM Spyre Accelerator Virtual Function (rev 02)
  1180:00:00.0 Processing accelerators: IBM Spyre Accelerator (rev 02)
  1180:00:00.1 Processing accelerators: IBM Spyre Accelerator Virtual Function (rev 02)
  root@b315lp04:/sys/bus/pci# echo 0 > devices/1100\:00\:00.0/sriov_numvfs
  root@b315lp04:/sys/bus/pci# echo 1 > devices/1100\:00\:00.0/sriov_numvfs
  root@b315lp04:/sys/bus/pci# echo 1100:00:00.0 > drivers/vfio-pci/unbind
  root@b315lp04:/sys/bus/pci# echo 1100:00:00.0 > drivers/vfio-pci/bind
  root@b315lp04:/sys/bus/pci# echo 1 > devices/1100\:00\:00.0/sriov_numvfs
  root@b315lp04:/sys/bus/pci# echo 1 > devices/1100\:00\:00.0/remove
  root@b315lp04:/sys/bus/pci# lspci
  0103:00:00.0 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
  1180:00:00.0 Processing accelerators: IBM Spyre Accelerator (rev 02)
  1180:00:00.1 Processing accelerators: IBM Spyre Accelerator Virtual Function (rev 02)
  root@b315lp04:/sys/bus/pci# rmmod vfio_ccw vfio_pci vfio_pci_core vfio_iommu_type1 vfio
  root@b315lp04:/sys/bus/pci# modprobe -a vfio_ccw vfio_pci vfio_pci_core vfio_iommu_type1 vfio
  root@b315lp04:/sys/bus/pci# echo 1 > rescan
  root@b315lp04:/sys/bus/pci# echo 1 > devices/1100\:00\:00.0/sriov_numvfs
  root@b315lp04:/sys/bus/pci# echo 1 > devices/1180\:00\:00.0/sriov_numvfs
  root@b315lp04:/sys/bus/pci# lspci
  0103:00:00.0 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
  1100:00:00.0 Processing accelerators: IBM Spyre Accelerator (rev 02)
  1100:00:00.1 Processing accelerators: IBM Spyre Accelerator Virtual Function (rev 02)
  1180:00:00.0 Processing accelerators: IBM Spyre Accelerator (rev 02)
  1180:00:00.1 Processing accelerators: IBM Spyre Accelerator Virtual Function (rev 02)
  root@b315lp04:/sys/bus/pci# echo 0 > devices/1180\:00\:00.0/sriov_numvfs
  root@b315lp04:/sys/bus/pci# echo 1 > devices/1180\:00\:00.0/sriov_numvfs
  root@b315lp04:/sys/bus/pci# echo 1180:00:00.0 > drivers/vfio-pci/unbind
  root@b315lp04:/sys/bus/pci# echo 1180:00:00.0 > drivers/vfio-pci/bind
  root@b315lp04:/sys/bus/pci# echo 1 > devices/1180\:00\:00.0/sriov_numvfs
  root@b315lp04:/sys/bus/pci# echo 1 > devices/1180\:00\:00.0/remove
  root@b315lp04:/sys/bus/pci# lspci
  0103:00:00.0 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
  1100:00:00.0 Processing accelerators: IBM Spyre Accelerator (rev 02)
  1100:00:00.1 Processing accelerators: IBM Spyre Accelerator Virtual Function (rev 02)

Same result: all without any Warnings/Lockdep-Splats/Kmemleak-Splats/etc.

-- 
Best Regards, Benjamin Block        /        Linux on IBM Z Kernel Development
IBM Deutschland Research & Development GmbH    /   https://www.ibm.com/privacy
Vors. Aufs.-R.: Wolfgang Wendt         /        Geschäftsführung: David Faller
Sitz der Ges.: Ehningen     /     Registergericht: AmtsG Stuttgart, HRB 243294