[PATCH rdma-next 9/9] RDMA/mlx5: Register event notifier before IB registration
From: Edward Srouji
Date: Sun Nov 22 2026 - 12:40:23 EST
The device event notifier is registered after ib_register_device(),
so a port change during registration is missed once ib_core or a
client has read the port configuration.
Register it before ib_register_device(), as the system error notifier
already is. Until the device is active the notifier queues no handler,
so the P_Key change work still cannot run while ib_mad creates or
destroys the GSI QPs.
Commit ede132a5cf55 ("RDMA/mlx5: Move events notifier registration to
be after device registration") moved the notifier after registration
so that it is unregistered before the GSI QPs are destroyed. What has
to come first is stopping its work. A new stage,
MLX5_IB_STAGE_EVENT_FLUSH, cleaned up right after MLX5_IB_STAGE_ACTIVE
deactivates the device, flushes mlx5_ib_event_wq and then cancels the
P_Key change work, so a handler queued while the device was active can
no longer rearm it, the race that commit narrowed. A handler that runs
later, queued by a callback that saw the device active or by a slave
port, stops at the ib_active test now at the top of
mlx5_ib_handle_event(), which replaces the tests before dispatching.
MACsec event registration keeps its place, in a stage of its own.
The workqueue is shared by all mlx5_ib devices, and a flush may wait
for another device's replay. That should not deadlock: the replay
sleeps only on its own device's firmware commands, cache and event
handlers, and the slave port handlers on mlx5_ib_multiport_mutex,
which no flush holds; none of these take the E-switch reps_lock held
across representor removal.
Fixes: ede132a5cf55 ("RDMA/mlx5: Move events notifier registration to be after device registration")
Fixes: 7722f47e71e5 ("IB/mlx5: Create GSI transmission QPs when P_Key table is changed")
Reviewed-by: Michael Gur <michaelgur@xxxxxxxxxx>
Signed-off-by: Edward Srouji <edwards@xxxxxxxxxx>
---
drivers/infiniband/hw/mlx5/main.c | 72 ++++++++++++++++++++++++++----------
drivers/infiniband/hw/mlx5/mlx5_ib.h | 4 +-
2 files changed, 55 insertions(+), 21 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 8a9e68782a5c449e5565a098e75636d5054ee6c1..1e58aa132c5b5c6d6de694f298fe1d6ed11e4cd1 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3075,14 +3075,12 @@ static int handle_port_change(struct mlx5_ib_dev *ibdev, u32 port, u8 sub_type,
switch (sub_type) {
case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
- case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
- if (ibdev->ib_active) {
- struct ib_event speed_event = {};
+ case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED: {
+ struct ib_event speed_event = {};
- speed_event.device = &ibdev->ib_dev;
- speed_event.event = IB_EVENT_DEVICE_SPEED_CHANGE;
- ib_dispatch_event(&speed_event);
- }
+ speed_event.device = &ibdev->ib_dev;
+ speed_event.event = IB_EVENT_DEVICE_SPEED_CHANGE;
+ ib_dispatch_event(&speed_event);
/* In RoCE, port up/down events are handled in
* mlx5_netdev_event().
@@ -3094,6 +3092,7 @@ static int handle_port_change(struct mlx5_ib_dev *ibdev, u32 port, u8 sub_type,
ibev->event = (sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ?
IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
break;
+ }
case MLX5_PORT_CHANGE_SUBTYPE_LID:
ibev->event = IB_EVENT_LID_CHANGE;
@@ -3200,6 +3199,15 @@ static void mlx5_ib_handle_event(struct work_struct *_work)
ibdev = work->dev;
}
+ /*
+ * Teardown may have cancelled the work the handlers below rearm. A
+ * device seen active here stays usable until this returns: the system
+ * error handler runs on this same ordered workqueue, and teardown
+ * flushes it before releasing anything used below.
+ */
+ if (!mlx5_ib_dev_active(ibdev))
+ goto out;
+
switch (work->event) {
case MLX5_EVENT_TYPE_PORT_CHANGE: {
struct mlx5_eqe *eqe = work->param;
@@ -3217,9 +3225,7 @@ static void mlx5_ib_handle_event(struct work_struct *_work)
}
ibev.device = &ibdev->ib_dev;
-
- if (ibdev->ib_active)
- ib_dispatch_event(&ibev);
+ ib_dispatch_event(&ibev);
out:
kfree(work);
@@ -5095,18 +5101,32 @@ static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)
dev->mdev_events.notifier_call = mlx5_ib_event;
mlx5_notifier_register(dev->mdev, &dev->mdev_events);
- mlx5r_macsec_event_register(dev);
-
return 0;
}
static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
+{
+ mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
+ flush_workqueue(mlx5_ib_event_wq);
+}
+
+static int mlx5_ib_stage_macsec_notifier_init(struct mlx5_ib_dev *dev)
+{
+ mlx5r_macsec_event_register(dev);
+ return 0;
+}
+
+static void mlx5_ib_stage_macsec_notifier_cleanup(struct mlx5_ib_dev *dev)
+{
+ mlx5r_macsec_event_unregister(dev);
+}
+
+static void mlx5_ib_stage_event_flush_cleanup(struct mlx5_ib_dev *dev)
{
struct mlx5_ib_resources *devr = &dev->devr;
int port;
- mlx5r_macsec_event_unregister(dev);
- mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
+ flush_workqueue(mlx5_ib_event_wq);
for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
cancel_work_sync(&devr->ports[port].pkey_change_work);
@@ -5230,12 +5250,15 @@ static const struct mlx5_ib_profile pf_profile = {
STAGE_CREATE(MLX5_IB_STAGE_SYS_ERROR_NOTIFIER,
mlx5_ib_stage_sys_error_notifier_init,
mlx5_ib_stage_sys_error_notifier_cleanup),
- STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
- mlx5_ib_stage_ib_reg_init,
- mlx5_ib_stage_ib_reg_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
mlx5_ib_stage_dev_notifier_init,
mlx5_ib_stage_dev_notifier_cleanup),
+ STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
+ mlx5_ib_stage_ib_reg_init,
+ mlx5_ib_stage_ib_reg_cleanup),
+ STAGE_CREATE(MLX5_IB_STAGE_MACSEC_NOTIFIER,
+ mlx5_ib_stage_macsec_notifier_init,
+ mlx5_ib_stage_macsec_notifier_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
mlx5_ib_stage_post_ib_reg_umr_init,
NULL),
@@ -5245,6 +5268,9 @@ static const struct mlx5_ib_profile pf_profile = {
STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
mlx5_ib_restrack_init,
NULL),
+ STAGE_CREATE(MLX5_IB_STAGE_EVENT_FLUSH,
+ NULL,
+ mlx5_ib_stage_event_flush_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_ACTIVE,
mlx5_ib_stage_active_init,
mlx5_ib_stage_active_cleanup),
@@ -5293,12 +5319,15 @@ const struct mlx5_ib_profile raw_eth_profile = {
STAGE_CREATE(MLX5_IB_STAGE_SYS_ERROR_NOTIFIER,
mlx5_ib_stage_sys_error_notifier_init,
mlx5_ib_stage_sys_error_notifier_cleanup),
- STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
- mlx5_ib_stage_ib_reg_init,
- mlx5_ib_stage_ib_reg_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
mlx5_ib_stage_dev_notifier_init,
mlx5_ib_stage_dev_notifier_cleanup),
+ STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
+ mlx5_ib_stage_ib_reg_init,
+ mlx5_ib_stage_ib_reg_cleanup),
+ STAGE_CREATE(MLX5_IB_STAGE_MACSEC_NOTIFIER,
+ mlx5_ib_stage_macsec_notifier_init,
+ mlx5_ib_stage_macsec_notifier_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
mlx5_ib_stage_post_ib_reg_umr_init,
NULL),
@@ -5308,6 +5337,9 @@ const struct mlx5_ib_profile raw_eth_profile = {
STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
mlx5_ib_restrack_init,
NULL),
+ STAGE_CREATE(MLX5_IB_STAGE_EVENT_FLUSH,
+ NULL,
+ mlx5_ib_stage_event_flush_cleanup),
STAGE_CREATE(MLX5_IB_STAGE_ACTIVE,
mlx5_ib_stage_active_init,
mlx5_ib_stage_active_cleanup),
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 7e1e0bcd096c994473dcfe2db5655b47bd48915e..53ca49e6c4b4b6c7bedd3a1cba57a2dd5fa5093f 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -937,11 +937,13 @@ enum mlx5_ib_stages {
MLX5_IB_STAGE_PRE_IB_REG_UMR,
MLX5_IB_STAGE_WHITELIST_UID,
MLX5_IB_STAGE_SYS_ERROR_NOTIFIER,
- MLX5_IB_STAGE_IB_REG,
MLX5_IB_STAGE_DEVICE_NOTIFIER,
+ MLX5_IB_STAGE_IB_REG,
+ MLX5_IB_STAGE_MACSEC_NOTIFIER,
MLX5_IB_STAGE_POST_IB_REG_UMR,
MLX5_IB_STAGE_DELAY_DROP,
MLX5_IB_STAGE_RESTRACK,
+ MLX5_IB_STAGE_EVENT_FLUSH,
MLX5_IB_STAGE_ACTIVE,
MLX5_IB_STAGE_MAX,
};
--
2.49.0