[PATCH][net-next v2] net/mlx5: Expedite notifier unregistration during device teardown

[lirongqing]

From: Li RongQing <lirongqing@xxxxxxxxx>

During device hot-unplug, the mlx5 driver expects quickly unregister
notification chains. The standard atomic_notifier_chain_unregister()
calls synchronize_rcu(), which introduces significant latency and
can become a bottleneck during mass resource cleanup.

Introduce atomic_notifier_chain_unregister_expedited() to leverage
synchronize_rcu_expedited(), and use it significantly reducing wait
times in the following paths:
 - Event Queue (EQ) notifier chain
 - Firmware event notifier chain
 - IRQ notifier chain

On x86-64 with HZ=1000, 64 networking channels:
- Average teardown time: 3.59s -> 1.9s (47% reduction)
On x86-64 with HZ=250, 64 networking channels:
- Average teardown time: 5.5s -> 1.9s (65% reduction)

Co-developed-by: liyongkang <liyongkang01@xxxxxxxxx>
Signed-off-by: liyongkang <liyongkang01@xxxxxxxxx>
Signed-off-by: Li RongQing <lirongqing@xxxxxxxxx>
---
Diff with v1: fix doc warning and add detailed example

 drivers/net/ethernet/mellanox/mlx5/core/eq.c      |  2 +-
 drivers/net/ethernet/mellanox/mlx5/core/events.c  |  2 +-
 drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c |  2 +-
 include/linux/notifier.h                          |  2 ++
 kernel/notifier.c                                 | 24 +++++++++++++++++++++++
 5 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 22a6371..03ae6ed 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -1244,6 +1244,6 @@ int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
 {
 	struct mlx5_eq_table *eqt = dev->priv.eq_table;
 
-	return atomic_notifier_chain_unregister(&eqt->nh[nb->event_type], &nb->nb);
+	return atomic_notifier_chain_unregister_expedited(&eqt->nh[nb->event_type], &nb->nb);
 }
 EXPORT_SYMBOL(mlx5_eq_notifier_unregister);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c
index 4d7f35b..753cb15 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/events.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c
@@ -436,7 +436,7 @@ int mlx5_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *n
 {
 	struct mlx5_events *events = dev->priv.events;
 
-	return atomic_notifier_chain_unregister(&events->fw_nh, nb);
+	return atomic_notifier_chain_unregister_expedited(&events->fw_nh, nb);
 }
 EXPORT_SYMBOL(mlx5_notifier_unregister);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
index e051b9a..826685d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
@@ -356,7 +356,7 @@ int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb)
 {
 	int err = 0;
 
-	err = atomic_notifier_chain_unregister(&irq->nh, nb);
+	err = atomic_notifier_chain_unregister_expedited(&irq->nh, nb);
 	mlx5_irq_put(irq);
 	return err;
 }
diff --git a/include/linux/notifier.h b/include/linux/notifier.h
index 01b6c9d..156d958 100644
--- a/include/linux/notifier.h
+++ b/include/linux/notifier.h
@@ -159,6 +159,8 @@ extern int blocking_notifier_chain_register_unique_prio(
 
 extern int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
 		struct notifier_block *nb);
+extern int atomic_notifier_chain_unregister_expedited(struct atomic_notifier_head *nh,
+		struct notifier_block *nb);
 extern int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
 		struct notifier_block *nb);
 extern int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
diff --git a/kernel/notifier.c b/kernel/notifier.c
index 2f9fe7c..9b35822 100644
--- a/kernel/notifier.c
+++ b/kernel/notifier.c
@@ -198,6 +198,30 @@ int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
 
 /**
+ *	atomic_notifier_chain_unregister_expedited - Remove notifier from an atomic notifier chain
+ *	@nh: Pointer to head of the atomic notifier chain
+ *	@n: Entry to remove from notifier chain
+ *
+ *	Removes a notifier from an atomic notifier chain and forcefully
+ *	accelerates the RCU grace period.
+ *
+ *	Return: 0 on success, or -ENOENT on failure.
+ */
+int atomic_notifier_chain_unregister_expedited(struct atomic_notifier_head *nh,
+		struct notifier_block *n)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&nh->lock, flags);
+	ret = notifier_chain_unregister(&nh->head, n);
+	spin_unlock_irqrestore(&nh->lock, flags);
+	synchronize_rcu_expedited();
+	return ret;
+}
+EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister_expedited);
+
+/**
  *	atomic_notifier_call_chain - Call functions in an atomic notifier chain
  *	@nh: Pointer to head of the atomic notifier chain
  *	@val: Value passed unmodified to notifier function
-- 
2.9.4