[PATCH net-next V3] net/mlx5: Expedite notifier unregistration during device teardown

From: Li RongQing <[email protected]>

During device hot-unplug, the mlx5 driver expects quickly unregister
notification chains. The standard atomic_notifier_chain_unregister()
calls synchronize_rcu(), which introduces significant latency and
can become a bottleneck during mass resource cleanup.

Introduce atomic_notifier_chain_unregister_expedited() to leverage
synchronize_rcu_expedited(), and use it significantly reducing wait
times in the following paths:
 - Event Queue (EQ) notifier chain
 - Firmware event notifier chain
 - IRQ notifier chain

On x86-64 with HZ=1000, 64 networking channels:
- Average teardown time: 3.59s -> 1.9s (47% reduction)
On x86-64 with HZ=250, 64 networking channels:
- Average teardown time: 5.5s -> 1.9s (65% reduction)

Co-developed-by: liyongkang <[email protected]>
Signed-off-by: liyongkang <[email protected]>
Signed-off-by: Li RongQing <[email protected]>
Signed-off-by: Tariq Toukan <[email protected]>
---
V3: rebased.

V2:
https://lore.kernel.org/all/[email protected]/

drivers/net/ethernet/mellanox/mlx5/core/eq.c  |  2 +-
 .../net/ethernet/mellanox/mlx5/core/events.c  |  2 +-
 .../net/ethernet/mellanox/mlx5/core/pci_irq.c |  2 +-
 include/linux/notifier.h                      |  2 ++
 kernel/notifier.c                             | 24 +++++++++++++++++++
 5 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index d11ec263d53c..c86090cb58c6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -1243,6 +1243,6 @@ int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
 {
 	struct mlx5_eq_table *eqt = dev->priv.eq_table;
 
-	return atomic_notifier_chain_unregister(&eqt->nh[nb->event_type], &nb->nb);
+	return atomic_notifier_chain_unregister_expedited(&eqt->nh[nb->event_type], &nb->nb);
 }
 EXPORT_SYMBOL(mlx5_eq_notifier_unregister);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c
index 4d7f35b96876..753cb15cdeee 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/events.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c
@@ -436,7 +436,7 @@ int mlx5_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *n
 {
 	struct mlx5_events *events = dev->priv.events;
 
-	return atomic_notifier_chain_unregister(&events->fw_nh, nb);
+	return atomic_notifier_chain_unregister_expedited(&events->fw_nh, nb);
 }
 EXPORT_SYMBOL(mlx5_notifier_unregister);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
index 0f5b8bc7861e..cb31e8c7a244 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c
@@ -343,7 +343,7 @@ int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb)
 {
 	int err = 0;
 
-	err = atomic_notifier_chain_unregister(&irq->nh, nb);
+	err = atomic_notifier_chain_unregister_expedited(&irq->nh, nb);
 	mlx5_irq_put(irq);
 	return err;
 }
diff --git a/include/linux/notifier.h b/include/linux/notifier.h
index 01b6c9d9956f..156d958f98f0 100644
--- a/include/linux/notifier.h
+++ b/include/linux/notifier.h
@@ -159,6 +159,8 @@ extern int blocking_notifier_chain_register_unique_prio(
 
 extern int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
 		struct notifier_block *nb);
+extern int atomic_notifier_chain_unregister_expedited(struct atomic_notifier_head *nh,
+		struct notifier_block *nb);
 extern int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
 		struct notifier_block *nb);
 extern int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
diff --git a/kernel/notifier.c b/kernel/notifier.c
index 2f9fe7c30287..9b35822fc2c0 100644
--- a/kernel/notifier.c
+++ b/kernel/notifier.c
@@ -197,6 +197,30 @@ int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
 }
 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
 
+/**
+ *	atomic_notifier_chain_unregister_expedited - Remove notifier from an atomic notifier chain
+ *	@nh: Pointer to head of the atomic notifier chain
+ *	@n: Entry to remove from notifier chain
+ *
+ *	Removes a notifier from an atomic notifier chain and forcefully
+ *	accelerates the RCU grace period.
+ *
+ *	Return: 0 on success, or -ENOENT on failure.
+ */
+int atomic_notifier_chain_unregister_expedited(struct atomic_notifier_head *nh,
+		struct notifier_block *n)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&nh->lock, flags);
+	ret = notifier_chain_unregister(&nh->head, n);
+	spin_unlock_irqrestore(&nh->lock, flags);
+	synchronize_rcu_expedited();
+	return ret;
+}
+EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister_expedited);
+
 /**
  *	atomic_notifier_call_chain - Call functions in an atomic notifier chain
  *	@nh: Pointer to head of the atomic notifier chain

base-commit: d67e5dbda22604d0fcde32fce58c65f88676e676
-- 
2.44.0