Re: [RESEND RFC PATCH v2] nvme-pci: add adaptive interrupt polling

From c3d9989d99c901abde1397651801309df3e11347 Mon Sep 17 00:00:00 2001
From: Fengnan Chang <[email protected]>
Date: Tue, 11 Aug 2026 20:22:27 +0800
Subject: [PATCH] nvme-pci: add adaptive interrupt polling

Add opt-in adaptive polling for nvme, measure an IRQ
completion-rate baseline over 8192 CQEs, then admit a bounded
polling trial only when the observed rate can sustain the fixed
10 us polling period.

Signed-off-by: Fengnan Chang <[email protected]>
---
 Documentation/ABI/testing/sysfs-nvme |  15 +
 drivers/nvme/host/Kconfig            |   1 +
 drivers/nvme/host/pci.c              | 564 +++++++++++++++++++++++++--
 3 files changed, 554 insertions(+), 26 deletions(-)

diff --git a/Documentation/ABI/testing/sysfs-nvme b/Documentation/ABI/testing/sysfs-nvme
index 499d5f843cd43..309d59dc6b872 100644
--- a/Documentation/ABI/testing/sysfs-nvme
+++ b/Documentation/ABI/testing/sysfs-nvme
@@ -11,3 +11,18 @@ Description:
 		(REPLACETLSPSK) with the target. After a reauthentication
 		the value returned by tls_configured_key will be the new
 		serial.
+
+What:		/sys/class/nvme/nvmeX/adaptive_irq_polling
+Date:		August 2026
+KernelVersion:	7.2
+Contact:	Linux NVMe mailing list <[email protected]>
+Description:
+		Enable (1) or disable (0) adaptive IRQ polling on eligible I/O
+		queues of one PCI NVMe controller.  Changing the mode freezes its
+		namespace request queues and waits for all outstanding namespace I/O
+		to complete before updating the queue IRQ state.  Writes fail with
+		EBUSY unless the controller is live.
+
+		The attribute is not available with threaded NVMe interrupts.  The
+		use_adaptive_irq_polling module parameter supplies only the initial
+		value for newly probed controllers.
diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index 31974c7dd20c9..22164b901da85 100644
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -5,6 +5,7 @@ config NVME_CORE
 config BLK_DEV_NVME
 	tristate "NVM Express block device"
 	depends on PCI && BLOCK
+	select IRQ_POLL
 	select NVME_CORE
 	help
 	  The NVM Express driver is for solid state drives directly
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 69932d640b537..b560278247d1a 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -10,9 +10,12 @@
 #include <linux/blk-mq-dma.h>
 #include <linux/blk-integrity.h>
 #include <linux/dmi.h>
+#include <linux/hrtimer.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/irq_poll.h>
+#include <linux/ktime.h>
 #include <linux/kstrtox.h>
 #include <linux/memremap.h>
 #include <linux/mm.h>
@@ -82,6 +85,42 @@ struct quirk_entry {
 static int use_threaded_interrupts;
 module_param(use_threaded_interrupts, int, 0444);
 
+static bool use_adaptive_irq_polling;
+module_param(use_adaptive_irq_polling, bool, 0444);
+MODULE_PARM_DESC(use_adaptive_irq_polling,
+		 "default adaptive polling mode for non-threaded MSI-X I/O queues");
+
+/*
+ * Adaptive IRQ polling moves an eligible interrupt-driven queue to a
+ * timer-based poll path according to its observed completion rate.  Each
+ * queue goes through three stages:
+ *
+ *  1. Baseline (still in IRQ mode): measure completion throughput over one
+ *     NVME_ADAPTIVE_EPISODE_CQES window.  Trial polling only while the average
+ *     completion gap is short enough that a poll fired once per
+ *     NVME_ADAPTIVE_POLL_PERIOD_NS would still find a completion.
+ *  2. Trial: mask the queue's IRQ and drain the CQ from an hrtimer every
+ *     NVME_ADAPTIVE_POLL_PERIOD_NS.  Reject the trial immediately if its
+ *     cumulative completion rate falls behind the IRQ baseline.  At the end
+ *     of a full completion window, accept polling only if its throughput is
+ *     strictly higher than the IRQ baseline.
+ *  3. Park or back off: keep an accepted queue in polling mode, recheck each
+ *     completion window, and periodically return to IRQ mode for a fresh
+ *     baseline.  A rejected or slowing queue may retry twice; three consecutive
+ *     failures trigger a long, completion-counted backoff.  This lets a
+ *     profitable queue get past a noisy first trial without making failed
+ *     trials a measurable part of workloads where IRQs are faster.
+ *
+ * The poll period bounds the extra completion latency polling may add and
+ * defines the admission rate above.  The completion-counted windows make the
+ * comparison and retry duty cycle independent of wall-clock load changes.
+ */
+#define NVME_ADAPTIVE_TARGET_BATCH	5U
+#define NVME_ADAPTIVE_POLL_PERIOD_NS	(10U * NSEC_PER_USEC)
+#define NVME_ADAPTIVE_EPISODE_CQES	8192U
+#define NVME_ADAPTIVE_REEVAL_CQES	(64U * NVME_ADAPTIVE_EPISODE_CQES)
+#define NVME_ADAPTIVE_POLL_RETRIES	2U
+
 static bool use_cmb_sqes = true;
 module_param(use_cmb_sqes, bool, 0444);
 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
@@ -307,6 +346,7 @@ struct nvme_dev {
 	void __iomem *bar;
 	unsigned long bar_mapped_size;
 	struct mutex shutdown_lock;
+	bool adaptive_irq_polling;
 	bool subsystem;
 	u64 cmb_size;
 	bool cmb_use_sqes;
@@ -358,6 +398,22 @@ static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
 	return container_of(ctrl, struct nvme_dev, ctrl);
 }
 
+/*
+ * Allocated only for interrupt queues that can poll adaptively.  cq_poll_lock
+ * serializes this state and CQ access between the IRQ and timer poll paths.
+ */
+struct nvme_adaptive_poll {
+	struct hrtimer timer;		/* fires the next poll drain */
+	struct irq_poll iopoll;		/* softirq context for the drain */
+	struct nvme_queue *nvmeq;
+	u64 start_ns;			/* when the current sample/episode started */
+	u32 retry_completions;		/* completions until retry or IRQ rebaseline */
+	u32 interval_ns;		/* sampled average gap between completions */
+	u32 completions;		/* completions seen so far this sample/episode */
+	int irq;
+	u8 poll_failures;		/* consecutive rejected polling trials */
+};
+
 /*
  * An NVM Express queue.  Each device has at least two (one for admin
  * commands and one for I/O commands).
@@ -367,7 +423,8 @@ struct nvme_queue {
 	struct nvme_descriptor_pools descriptor_pools;
 	spinlock_t sq_lock;
 	void *sq_cmds;
-	 /* only used for poll queues: */
+	struct nvme_adaptive_poll *adaptive;
+	/* Used for both poll queues and adaptive interrupt polling. */
 	spinlock_t cq_poll_lock ____cacheline_aligned_in_smp;
 	struct nvme_completion *cqes;
 	dma_addr_t sq_dma_addr;
@@ -386,6 +443,9 @@ struct nvme_queue {
 #define NVMEQ_SQ_CMB		1
 #define NVMEQ_DELETE_ERROR	2
 #define NVMEQ_POLLED		3
+#define NVMEQ_ADAPTIVE_POLLING	4
+#define NVMEQ_ADAPTIVE_ENABLED	5
+#define NVMEQ_ADAPTIVE_STALE_IRQ	6
 	__le32 *dbbuf_sq_db;
 	__le32 *dbbuf_cq_db;
 	__le32 *dbbuf_sq_ei;
@@ -1606,13 +1666,12 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
 	}
 }
 
-static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
-			        struct io_comp_batch *iob)
+static inline unsigned int nvme_poll_cq(struct nvme_queue *nvmeq,
+					struct io_comp_batch *iob)
 {
-	bool found = false;
+	unsigned int found = 0;
 
 	while (nvme_cqe_pending(nvmeq)) {
-		found = true;
 		/*
 		 * load-load control dependency between phase and the rest of
 		 * the cqe requires a full read memory barrier
@@ -1620,6 +1679,7 @@ static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
 		dma_rmb();
 		nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head);
 		nvme_update_cq_head(nvmeq);
+		found++;
 	}
 
 	if (found)
@@ -1627,17 +1687,22 @@ static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
 	return found;
 }
 
-static irqreturn_t nvme_irq(int irq, void *data)
+/* Keep the normal completion loop branch-free. */
+static unsigned int nvme_poll_cq_bounded(struct nvme_queue *nvmeq,
+					 struct io_comp_batch *iob,
+					 unsigned int limit)
 {
-	struct nvme_queue *nvmeq = data;
-	DEFINE_IO_COMP_BATCH(iob);
+	unsigned int found = 0;
 
-	if (nvme_poll_cq(nvmeq, &iob)) {
-		if (!rq_list_empty(&iob.req_list))
-			nvme_pci_complete_batch(&iob);
-		return IRQ_HANDLED;
+	while (found < limit && nvme_cqe_pending(nvmeq)) {
+		dma_rmb();
+		nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head);
+		nvme_update_cq_head(nvmeq);
+		found++;
 	}
-	return IRQ_NONE;
+	if (found)
+		nvme_ring_cq_doorbell(nvmeq);
+	return found;
 }
 
 static irqreturn_t nvme_irq_check(int irq, void *data)
@@ -1649,6 +1714,262 @@ static irqreturn_t nvme_irq_check(int irq, void *data)
 	return IRQ_NONE;
 }
 
+static void nvme_adaptive_state_reset(struct nvme_adaptive_poll *adaptive)
+{
+	adaptive->start_ns = 0;
+	adaptive->retry_completions = 0;
+	adaptive->interval_ns = 0;
+	adaptive->completions = 0;
+	adaptive->poll_failures = 0;
+}
+
+/*
+ * Stop polling and turn the queue's IRQ back on.  Retry two rejected trials
+ * promptly so a noisy transition does not hide a profitable state; three
+ * consecutive failures use a long IRQ backoff before another baseline.  The
+ * resulting low retry duty cycle prevents failed trials from reducing
+ * steady-state IRQ throughput.  Called with cq_poll_lock held.
+ */
+static void nvme_adaptive_poll_end(struct nvme_queue *nvmeq, bool backoff)
+{
+	struct nvme_adaptive_poll *adaptive = nvmeq->adaptive;
+	u8 poll_failures = adaptive->poll_failures;
+
+	nvme_adaptive_state_reset(adaptive);
+	if (backoff) {
+		if (poll_failures < NVME_ADAPTIVE_POLL_RETRIES)
+			adaptive->poll_failures = poll_failures + 1;
+		else
+			adaptive->retry_completions =
+				NVME_ADAPTIVE_REEVAL_CQES;
+	}
+	clear_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags);
+	set_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags);
+	enable_irq(adaptive->irq);
+}
+
+static void nvme_adaptive_poll_window_start(struct nvme_adaptive_poll *adaptive,
+					    u64 now)
+{
+	adaptive->start_ns = now;
+	adaptive->completions = 0;
+}
+
+/*
+ * Fire the next poll one fixed period from now.  A per-queue cadence is not
+ * needed: the cumulative-rate check below keeps a queue in poll mode only
+ * while it stays faster than its sampled IRQ rate, and a fixed period bounds
+ * the completion latency polling may add.  Anchor the timer to the caller's
+ * "now" so per-poll processing time does not stretch the effective period.
+ */
+static void nvme_adaptive_arm(struct nvme_adaptive_poll *adaptive, u64 now)
+{
+	hrtimer_start(&adaptive->timer,
+		      ns_to_ktime(now + NVME_ADAPTIVE_POLL_PERIOD_NS),
+		      HRTIMER_MODE_ABS_PINNED_HARD);
+}
+
+static enum hrtimer_restart nvme_adaptive_poll_timer(struct hrtimer *timer)
+{
+	struct nvme_adaptive_poll *adaptive = container_of(timer,
+					struct nvme_adaptive_poll, timer);
+	struct nvme_queue *nvmeq = adaptive->nvmeq;
+
+	if (test_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags))
+		irq_poll_sched(&adaptive->iopoll);
+	return HRTIMER_NORESTART;
+}
+
+/*
+ * The poll drain, run from softirq when the timer fires.  Reap some CQEs, then
+ * pick one of three things: stop if we hit the episode cap, wait again if the
+ * queue is keeping up, or go back to IRQ mode if it went idle or slowed down.
+ */
+static int nvme_adaptive_irq_poll(struct irq_poll *iop, int budget)
+{
+	struct nvme_adaptive_poll *adaptive = container_of(iop,
+					struct nvme_adaptive_poll, iopoll);
+	struct nvme_queue *nvmeq = adaptive->nvmeq;
+	unsigned int completions, limit;
+	unsigned long flags;
+	u64 deadline, elapsed, now;
+	DEFINE_IO_COMP_BATCH(iob);
+
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
+	if (unlikely(!test_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags))) {
+		completions = 0;
+		irq_poll_complete(iop);
+		goto out;
+	}
+	if (!test_bit(NVMEQ_ENABLED, &nvmeq->flags)) {
+		completions = 0;
+		irq_poll_complete(iop);
+		nvme_adaptive_poll_end(nvmeq, false);
+		goto out;
+	}
+
+	limit = min_t(unsigned int,
+		      budget,
+		      NVME_ADAPTIVE_EPISODE_CQES - adaptive->completions);
+	completions = nvme_poll_cq_bounded(nvmeq, &iob, limit);
+	adaptive->completions += completions;
+
+	if (completions >= budget &&
+	    adaptive->completions < NVME_ADAPTIVE_EPISODE_CQES)
+		goto out;
+	irq_poll_complete(iop);
+
+	/*
+	 * Before the window fills, tolerate two poll periods of cumulative lag
+	 * to absorb timer jitter and bursty completions.  At the window boundary,
+	 * remove that slack and require a strictly shorter elapsed time than the
+	 * sampled IRQ interval.  interval_ns is rounded down, so this final test
+	 * cannot accept polling that is equal to or slower than the IRQ baseline.
+	 */
+	now = ktime_get_ns();
+	elapsed = now - adaptive->start_ns;
+	deadline = (u64)adaptive->completions * adaptive->interval_ns;
+	if (adaptive->completions < NVME_ADAPTIVE_EPISODE_CQES) {
+		if (elapsed > deadline +
+		    2U * NVME_ADAPTIVE_POLL_PERIOD_NS)
+			nvme_adaptive_poll_end(nvmeq, true);
+		else
+			nvme_adaptive_arm(adaptive, now);
+		goto out;
+	}
+	if (elapsed >= deadline) {
+		nvme_adaptive_poll_end(nvmeq, true);
+		goto out;
+	}
+
+	adaptive->poll_failures = 0;
+	/* Reuse the retry counter, updating it once per window. */
+	adaptive->retry_completions -= NVME_ADAPTIVE_EPISODE_CQES;
+	if (!adaptive->retry_completions) {
+		nvme_adaptive_poll_end(nvmeq, false);
+		goto out;
+	}
+	nvme_adaptive_poll_window_start(adaptive, now);
+	nvme_adaptive_arm(adaptive, now);
+out:
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
+	if (!rq_list_empty(&iob.req_list))
+		nvme_pci_complete_batch(&iob);
+	return completions;
+}
+
+/*
+ * Called from the IRQ handler after a reap that found something.  If we're
+ * still in backoff, just count it down.  Otherwise establish an IRQ completion
+ * rate over one episode.  If that rate is high enough that a poll every
+ * NVME_ADAPTIVE_POLL_PERIOD_NS would still find work, mask the IRQ and switch
+ * to a same-sized polling trial; if not, leave the queue on interrupts.
+ *
+ * Called with nvmeq->cq_poll_lock held.
+ */
+static void nvme_adaptive_sample(struct nvme_queue *nvmeq,
+				 unsigned int completions)
+{
+	struct nvme_adaptive_poll *adaptive = nvmeq->adaptive;
+	u64 delta, interval, now;
+
+	if (adaptive->retry_completions) {
+		adaptive->retry_completions -= min(completions,
+						   adaptive->retry_completions);
+		return;
+	}
+	if (!adaptive->start_ns) {
+		adaptive->start_ns = ktime_get_ns();
+		return;
+	}
+	adaptive->completions += completions;
+	if (adaptive->completions < NVME_ADAPTIVE_EPISODE_CQES)
+		return;
+
+	now = ktime_get_ns();
+	delta = now - adaptive->start_ns;
+	/*
+	 * The completion-rate test is an admission gate, not the trial result.
+	 * Comparing a full IRQ window against a full poll window below decides
+	 * whether polling actually pays for itself on this queue.
+	 */
+	interval = div64_u64(delta, adaptive->completions);
+	if (!interval || interval > NVME_ADAPTIVE_POLL_PERIOD_NS ||
+	    !test_bit(NVMEQ_ENABLED, &nvmeq->flags)) {
+		nvme_adaptive_poll_window_start(adaptive, now);
+		return;
+	}
+
+	adaptive->interval_ns = interval;
+	adaptive->retry_completions = NVME_ADAPTIVE_REEVAL_CQES;
+	nvme_adaptive_poll_window_start(adaptive, now);
+	set_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags);
+	disable_irq_nosync(adaptive->irq);
+	nvme_adaptive_arm(adaptive, now);
+}
+
+static irqreturn_t nvme_irq(int irq, void *data);
+
+/*
+ * IRQ handler for queues that may switch to adaptive polling.  It reaps the CQ
+ * like the normal handler, then feeds the count to the sampler, which may flip
+ * the queue into poll mode.  The CQ lock makes an IRQ racing with the timer
+ * harmless: it either drains the CQ before the switch or observes poll mode
+ * and leaves the CQ to the timer.  An empty IRQ after polling is the interrupt
+ * that was pending when the vector was masked, so consume it as handled.
+ */
+static noinline irqreturn_t nvme_irq_adaptive_enabled(int irq, void *data)
+{
+	struct nvme_queue *nvmeq = data;
+	unsigned int completions;
+	unsigned long flags;
+	DEFINE_IO_COMP_BATCH(iob);
+
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
+	if (unlikely(test_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags))) {
+		spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
+		return IRQ_HANDLED;
+	}
+	completions = nvme_poll_cq(nvmeq, &iob);
+	if (completions)
+		nvme_adaptive_sample(nvmeq, completions);
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
+	if (!completions)
+		return test_and_clear_bit(NVMEQ_ADAPTIVE_STALE_IRQ,
+					  &nvmeq->flags) ? IRQ_HANDLED : IRQ_NONE;
+	if (!rq_list_empty(&iob.req_list))
+		nvme_pci_complete_batch(&iob);
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t nvme_irq_adaptive(int irq, void *data)
+{
+	struct nvme_queue *nvmeq = data;
+	irqreturn_t ret;
+
+	if (!test_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags)) {
+		ret = nvme_irq(irq, data);
+		if (ret == IRQ_NONE &&
+		    test_and_clear_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags))
+			return IRQ_HANDLED;
+		return ret;
+	}
+	return nvme_irq_adaptive_enabled(irq, data);
+}
+
+static irqreturn_t nvme_irq(int irq, void *data)
+{
+	struct nvme_queue *nvmeq = data;
+	DEFINE_IO_COMP_BATCH(iob);
+
+	if (nvme_poll_cq(nvmeq, &iob)) {
+		if (!rq_list_empty(&iob.req_list))
+			nvme_pci_complete_batch(&iob);
+		return IRQ_HANDLED;
+	}
+	return IRQ_NONE;
+}
+
 /*
  * Poll for completions for any interrupt driven queue
  * Can be called from any context.
@@ -1656,30 +1977,38 @@ static irqreturn_t nvme_irq_check(int irq, void *data)
 static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
 {
 	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
+	unsigned long flags;
 	int irq;
 
 	WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags));
 
 	irq = pci_irq_vector(pdev, nvmeq->cq_vector);
 	disable_irq(irq);
-	spin_lock(&nvmeq->cq_poll_lock);
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
 	nvme_poll_cq(nvmeq, NULL);
-	spin_unlock(&nvmeq->cq_poll_lock);
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
 	enable_irq(irq);
 }
 
 static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 {
 	struct nvme_queue *nvmeq = hctx->driver_data;
+	unsigned long flags;
 	bool found;
 
 	if (!test_bit(NVMEQ_POLLED, &nvmeq->flags) ||
 	    !nvme_cqe_pending(nvmeq))
 		return 0;
 
-	spin_lock(&nvmeq->cq_poll_lock);
+	/*
+	 * cq_poll_lock is also acquired from hardirq context by adaptive IRQ
+	 * polling on interrupt-driven queues.  Disable IRQs here so all
+	 * acquirers share a consistent context and lockdep cannot see an
+	 * IRQ-safe class taken with IRQs enabled.
+	 */
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
 	found = nvme_poll_cq(nvmeq, iob);
-	spin_unlock(&nvmeq->cq_poll_lock);
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
 
 	return found;
 }
@@ -2017,8 +2346,7 @@ static void nvme_free_queue(struct nvme_queue *nvmeq)
 	dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq),
 				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
 	if (!nvmeq->sq_cmds)
-		return;
-
+		goto free_adaptive;
 	if (test_and_clear_bit(NVMEQ_SQ_CMB, &nvmeq->flags)) {
 		pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev),
 				nvmeq->sq_cmds, SQ_SIZE(nvmeq));
@@ -2026,6 +2354,9 @@ static void nvme_free_queue(struct nvme_queue *nvmeq)
 		dma_free_coherent(nvmeq->dev->dev, SQ_SIZE(nvmeq),
 				nvmeq->sq_cmds, nvmeq->sq_dma_addr);
 	}
+free_adaptive:
+	kfree(nvmeq->adaptive);
+	nvmeq->adaptive = NULL;
 }
 
 static void nvme_free_queues(struct nvme_dev *dev, int lowest)
@@ -2038,9 +2369,97 @@ static void nvme_free_queues(struct nvme_dev *dev, int lowest)
 	}
 }
 
+static int nvme_adaptive_suspend(struct nvme_queue *nvmeq)
+{
+	struct nvme_adaptive_poll *adaptive = nvmeq->adaptive;
+	unsigned long flags;
+	int irq;
+
+	if (!adaptive || adaptive->irq < 0)
+		return -1;
+	irq = adaptive->irq;
+	synchronize_irq(irq);
+	irq_poll_disable(&adaptive->iopoll);
+	/* irq_poll_complete() can run before the poll callback returns. */
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
+	if (test_and_clear_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags)) {
+		set_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags);
+		enable_irq(irq);
+	}
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
+	hrtimer_cancel(&adaptive->timer);
+	return irq;
+}
+
+static void nvme_adaptive_set_queue(struct nvme_queue *nvmeq, bool enable)
+{
+	struct nvme_adaptive_poll *adaptive = nvmeq->adaptive;
+	unsigned long flags;
+
+	if (nvme_adaptive_suspend(nvmeq) < 0) {
+		clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+		return;
+	}
+
+	spin_lock_irqsave(&nvmeq->cq_poll_lock, flags);
+	nvme_adaptive_state_reset(adaptive);
+	if (enable)
+		set_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+	else
+		clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+	spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags);
+	irq_poll_enable(&adaptive->iopoll);
+}
+
+/*
+ * Freeze and drain namespace I/O before changing the completion mode.  The
+ * locks keep namespace and controller teardown paths from changing the queue
+ * set while its IRQ state is updated.
+ */
+static int nvme_adaptive_switch(struct nvme_dev *dev, bool enable)
+{
+	int qid, ret = 0;
+
+	mutex_lock(&dev->ctrl.scan_lock);
+	if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_LIVE) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+	if (enable == READ_ONCE(dev->adaptive_irq_polling))
+		goto out_unlock;
+
+	nvme_start_freeze(&dev->ctrl);
+	nvme_wait_freeze(&dev->ctrl);
+
+	mutex_lock(&dev->shutdown_lock);
+	if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_LIVE) {
+		ret = -EBUSY;
+	} else {
+		for (qid = 1; qid < dev->ctrl.queue_count; qid++)
+			nvme_adaptive_set_queue(&dev->queues[qid], enable);
+		WRITE_ONCE(dev->adaptive_irq_polling, enable);
+	}
+	mutex_unlock(&dev->shutdown_lock);
+
+	nvme_unfreeze(&dev->ctrl);
+out_unlock:
+	mutex_unlock(&dev->ctrl.scan_lock);
+	return ret;
+}
+
+static void nvme_adaptive_suspend_done(struct nvme_queue *nvmeq, int irq)
+{
+	if (irq < 0)
+		return;
+	nvmeq->adaptive->irq = -1;
+	irq_poll_enable(&nvmeq->adaptive->iopoll);
+}
+
 static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid)
 {
 	struct nvme_queue *nvmeq = &dev->queues[qid];
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+	int irq;
 
 	if (!test_and_clear_bit(NVMEQ_ENABLED, &nvmeq->flags))
 		return;
@@ -2051,8 +2470,11 @@ static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid)
 	nvmeq->dev->online_queues--;
 	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
 		nvme_quiesce_admin_queue(&nvmeq->dev->ctrl);
-	if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags))
-		pci_free_irq(to_pci_dev(dev->dev), nvmeq->cq_vector, nvmeq);
+	if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags)) {
+		irq = nvme_adaptive_suspend(nvmeq);
+		pci_free_irq(pdev, nvmeq->cq_vector, nvmeq);
+		nvme_adaptive_suspend_done(nvmeq, irq);
+	}
 }
 
 static void nvme_suspend_io_queues(struct nvme_dev *dev)
@@ -2071,12 +2493,13 @@ static void nvme_suspend_io_queues(struct nvme_dev *dev)
  */
 static void nvme_reap_pending_cqes(struct nvme_dev *dev)
 {
+	unsigned long flags;
 	int i;
 
 	for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
-		spin_lock(&dev->queues[i].cq_poll_lock);
+		spin_lock_irqsave(&dev->queues[i].cq_poll_lock, flags);
 		nvme_poll_cq(&dev->queues[i], NULL);
-		spin_unlock(&dev->queues[i].cq_poll_lock);
+		spin_unlock_irqrestore(&dev->queues[i].cq_poll_lock, flags);
 	}
 }
 
@@ -2166,18 +2589,75 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
 	return -ENOMEM;
 }
 
+static bool nvme_adaptive_init(struct nvme_queue *nvmeq)
+{
+	struct nvme_adaptive_poll *adaptive = nvmeq->adaptive;
+	int irq = pci_irq_vector(to_pci_dev(nvmeq->dev->dev),
+				 nvmeq->cq_vector);
+
+	if (irq < 0)
+		return false;
+	if (!adaptive) {
+		adaptive = kzalloc_node(sizeof(*adaptive), GFP_KERNEL,
+					dev_to_node(nvmeq->dev->dev));
+		if (!adaptive)
+			return false;
+		adaptive->nvmeq = nvmeq;
+		hrtimer_setup(&adaptive->timer, nvme_adaptive_poll_timer,
+			      CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
+		irq_poll_init(&adaptive->iopoll, 64, nvme_adaptive_irq_poll);
+		adaptive->irq = irq;
+		WRITE_ONCE(nvmeq->adaptive, adaptive);
+		return true;
+	}
+	adaptive->irq = irq;
+	return true;
+}
+
 static int queue_request_irq(struct nvme_queue *nvmeq)
 {
 	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
 	int nr = nvmeq->dev->ctrl.instance;
+	bool adaptive_queue;
+	int ret;
 
 	if (use_threaded_interrupts) {
+		clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
 		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
 				nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
-	} else {
-		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
-				NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
 	}
+	/* Decide static eligibility once, before installing the IRQ handler. */
+	adaptive_queue = nvmeq->qid && nvmeq->dev->num_vecs > 1 &&
+		pdev->msix_enabled &&
+		nvmeq->q_depth >= NVME_ADAPTIVE_TARGET_BATCH;
+	if (adaptive_queue)
+		adaptive_queue = nvme_adaptive_init(nvmeq);
+	if (adaptive_queue && READ_ONCE(nvmeq->dev->adaptive_irq_polling))
+		set_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+	else
+		clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+	ret = pci_request_irq(pdev, nvmeq->cq_vector,
+			      adaptive_queue ? nvme_irq_adaptive : nvme_irq,
+			      NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
+	if (!adaptive_queue || ret) {
+		clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+		if (ret && nvmeq->adaptive)
+			nvmeq->adaptive->irq = -1;
+	}
+	return ret;
+}
+
+static void nvme_adaptive_reset(struct nvme_queue *nvmeq)
+{
+	struct nvme_adaptive_poll *adaptive = nvmeq->adaptive;
+
+	clear_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags);
+	clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags);
+	clear_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags);
+	if (!adaptive)
+		return;
+	nvme_adaptive_state_reset(adaptive);
+	adaptive->irq = -1;
 }
 
 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
@@ -2188,6 +2668,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
 	nvmeq->last_sq_tail = 0;
 	nvmeq->cq_head = 0;
 	nvmeq->cq_phase = 1;
+	nvme_adaptive_reset(nvmeq);
 	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
 	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq));
 	nvme_dbbuf_init(dev, nvmeq, qid);
@@ -2808,6 +3289,33 @@ static ssize_t hmb_store(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR_RW(hmb);
 
+static ssize_t adaptive_irq_polling_show(struct device *dev,
+					 struct device_attribute *attr,
+					 char *buf)
+{
+	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
+
+	return sysfs_emit(buf, "%d\n", READ_ONCE(ndev->adaptive_irq_polling));
+}
+
+static ssize_t adaptive_irq_polling_store(struct device *dev,
+					  struct device_attribute *attr,
+					  const char *buf, size_t count)
+{
+	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
+	bool enable;
+	int ret;
+
+	ret = kstrtobool(buf, &enable);
+	if (ret)
+		return ret;
+	ret = nvme_adaptive_switch(ndev, enable);
+	if (ret)
+		return ret;
+	return count;
+}
+static DEVICE_ATTR_RW(adaptive_irq_polling);
+
 static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj,
 		struct attribute *a, int n)
 {
@@ -2823,6 +3331,8 @@ static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj,
 	}
 	if (a == &dev_attr_hmb.attr && !ctrl->hmpre)
 		return 0;
+	if (a == &dev_attr_adaptive_irq_polling.attr && use_threaded_interrupts)
+		return 0;
 
 	return a->mode;
 }
@@ -2832,6 +3342,7 @@ static struct attribute *nvme_pci_attrs[] = {
 	&dev_attr_cmbloc.attr,
 	&dev_attr_cmbsz.attr,
 	&dev_attr_hmb.attr,
+	&dev_attr_adaptive_irq_polling.attr,
 	NULL,
 };
 
@@ -3685,6 +4196,7 @@ static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev,
 		return ERR_PTR(-ENOMEM);
 	INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
 	mutex_init(&dev->shutdown_lock);
+	dev->adaptive_irq_polling = use_adaptive_irq_polling;
 
 	dev->nr_write_queues = write_queues;
 	dev->nr_poll_queues = poll_queues;
-- 
2.39.5 (Apple Git-154)