[PATCH] nvme-pci: add adaptive interrupt polling
"Fengnan Chang" <[email protected]>
| Newsgroups | org.infradead.lists.linux-nvme,org.kernel.vger.linux-kernel |
|---|---|
| Message-ID | <[email protected]> |
In high-IOPS scenarios, relying on interrupts to handle I/O operations can limit performance. This issue becomes particularly pronounced in multi-disk environments, where performance is constrained by the CPU's interrupt-handling capacity. Add an opt-in adaptive policy for eligible non-threaded MSI-X I/O queues. The idea behind this approach is: let each I/O queue switch itself between interrupt and poll mode based on its own recent completion rate. In IRQ mode, sample at least 8192 completions and start a polling attempt when the average completion interval is at most 10 us. During the trial, hrtimer-driven irq_poll drains the CQ. Progress may trail the IRQ baseline by 20 us within a window, but a full 8192-CQE poll window must be strictly faster. Keep successful polling for at most 64 windows before refreshing the IRQ baseline. Allow two retries; after the third failed trial, count 64 windows worth of IRQ completions before resampling. The default-off module parameter initializes new controllers. A per-controller sysfs attribute changes the policy after freezing namespace request queues and draining outstanding I/O. Threaded interrupts are not supported. Link: https://lore.kernel.org/linux-nvme/d9210bcdf73fbe1ac8b6ec132865609a3ed68688.ff265e95.1296.491e.89f9.8ae888a03346@bytedance.com/T/#mea881a7898c85b73992f568864001913cb456d59 Link: https://lore.kernel.org/linux-nvme/[email protected]/T/#u Signed-off-by: Guzebing <[email protected]> Signed-off-by: Fengnan Chang <[email protected]> --- Documentation/ABI/testing/sysfs-nvme | 15 + drivers/nvme/host/Kconfig | 1 + drivers/nvme/host/pci.c | 507 +++++++++++++++++++++++++-- 3 files changed, 497 insertions(+), 26 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-nvme b/Documentation/ABI/testing/sysfs-nvme index 499d5f843cd43..695da343f65ef 100644 --- a/Documentation/ABI/testing/sysfs-nvme +++ b/Documentation/ABI/testing/sysfs-nvme @@ -11,3 +11,18 @@ Description: (REPLACETLSPSK) with the target. After a reauthentication the value returned by tls_configured_key will be the new serial. + +What: /sys/class/nvme/nvmeX/adaptive_irq_polling +Date: August 2026 +KernelVersion: 7.2 +Contact: Linux NVMe mailing list <[email protected]> +Description: + Set the adaptive IRQ polling policy (0 or 1) for eligible I/O queues of one + PCI NVMe controller. Reading returns the policy, not whether a queue + is currently polling. Changing it freezes the namespace request + queues and waits for outstanding namespace I/O. Writes fail with + EBUSY unless the controller is live. + + Eligible queues use non-threaded MSI-X with a dedicated vector. The + attribute is unavailable with threaded interrupts. The module + parameter supplies only the initial policy for new controllers. diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig index 31974c7dd20c9..22164b901da85 100644 --- a/drivers/nvme/host/Kconfig +++ b/drivers/nvme/host/Kconfig @@ -5,6 +5,7 @@ config NVME_CORE config BLK_DEV_NVME tristate "NVM Express block device" depends on PCI && BLOCK + select IRQ_POLL select NVME_CORE help The NVM Express driver is for solid state drives directly diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 69932d640b537..b24c462d5aff2 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -10,9 +10,12 @@ #include <linux/blk-mq-dma.h> #include <linux/blk-integrity.h> #include <linux/dmi.h> +#include <linux/hrtimer.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/irq_poll.h> +#include <linux/ktime.h> #include <linux/kstrtox.h> #include <linux/memremap.h> #include <linux/mm.h> @@ -82,6 +85,16 @@ struct quirk_entry { static int use_threaded_interrupts; module_param(use_threaded_interrupts, int, 0444); +static bool use_adaptive_irq_polling; +module_param(use_adaptive_irq_polling, bool, 0444); +MODULE_PARM_DESC(use_adaptive_irq_polling, + "default adaptive polling policy for eligible I/O queues"); + +#define NVME_ADAPTIVE_POLL_PERIOD_NS (10U * NSEC_PER_USEC) +#define NVME_ADAPTIVE_EPISODE_CQES 8192U +#define NVME_ADAPTIVE_REEVAL_CQES (64U * NVME_ADAPTIVE_EPISODE_CQES) +#define NVME_ADAPTIVE_POLL_RETRIES 2U + static bool use_cmb_sqes = true; module_param(use_cmb_sqes, bool, 0444); MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes"); @@ -307,6 +320,7 @@ struct nvme_dev { void __iomem *bar; unsigned long bar_mapped_size; struct mutex shutdown_lock; + bool adaptive_irq_polling; bool subsystem; u64 cmb_size; bool cmb_use_sqes; @@ -358,6 +372,18 @@ static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl) return container_of(ctrl, struct nvme_dev, ctrl); } +struct nvme_adaptive_poll { + struct hrtimer timer; /* fires the next poll drain */ + struct irq_poll iopoll; /* softirq context for the drain */ + struct nvme_queue *nvmeq; + u64 start_ns; /* when the current sample/episode started */ + u32 retry_completions; /* completions until retry or IRQ rebaseline */ + u32 interval_ns; /* sampled average gap between completions */ + u32 completions; /* completions seen so far this sample/episode */ + int irq; + u8 poll_failures; /* consecutive rejected polling trials */ +}; + /* * An NVM Express queue. Each device has at least two (one for admin * commands and one for I/O commands). @@ -367,7 +393,8 @@ struct nvme_queue { struct nvme_descriptor_pools descriptor_pools; spinlock_t sq_lock; void *sq_cmds; - /* only used for poll queues: */ + struct nvme_adaptive_poll *adaptive; + /* Used for both poll queues and adaptive interrupt polling. */ spinlock_t cq_poll_lock ____cacheline_aligned_in_smp; struct nvme_completion *cqes; dma_addr_t sq_dma_addr; @@ -386,6 +413,9 @@ struct nvme_queue { #define NVMEQ_SQ_CMB 1 #define NVMEQ_DELETE_ERROR 2 #define NVMEQ_POLLED 3 +#define NVMEQ_ADAPTIVE_POLLING 4 +#define NVMEQ_ADAPTIVE_ENABLED 5 +#define NVMEQ_ADAPTIVE_STALE_IRQ 6 __le32 *dbbuf_sq_db; __le32 *dbbuf_cq_db; __le32 *dbbuf_sq_ei; @@ -1606,13 +1636,12 @@ static inline void nvme_update_cq_head(struct nvme_queue *nvmeq) } } -static inline bool nvme_poll_cq(struct nvme_queue *nvmeq, - struct io_comp_batch *iob) +static inline unsigned int nvme_poll_cq(struct nvme_queue *nvmeq, + struct io_comp_batch *iob) { - bool found = false; + unsigned int found = 0; while (nvme_cqe_pending(nvmeq)) { - found = true; /* * load-load control dependency between phase and the rest of * the cqe requires a full read memory barrier @@ -1620,6 +1649,7 @@ static inline bool nvme_poll_cq(struct nvme_queue *nvmeq, dma_rmb(); nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head); nvme_update_cq_head(nvmeq); + found++; } if (found) @@ -1627,17 +1657,22 @@ static inline bool nvme_poll_cq(struct nvme_queue *nvmeq, return found; } -static irqreturn_t nvme_irq(int irq, void *data) +/* Keep the normal completion loop branch-free. */ +static unsigned int nvme_poll_cq_bounded(struct nvme_queue *nvmeq, + struct io_comp_batch *iob, + unsigned int limit) { - struct nvme_queue *nvmeq = data; - DEFINE_IO_COMP_BATCH(iob); + unsigned int found = 0; - if (nvme_poll_cq(nvmeq, &iob)) { - if (!rq_list_empty(&iob.req_list)) - nvme_pci_complete_batch(&iob); - return IRQ_HANDLED; + while (found < limit && nvme_cqe_pending(nvmeq)) { + dma_rmb(); + nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head); + nvme_update_cq_head(nvmeq); + found++; } - return IRQ_NONE; + if (found) + nvme_ring_cq_doorbell(nvmeq); + return found; } static irqreturn_t nvme_irq_check(int irq, void *data) @@ -1649,6 +1684,235 @@ static irqreturn_t nvme_irq_check(int irq, void *data) return IRQ_NONE; } +/* Reset adaptive state to an uninitialised IRQ baseline. */ +static void nvme_adaptive_state_reset(struct nvme_adaptive_poll *adaptive) +{ + adaptive->start_ns = 0; + adaptive->retry_completions = 0; + adaptive->interval_ns = 0; + adaptive->completions = 0; + adaptive->poll_failures = 0; +} + +/* + * Restore IRQ mode. The first two failed trials rebaseline immediately; + * the third backs off for 64 windows. + */ +static void nvme_adaptive_poll_end(struct nvme_queue *nvmeq, bool backoff) +{ + struct nvme_adaptive_poll *adaptive = nvmeq->adaptive; + u8 poll_failures = adaptive->poll_failures; + + nvme_adaptive_state_reset(adaptive); + if (backoff) { + if (poll_failures < NVME_ADAPTIVE_POLL_RETRIES) + adaptive->poll_failures = poll_failures + 1; + else + adaptive->retry_completions = + NVME_ADAPTIVE_REEVAL_CQES; + } + clear_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags); + set_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags); + enable_irq(adaptive->irq); +} + +static void nvme_adaptive_poll_window_start(struct nvme_adaptive_poll *adaptive, + u64 now) +{ + adaptive->start_ns = now; + adaptive->completions = 0; +} + +static void nvme_adaptive_arm(struct nvme_adaptive_poll *adaptive, u64 now) +{ + hrtimer_start(&adaptive->timer, + ns_to_ktime(now + NVME_ADAPTIVE_POLL_PERIOD_NS), + HRTIMER_MODE_ABS_PINNED_HARD); +} + +static enum hrtimer_restart nvme_adaptive_poll_timer(struct hrtimer *timer) +{ + struct nvme_adaptive_poll *adaptive = container_of(timer, + struct nvme_adaptive_poll, timer); + struct nvme_queue *nvmeq = adaptive->nvmeq; + + if (test_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags)) + irq_poll_sched(&adaptive->iopoll); + return HRTIMER_NORESTART; +} + +/* + * Drain CQEs from IRQ_POLL_SOFTIRQ and compare completion progress with the + * IRQ baseline. Re-arm while within the allowed lag; leave poll mode on lag + * or teardown, and start another window only after a faster full window. + */ +static int nvme_adaptive_irq_poll(struct irq_poll *iop, int budget) +{ + struct nvme_adaptive_poll *adaptive = container_of(iop, + struct nvme_adaptive_poll, iopoll); + struct nvme_queue *nvmeq = adaptive->nvmeq; + unsigned int completions, limit; + unsigned long flags; + u64 deadline, elapsed, now; + DEFINE_IO_COMP_BATCH(iob); + + spin_lock_irqsave(&nvmeq->cq_poll_lock, flags); + if (unlikely(!test_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags))) { + completions = 0; + irq_poll_complete(iop); + goto out; + } + if (!test_bit(NVMEQ_ENABLED, &nvmeq->flags)) { + completions = 0; + irq_poll_complete(iop); + nvme_adaptive_poll_end(nvmeq, false); + goto out; + } + + limit = min_t(unsigned int, + budget, + NVME_ADAPTIVE_EPISODE_CQES - adaptive->completions); + completions = nvme_poll_cq_bounded(nvmeq, &iob, limit); + adaptive->completions += completions; + + if (completions >= budget && + adaptive->completions < NVME_ADAPTIVE_EPISODE_CQES) + goto out; + irq_poll_complete(iop); + + /* + * Before the window fills, allow progress to trail the IRQ baseline by + * two poll periods. At the boundary, require a strictly shorter time. + * interval_ns is rounded down, so equal or slower never passes. + */ + now = ktime_get_ns(); + elapsed = now - adaptive->start_ns; + deadline = (u64)adaptive->completions * adaptive->interval_ns; + if (adaptive->completions < NVME_ADAPTIVE_EPISODE_CQES) { + if (elapsed > deadline + + 2U * NVME_ADAPTIVE_POLL_PERIOD_NS) + nvme_adaptive_poll_end(nvmeq, true); + else + nvme_adaptive_arm(adaptive, now); + goto out; + } + if (elapsed >= deadline) { + nvme_adaptive_poll_end(nvmeq, true); + goto out; + } + + adaptive->poll_failures = 0; + /* Bound polling to 64 successful windows before an IRQ rebaseline. */ + adaptive->retry_completions -= NVME_ADAPTIVE_EPISODE_CQES; + if (!adaptive->retry_completions) { + nvme_adaptive_poll_end(nvmeq, false); + goto out; + } + nvme_adaptive_poll_window_start(adaptive, now); + nvme_adaptive_arm(adaptive, now); +out: + spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags); + if (!rq_list_empty(&iob.req_list)) + nvme_pci_complete_batch(&iob); + return completions; +} + +/* + * Count down an IRQ backoff or sample at least one completion window. + * A polling attempt is permitted when the average interval is no + * greater than the poll period. + */ +static void nvme_adaptive_sample(struct nvme_queue *nvmeq, + unsigned int completions) +{ + struct nvme_adaptive_poll *adaptive = nvmeq->adaptive; + u64 delta, interval, now; + + if (adaptive->retry_completions) { + adaptive->retry_completions -= min(completions, + adaptive->retry_completions); + return; + } + if (!adaptive->start_ns) { + adaptive->start_ns = ktime_get_ns(); + return; + } + adaptive->completions += completions; + if (adaptive->completions < NVME_ADAPTIVE_EPISODE_CQES) + return; + + now = ktime_get_ns(); + delta = now - adaptive->start_ns; + /* Admission only; a full poll window decides whether polling wins. */ + interval = div64_u64(delta, adaptive->completions); + if (!interval || interval > NVME_ADAPTIVE_POLL_PERIOD_NS || + !test_bit(NVMEQ_ENABLED, &nvmeq->flags)) { + nvme_adaptive_poll_window_start(adaptive, now); + return; + } + + adaptive->interval_ns = interval; + adaptive->retry_completions = NVME_ADAPTIVE_REEVAL_CQES; + nvme_adaptive_poll_window_start(adaptive, now); + set_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags); + disable_irq_nosync(adaptive->irq); + nvme_adaptive_arm(adaptive, now); +} + +static irqreturn_t nvme_irq(int irq, void *data); + +static noinline irqreturn_t nvme_irq_adaptive_enabled(int irq, void *data) +{ + struct nvme_queue *nvmeq = data; + unsigned int completions; + unsigned long flags; + DEFINE_IO_COMP_BATCH(iob); + + spin_lock_irqsave(&nvmeq->cq_poll_lock, flags); + if (unlikely(test_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags))) { + spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags); + return IRQ_HANDLED; + } + completions = nvme_poll_cq(nvmeq, &iob); + if (completions) + nvme_adaptive_sample(nvmeq, completions); + spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags); + if (!completions) + return test_and_clear_bit(NVMEQ_ADAPTIVE_STALE_IRQ, + &nvmeq->flags) ? IRQ_HANDLED : IRQ_NONE; + if (!rq_list_empty(&iob.req_list)) + nvme_pci_complete_batch(&iob); + return IRQ_HANDLED; +} + +static irqreturn_t nvme_irq_adaptive(int irq, void *data) +{ + struct nvme_queue *nvmeq = data; + irqreturn_t ret; + + if (!test_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags)) { + ret = nvme_irq(irq, data); + if (ret == IRQ_NONE && + test_and_clear_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags)) + return IRQ_HANDLED; + return ret; + } + return nvme_irq_adaptive_enabled(irq, data); +} + +static irqreturn_t nvme_irq(int irq, void *data) +{ + struct nvme_queue *nvmeq = data; + DEFINE_IO_COMP_BATCH(iob); + + if (nvme_poll_cq(nvmeq, &iob)) { + if (!rq_list_empty(&iob.req_list)) + nvme_pci_complete_batch(&iob); + return IRQ_HANDLED; + } + return IRQ_NONE; +} + /* * Poll for completions for any interrupt driven queue * Can be called from any context. @@ -1656,30 +1920,36 @@ static irqreturn_t nvme_irq_check(int irq, void *data) static void nvme_poll_irqdisable(struct nvme_queue *nvmeq) { struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); + unsigned long flags; int irq; WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags)); irq = pci_irq_vector(pdev, nvmeq->cq_vector); disable_irq(irq); - spin_lock(&nvmeq->cq_poll_lock); + spin_lock_irqsave(&nvmeq->cq_poll_lock, flags); nvme_poll_cq(nvmeq, NULL); - spin_unlock(&nvmeq->cq_poll_lock); + spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags); enable_irq(irq); } static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) { struct nvme_queue *nvmeq = hctx->driver_data; + unsigned long flags; bool found; if (!test_bit(NVMEQ_POLLED, &nvmeq->flags) || !nvme_cqe_pending(nvmeq)) return 0; - spin_lock(&nvmeq->cq_poll_lock); + /* + * cq_poll_lock is also taken from hardirq by the adaptive handler. + * Disable IRQs here so lockdep sees a consistent lock class. + */ + spin_lock_irqsave(&nvmeq->cq_poll_lock, flags); found = nvme_poll_cq(nvmeq, iob); - spin_unlock(&nvmeq->cq_poll_lock); + spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags); return found; } @@ -2017,8 +2287,7 @@ static void nvme_free_queue(struct nvme_queue *nvmeq) dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq), (void *)nvmeq->cqes, nvmeq->cq_dma_addr); if (!nvmeq->sq_cmds) - return; - + goto free_adaptive; if (test_and_clear_bit(NVMEQ_SQ_CMB, &nvmeq->flags)) { pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev), nvmeq->sq_cmds, SQ_SIZE(nvmeq)); @@ -2026,6 +2295,9 @@ static void nvme_free_queue(struct nvme_queue *nvmeq) dma_free_coherent(nvmeq->dev->dev, SQ_SIZE(nvmeq), nvmeq->sq_cmds, nvmeq->sq_dma_addr); } +free_adaptive: + kfree(nvmeq->adaptive); + nvmeq->adaptive = NULL; } static void nvme_free_queues(struct nvme_dev *dev, int lowest) @@ -2038,9 +2310,96 @@ static void nvme_free_queues(struct nvme_dev *dev, int lowest) } } +static int nvme_adaptive_suspend(struct nvme_queue *nvmeq) +{ + struct nvme_adaptive_poll *adaptive = nvmeq->adaptive; + unsigned long flags; + int irq; + + if (!adaptive || adaptive->irq < 0) + return -1; + irq = adaptive->irq; + synchronize_irq(irq); + irq_poll_disable(&adaptive->iopoll); + /* irq_poll_complete() can run before the poll callback returns. */ + spin_lock_irqsave(&nvmeq->cq_poll_lock, flags); + if (test_and_clear_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags)) { + set_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags); + enable_irq(irq); + } + spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags); + hrtimer_cancel(&adaptive->timer); + return irq; +} + +static void nvme_adaptive_set_queue(struct nvme_queue *nvmeq, bool enable) +{ + struct nvme_adaptive_poll *adaptive = nvmeq->adaptive; + unsigned long flags; + + if (nvme_adaptive_suspend(nvmeq) < 0) { + clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags); + return; + } + + spin_lock_irqsave(&nvmeq->cq_poll_lock, flags); + nvme_adaptive_state_reset(adaptive); + if (enable) + set_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags); + else + clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags); + spin_unlock_irqrestore(&nvmeq->cq_poll_lock, flags); + irq_poll_enable(&adaptive->iopoll); +} + +/* + * Freeze namespace I/O before switching completion mode. scan_lock keeps + * the namespace set stable; shutdown_lock prevents concurrent reset. + */ +static int nvme_adaptive_switch(struct nvme_dev *dev, bool enable) +{ + int qid, ret = 0; + + mutex_lock(&dev->ctrl.scan_lock); + if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_LIVE) { + ret = -EBUSY; + goto out_unlock; + } + if (enable == READ_ONCE(dev->adaptive_irq_polling)) + goto out_unlock; + + nvme_start_freeze(&dev->ctrl); + nvme_wait_freeze(&dev->ctrl); + + mutex_lock(&dev->shutdown_lock); + if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_LIVE) { + ret = -EBUSY; + } else { + for (qid = 1; qid < dev->ctrl.queue_count; qid++) + nvme_adaptive_set_queue(&dev->queues[qid], enable); + WRITE_ONCE(dev->adaptive_irq_polling, enable); + } + mutex_unlock(&dev->shutdown_lock); + + nvme_unfreeze(&dev->ctrl); +out_unlock: + mutex_unlock(&dev->ctrl.scan_lock); + return ret; +} + +static void nvme_adaptive_suspend_done(struct nvme_queue *nvmeq, int irq) +{ + if (irq < 0) + return; + nvmeq->adaptive->irq = -1; + irq_poll_enable(&nvmeq->adaptive->iopoll); +} + static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid) { struct nvme_queue *nvmeq = &dev->queues[qid]; + struct pci_dev *pdev = to_pci_dev(dev->dev); + int irq; if (!test_and_clear_bit(NVMEQ_ENABLED, &nvmeq->flags)) return; @@ -2051,8 +2410,11 @@ static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid) nvmeq->dev->online_queues--; if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q) nvme_quiesce_admin_queue(&nvmeq->dev->ctrl); - if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags)) - pci_free_irq(to_pci_dev(dev->dev), nvmeq->cq_vector, nvmeq); + if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags)) { + irq = nvme_adaptive_suspend(nvmeq); + pci_free_irq(pdev, nvmeq->cq_vector, nvmeq); + nvme_adaptive_suspend_done(nvmeq, irq); + } } static void nvme_suspend_io_queues(struct nvme_dev *dev) @@ -2071,12 +2433,13 @@ static void nvme_suspend_io_queues(struct nvme_dev *dev) */ static void nvme_reap_pending_cqes(struct nvme_dev *dev) { + unsigned long flags; int i; for (i = dev->ctrl.queue_count - 1; i > 0; i--) { - spin_lock(&dev->queues[i].cq_poll_lock); + spin_lock_irqsave(&dev->queues[i].cq_poll_lock, flags); nvme_poll_cq(&dev->queues[i], NULL); - spin_unlock(&dev->queues[i].cq_poll_lock); + spin_unlock_irqrestore(&dev->queues[i].cq_poll_lock, flags); } } @@ -2166,18 +2529,78 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth) return -ENOMEM; } +/* + * Allocate or re-arm adaptive state after reset. The caller has established + * MSI-X eligibility; return false if vector lookup or allocation fails. + */ +static bool nvme_adaptive_init(struct nvme_queue *nvmeq) +{ + struct nvme_adaptive_poll *adaptive = nvmeq->adaptive; + int irq = pci_irq_vector(to_pci_dev(nvmeq->dev->dev), + nvmeq->cq_vector); + + if (irq < 0) + return false; + if (!adaptive) { + adaptive = kzalloc_node(sizeof(*adaptive), GFP_KERNEL, + dev_to_node(nvmeq->dev->dev)); + if (!adaptive) + return false; + adaptive->nvmeq = nvmeq; + hrtimer_setup(&adaptive->timer, nvme_adaptive_poll_timer, + CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD); + irq_poll_init(&adaptive->iopoll, 64, nvme_adaptive_irq_poll); + adaptive->irq = irq; + WRITE_ONCE(nvmeq->adaptive, adaptive); + return true; + } + adaptive->irq = irq; + return true; +} + static int queue_request_irq(struct nvme_queue *nvmeq) { struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); int nr = nvmeq->dev->ctrl.instance; + bool adaptive_queue; + int ret; if (use_threaded_interrupts) { + clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags); return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check, nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid); - } else { - return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq, - NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid); } + /* Install the adaptive-capable handler only on eligible queues. */ + adaptive_queue = nvmeq->qid && nvmeq->dev->num_vecs > 1 && + pdev->msix_enabled; + if (adaptive_queue) + adaptive_queue = nvme_adaptive_init(nvmeq); + if (adaptive_queue && READ_ONCE(nvmeq->dev->adaptive_irq_polling)) + set_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags); + else + clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags); + ret = pci_request_irq(pdev, nvmeq->cq_vector, + adaptive_queue ? nvme_irq_adaptive : nvme_irq, + NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid); + if (!adaptive_queue || ret) { + clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags); + if (ret && nvmeq->adaptive) + nvmeq->adaptive->irq = -1; + } + return ret; +} + +static void nvme_adaptive_reset(struct nvme_queue *nvmeq) +{ + struct nvme_adaptive_poll *adaptive = nvmeq->adaptive; + + clear_bit(NVMEQ_ADAPTIVE_POLLING, &nvmeq->flags); + clear_bit(NVMEQ_ADAPTIVE_ENABLED, &nvmeq->flags); + clear_bit(NVMEQ_ADAPTIVE_STALE_IRQ, &nvmeq->flags); + if (!adaptive) + return; + nvme_adaptive_state_reset(adaptive); + adaptive->irq = -1; } static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) @@ -2188,6 +2611,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) nvmeq->last_sq_tail = 0; nvmeq->cq_head = 0; nvmeq->cq_phase = 1; + nvme_adaptive_reset(nvmeq); nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq)); nvme_dbbuf_init(dev, nvmeq, qid); @@ -2808,6 +3232,33 @@ static ssize_t hmb_store(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RW(hmb); +static ssize_t adaptive_irq_polling_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); + + return sysfs_emit(buf, "%d\n", READ_ONCE(ndev->adaptive_irq_polling)); +} + +static ssize_t adaptive_irq_polling_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); + bool enable; + int ret; + + ret = kstrtobool(buf, &enable); + if (ret) + return ret; + ret = nvme_adaptive_switch(ndev, enable); + if (ret) + return ret; + return count; +} +static DEVICE_ATTR_RW(adaptive_irq_polling); + static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj, struct attribute *a, int n) { @@ -2823,6 +3274,8 @@ static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj, } if (a == &dev_attr_hmb.attr && !ctrl->hmpre) return 0; + if (a == &dev_attr_adaptive_irq_polling.attr && use_threaded_interrupts) + return 0; return a->mode; } @@ -2832,6 +3285,7 @@ static struct attribute *nvme_pci_attrs[] = { &dev_attr_cmbloc.attr, &dev_attr_cmbsz.attr, &dev_attr_hmb.attr, + &dev_attr_adaptive_irq_polling.attr, NULL, }; @@ -3685,6 +4139,7 @@ static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev, return ERR_PTR(-ENOMEM); INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work); mutex_init(&dev->shutdown_lock); + dev->adaptive_irq_polling = use_adaptive_irq_polling; dev->nr_write_queues = write_queues; dev->nr_poll_queues = poll_queues; -- 2.39.5 (Apple Git-154)