Re: [PATCH 3/6] io_uring: switch local task_work to a mpscq

On Thu, Jun 11, 2026 at 7:51 PM Jens Axboe <[email protected]> wrote:
>
> The local (DEFER_TASKRUN) task_work list is an llist, which is LIFO
> ordered, and hence __io_run_local_work() has to restore the right
> running order with an O(n) llist_reverse_order() pass first. On top of
> that, a batch that gets capped by max_events needs the leftover entries
> parked on a separate ->retry_llist, as they can't be pushed back to the
> shared list.
>
> Switch it to the FIFO mpscq. Adds are wait-free instead of a cmpxchg
> retry loop, entries are popped in queue order with no reversal pass,
> capping a run simply leaves the remainder on the queue, and
> ->retry_llist goes away entirely. The consumer cursor, ->work_head,
> lives with the rest of the ->uring_lock protected state rather than
> next to the queue, so that popping entries doesn't dirty the producer
> side cacheline.
>
> For low amounts of task_work, this ends up being a bit more efficient
> than the existing scheme. As an example of that, doing multishot
> receives for 8 clients has the following task_work overhead:
>
>      1.02%  sock-test  [kernel.kallsyms]  [k] io_req_local_work_add
>      0.88%  sock-test  [kernel.kallsyms]  [k] __io_run_local_work_loop
>      0.60%  sock-test  [kernel.kallsyms]  [k] llist_reverse_order
>      0.14%  sock-test  [kernel.kallsyms]  [k] __io_run_local_work
>      2.64% at ~46Gb/sec
>
> and after this change:
>
>      1.08%  sock-test  [kernel.kallsyms]  [k] io_req_local_work_add
>      1.03%  sock-test  [kernel.kallsyms]  [k] __io_run_local_work
>      2.11% at ~53Gb/sec
>
> which has less overhead even though that test run was faster. For a case
> of having 1024 clients on a single ring:
>
>      2.22%  sock-test  [kernel.kallsyms]  [k] llist_reverse_order
>      0.84%  sock-test  [kernel.kallsyms]  [k] __io_run_local_work_loop
>      0.42%  sock-test  [kernel.kallsyms]  [k] io_req_local_work_add
>      0.02%  sock-test  [kernel.kallsyms]  [k] __io_run_local_work
>      3.50% at ~24Gb/sec
>
> we start to see the llist reversing taking a considerable amount of
> time, and the total add+run task_work overhead is around 3.5%. After
> the change:
>
>      0.90%  sock-test  [kernel.kallsyms]  [k] __io_run_local_work
>      0.42%  sock-test  [kernel.kallsyms]  [k] io_req_local_work_add
>      1.32% at ~26Gb/sec
>
> most of that overhead is gone, and performance is better as well.
>
> Signed-off-by: Jens Axboe <[email protected]>
> ---
>  include/linux/io_uring_types.h |  13 +++-
>  io_uring/io_uring.c            |   2 +-
>  io_uring/tw.c                  | 135 ++++++++++++++-------------------
>  io_uring/tw.h                  |   4 +-
>  io_uring/wait.c                |   2 +-
>  io_uring/wait.h                |  10 ++-
>  6 files changed, 78 insertions(+), 88 deletions(-)
>
> diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
> index 85e12b4884a5..9df5584ec3b1 100644
> --- a/include/linux/io_uring_types.h
> +++ b/include/linux/io_uring_types.h
> @@ -351,6 +351,14 @@ struct io_ring_ctx {
>                  */
>                 atomic_t                cancel_seq;
>
> +               /*
> +                * Consumer cursor for ->work_list, protected by ->uring_lock.
> +                * Deliberately kept away from the producer side of the queue,
> +                * as it's written for every popped entry, and the producer
> +                * cacheline is contended enough as it is.
> +                */
> +               struct llist_node       *work_head;

Looks like this field has padding both before (next to atomic_t) and
after (next to bool). Probably doesn't matter currently, as the outer
struct is cache-aligned and has 16 bytes of padding at the end, but
could save 8 bytes of padding by reordering next to an existing
8-byte-aligned field.

> +
>                 /*
>                  * ->iopoll_list is protected by the ctx->uring_lock for
>                  * io_uring instances that don't use IORING_SETUP_SQPOLL.
> @@ -417,8 +425,7 @@ struct io_ring_ctx {
>          */
>         struct {
>                 struct io_rings __rcu   *rings_rcu;
> -               struct llist_head       work_llist;
> -               struct llist_head       retry_llist;
> +               struct mpscq            work_list;
>                 unsigned long           check_cq;
>                 atomic_t                cq_wait_nr;
>                 atomic_t                cq_timeouts;
> @@ -742,8 +749,6 @@ struct io_kiocb {
>          */
>         u16                             buf_index;
>
> -       unsigned                        nr_tw;
> -
>         /* REQ_F_* flags */
>         io_req_flags_t                  flags;
>
> diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
> index 753ac23401c5..16acd99ff083 100644
> --- a/io_uring/io_uring.c
> +++ b/io_uring/io_uring.c
> @@ -280,7 +280,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
>         INIT_LIST_HEAD(&ctx->defer_list);
>         INIT_LIST_HEAD(&ctx->timeout_list);
>         INIT_LIST_HEAD(&ctx->ltimeout_list);
> -       init_llist_head(&ctx->work_llist);
> +       mpscq_init(&ctx->work_list, &ctx->work_head);
>         INIT_LIST_HEAD(&ctx->tctx_list);
>         mutex_init(&ctx->tctx_lock);
>         ctx->submit_state.free_list.next = NULL;
> diff --git a/io_uring/tw.c b/io_uring/tw.c
> index f4335c8d50d9..b8d6027aaeff 100644
> --- a/io_uring/tw.c
> +++ b/io_uring/tw.c
> @@ -14,6 +14,7 @@
>  #include "rw.h"
>  #include "eventfd.h"
>  #include "wait.h"
> +#include "mpscq.h"
>
>  void io_fallback_req_func(struct work_struct *work)
>  {
> @@ -170,11 +171,7 @@ static void io_ctx_mark_taskrun(struct io_ring_ctx *ctx)
>  void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
>  {
>         struct io_ring_ctx *ctx = req->ctx;
> -       unsigned nr_wait, nr_tw, nr_tw_prev;
> -       struct llist_node *head;
> -
> -       /* See comment above IO_CQ_WAKE_INIT */
> -       BUILD_BUG_ON(IO_CQ_WAKE_FORCE <= IORING_MAX_CQ_ENTRIES);
> +       int nr_wait;
>
>         /*
>          * We don't know how many requests there are in the link and whether
> @@ -183,56 +180,37 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
>         if (req->flags & IO_REQ_LINK_FLAGS)
>                 flags &= ~IOU_F_TWQ_LAZY_WAKE;
>
> -       guard(rcu)();
> -
> -       head = READ_ONCE(ctx->work_llist.first);
> -       do {
> -               nr_tw_prev = 0;
> -               if (head) {
> -                       struct io_kiocb *first_req = container_of(head,
> -                                                       struct io_kiocb,
> -                                                       io_task_work.node);
> -                       /*
> -                        * Might be executed at any moment, rely on
> -                        * SLAB_TYPESAFE_BY_RCU to keep it alive.
> -                        */
> -                       nr_tw_prev = READ_ONCE(first_req->nr_tw);
> -               }
> -
> -               /*
> -                * Theoretically, it can overflow, but that's fine as one of
> -                * previous adds should've tried to wake the task.
> -                */
> -               nr_tw = nr_tw_prev + 1;
> -               if (!(flags & IOU_F_TWQ_LAZY_WAKE))
> -                       nr_tw = IO_CQ_WAKE_FORCE;
> -
> -               req->nr_tw = nr_tw;
> -               req->io_task_work.node.next = head;
> -       } while (!try_cmpxchg(&ctx->work_llist.first, &head,
> -                             &req->io_task_work.node));
> -
>         /*
> -        * cmpxchg implies a full barrier, which pairs with the barrier
> -        * in set_current_state() on the io_cqring_wait() side. It's used
> -        * to ensure that either we see updated ->cq_wait_nr, or waiters
> -        * going to sleep will observe the work added to the list, which
> -        * is similar to the wait/wawke task state sync.
> +        * The xchg() in mpscq_push() implies a full barrier, which pairs with
> +        * the barrier in set_current_state() on the io_cqring_wait() side. This
> +        * ensures that either we see the updated ->cq_wait_nr, or waiters going
> +        * to sleep will observe the work added to the list, which is similar to
> +        * the wait/wake task state sync.
>          */
> -
> -       if (!head) {
> +       if (mpscq_push(&ctx->work_list, &req->io_task_work.node)) {
>                 io_ctx_mark_taskrun(ctx);
>                 if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD)
>                         io_eventfd_signal(ctx, false);
>         }
>
> +       /*
> +        * No one is waiting (IO_CQ_WAKE_INIT), or this cycle's wake up has
> +        * already been issued (zero or negative, see below).
> +        */
>         nr_wait = atomic_read(&ctx->cq_wait_nr);
> -       /* not enough or no one is waiting */
> -       if (nr_tw < nr_wait)
> +       if (nr_wait <= 0)
>                 return;
> -       /* the previous add has already woken it up */
> -       if (nr_tw_prev >= nr_wait)
> +       if (flags & IOU_F_TWQ_LAZY_WAKE) {
> +               /*
> +                * ->cq_wait_nr counts down the number of lazy adds, once it
> +                * hits zero we're good to wake the waiter.
> +                */
> +               if (!atomic_dec_and_test(&ctx->cq_wait_nr))
> +                       return;

It's possible that another task work wakes up the task before this one
reaches the atomic_dec_and_test(), right? If the submitter task begins
a new wait in between, this could decrement cq_wait_nr even though the
queued task work has already been processed after the previous wakeup.
I guess that's okay; in the worse case, the waiter will be woken
prematurely.

> +       } else if (!atomic_try_cmpxchg(&ctx->cq_wait_nr, &nr_wait, IO_CQ_WAKE_INIT)) {
> +               /* lost the race against another wake up, this one is covered */
>                 return;
> +       }
>         wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE);
>  }
>
> @@ -273,21 +251,27 @@ void io_req_task_work_add_remote(struct io_kiocb *req, unsigned flags)
>
>  void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
>  {
> -       struct llist_node *node;
> +       struct llist_node *node, *first = NULL, **tail = &first;
>
>         /*
> -        * Running the work items may utilize ->retry_llist as a means
> -        * for capping the number of task_work entries run at the same
> -        * time. But that list can potentially race with moving the work
> -        * from here, if the task is exiting. As any normal task_work
> -        * running holds ->uring_lock already, just guard this slow path
> -        * with ->uring_lock to avoid racing on ->retry_llist.
> +        * The work list consumer side is serialized by ->uring_lock, see
> +        * __io_run_local_work(). Grab it to guard against racing with normal
> +        * task_work running, as the task may be exiting.
>          */
>         guard(mutex)(&ctx->uring_lock);
> -       node = llist_del_all(&ctx->work_llist);
> -       __io_fallback_tw(node, false);
> -       node = llist_del_all(&ctx->retry_llist);
> -       __io_fallback_tw(node, false);
> +
> +       while (!mpscq_empty(&ctx->work_list)) {
> +               node = mpscq_pop(&ctx->work_list, &ctx->work_head);
> +               if (!node) {
> +                       /* a producer is mid-push, wait for it to link */
> +                       cpu_relax();
> +                       continue;
> +               }
> +               *tail = node;
> +               tail = &node->next;
> +       }
> +       *tail = NULL;
> +       __io_fallback_tw(first, false);
>  }
>
>  static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events,
> @@ -302,22 +286,23 @@ static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events,
>         return false;
>  }
>
> -static int __io_run_local_work_loop(struct llist_node **node,
> +static int __io_run_local_work_loop(struct io_ring_ctx *ctx,
>                                     io_tw_token_t tw,
>                                     int events)
>  {
>         int ret = 0;
>
> -       while (*node) {
> -               struct llist_node *next = (*node)->next;
> -               struct io_kiocb *req = container_of(*node, struct io_kiocb,
> -                                                   io_task_work.node);
> +       while (ret < events) {
> +               struct llist_node *node = mpscq_pop(&ctx->work_list, &ctx->work_head);
> +               struct io_kiocb *req;
> +
> +               if (!node)
> +                       break;
> +               req = container_of(node, struct io_kiocb, io_task_work.node);
>                 INDIRECT_CALL_2(req->io_task_work.func,
>                                 io_poll_task_func, io_req_rw_complete,
>                                 (struct io_tw_req){req}, tw);
> -               *node = next;
> -               if (++ret >= events)
> -                       break;
> +               ret++;
>         }
>
>         return ret;
> @@ -326,7 +311,6 @@ static int __io_run_local_work_loop(struct llist_node **node,
>  static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,
>                                int min_events, int max_events)
>  {
> -       struct llist_node *node;
>         unsigned int loops = 0;
>         int ret = 0;
>
> @@ -335,24 +319,21 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,
>         if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
>                 atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
>  again:
> -       tw.cancel = io_should_terminate_tw(ctx);
> -       min_events -= ret;
> -       ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);
> -       if (ctx->retry_llist.first)
> -               goto retry_done;
> -
>         /*
> -        * llists are in reverse order, flip it back the right way before
> -        * running the pending items.
> +        * If the last loop made no progress while work is still pending,
> +        * a producer has published a node but hasn't linked it into the
> +        * queue yet (see mpscq_pop()). Give it a chance to finish rather
> +        * than spinning on the queue.
>          */
> -       node = llist_reverse_order(llist_del_all(&ctx->work_llist));
> -       ret += __io_run_local_work_loop(&node, tw, max_events - ret);
> -       ctx->retry_llist.first = node;
> +       if (unlikely(loops && !ret))
> +               cond_resched();
> +       tw.cancel = io_should_terminate_tw(ctx);
> +       min_events -= ret;
> +       ret = __io_run_local_work_loop(ctx, tw, max_events);
>         loops++;
>
>         if (io_run_local_work_continue(ctx, ret, min_events))
>                 goto again;
> -retry_done:
>         io_submit_flush_completions(ctx);
>         if (io_run_local_work_continue(ctx, ret, min_events))
>                 goto again;
> diff --git a/io_uring/tw.h b/io_uring/tw.h
> index 415e330fabde..f42db5fdbded 100644
> --- a/io_uring/tw.h
> +++ b/io_uring/tw.h
> @@ -6,6 +6,8 @@
>  #include <linux/percpu-refcount.h>
>  #include <linux/io_uring_types.h>
>
> +#include "mpscq.h"
> +
>  #define IO_LOCAL_TW_DEFAULT_MAX                20
>
>  /*
> @@ -89,7 +91,7 @@ static inline int io_run_task_work(void)
>
>  static inline bool io_local_work_pending(struct io_ring_ctx *ctx)
>  {
> -       return !llist_empty(&ctx->work_llist) || !llist_empty(&ctx->retry_llist);
> +       return !mpscq_empty(&ctx->work_list);
>  }
>
>  static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
> diff --git a/io_uring/wait.c b/io_uring/wait.c
> index ec01e78a216d..c5fc34d2ce97 100644
> --- a/io_uring/wait.c
> +++ b/io_uring/wait.c
> @@ -98,7 +98,7 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
>         if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
>                 atomic_set(&ctx->cq_wait_nr, 1);
>                 smp_mb();
> -               if (!llist_empty(&ctx->work_llist))
> +               if (io_local_work_pending(ctx))
>                         goto out_wake;
>         }
>
> diff --git a/io_uring/wait.h b/io_uring/wait.h
> index a4274b137f81..6d494297e1ce 100644
> --- a/io_uring/wait.h
> +++ b/io_uring/wait.h
> @@ -5,12 +5,14 @@
>  #include <linux/io_uring_types.h>
>
>  /*
> - * No waiters. It's larger than any valid value of the tw counter
> - * so that tests against ->cq_wait_nr would fail and skip wake_up().
> + * ->cq_wait_nr is armed with the number of lazy task_work adds the waiter
> + * still needs, and counted down by the add side, with the add reaching zero
> + * issuing the (single) wake up for this wait cycle. Zero and below means no
> + * wake up is to be issued: IO_CQ_WAKE_INIT when no task is waiting (also
> + * what a forced wake up resets it to when claiming one), zero once the
> + * countdown has fired.
>   */
>  #define IO_CQ_WAKE_INIT                (-1U)

Since cq_wait_nr is now used as a signed value, would it make sense to
drop the U here?

Best,
Caleb

> -/* Forced wake up if there is a waiter regardless of ->cq_wait_nr */
> -#define IO_CQ_WAKE_FORCE       (IO_CQ_WAKE_INIT >> 1)
>
>  struct ext_arg {
>         size_t argsz;
> --
> 2.53.0
>