git: a6847ba19592 - main - taskqueue: Move wakeup() out of tq_mutex

The branch main has been updated by mav:

URL: https://cgit.FreeBSD.org/src/commit/?id=a6847ba19592c0ac60326e6e1b421cf6e344016b

commit a6847ba19592c0ac60326e6e1b421cf6e344016b
Author:     Alexander Motin <[email protected]>
AuthorDate: 2026-08-10 20:18:34 +0000
Commit:     Alexander Motin <[email protected]>
CommitDate: 2026-08-10 20:42:20 +0000

    taskqueue: Move wakeup() out of tq_mutex
    
    taskqueue KPI require wakeup() to be called for each completed
    task.  With everything else there heavily optimized over the years,
    even when doing nothing this wakeup()'s lock/unlock is significant.
    Since no external taskqueue consumer can depend on the tq_mutex,
    we can move the wakeup() out of it.  It creates some complications
    for internal waiters, but those should be much more rare, and can
    be handled with separate locked wakeups on demand.
    
    My tests of taskqueue-intensive ZFS RAIDZ writes on 64-core system
    show performance improvement from this change ~4%, while same time
    reducing CPU usage by several percent due to lower lock contention,
    confirmed by CPU profiler.
---
 sys/kern/subr_taskqueue.c | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/sys/kern/subr_taskqueue.c b/sys/kern/subr_taskqueue.c
index b337aa83d69d..f1f6f18bccdc 100644
--- a/sys/kern/subr_taskqueue.c
+++ b/sys/kern/subr_taskqueue.c
@@ -58,6 +58,7 @@ struct taskqueue_busy {
 	struct task		*tb_running;
 	u_int			 tb_seq;
 	bool			 tb_canceling;
+	bool			 tb_wanted;
 	LIST_ENTRY(taskqueue_busy) tb_link;
 };
 
@@ -137,6 +138,15 @@ TQ_SLEEP(struct taskqueue *tq, void *p, const char *wm)
 	return (msleep(p, &tq->tq_mutex, 0, wm, 0));
 }
 
+static __inline int
+TQ_SLEEP_BUSY(struct taskqueue *tq, struct taskqueue_busy *tb, const char *wm)
+{
+
+	TQ_ASSERT_LOCKED(tq);
+	tb->tb_wanted = true;
+	return (TQ_SLEEP(tq, tb, wm));
+}
+
 static struct taskqueue_busy *
 task_get_busy(struct taskqueue *queue, struct task *task)
 {
@@ -462,7 +472,7 @@ taskqueue_drain_tq_active(struct taskqueue *queue)
 restart:
 	LIST_FOREACH(tb, &queue->tq_active, tb_link) {
 		if ((int)(tb->tb_seq - seq) <= 0) {
-			TQ_SLEEP(queue, tb->tb_running, "tq_adrain");
+			TQ_SLEEP_BUSY(queue, tb, "tq_adrain");
 			goto restart;
 		}
 	}
@@ -506,6 +516,7 @@ taskqueue_run_locked(struct taskqueue *queue)
 	KASSERT(queue != NULL, ("tq is NULL"));
 	TQ_ASSERT_LOCKED(queue);
 	tb.tb_running = NULL;
+	tb.tb_wanted = false;
 	LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link);
 
 	epochtasks = 0;
@@ -534,8 +545,13 @@ taskqueue_run_locked(struct taskqueue *queue)
 			epochtasks = 0;
 		}
 
-		TQ_LOCK(queue);
 		wakeup(task);
+
+		TQ_LOCK(queue);
+		if (__predict_false(tb.tb_wanted)) {
+			tb.tb_wanted = false;
+			wakeup(&tb);
+		}
 	}
 	if (epochtasks > 0)
 		NET_EPOCH_EXIT(et);
@@ -628,13 +644,20 @@ taskqueue_cancel_timeout(struct taskqueue *queue,
 void
 taskqueue_drain(struct taskqueue *queue, struct task *task)
 {
+	struct taskqueue_busy *tb;
 
 	if (!queue->tq_spin)
 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__);
 
 	TQ_LOCK(queue);
-	while (task->ta_pending != 0 || task_get_busy(queue, task) != NULL)
-		TQ_SLEEP(queue, task, "tq_drain");
+	for (;;) {
+		if (task->ta_pending != 0)
+			TQ_SLEEP(queue, task, "tq_drain");
+		else if ((tb = task_get_busy(queue, task)) != NULL)
+			TQ_SLEEP_BUSY(queue, tb, "tq_drain");
+		else
+			break;
+	}
 	TQ_UNLOCK(queue);
 }