[PATCH] fuse: select alternative io_uring queue if local queue is full

fuse_uring_task_to_queue() always maps requests to the queue for
task_cpu(current). A single-threaded submitter with high queue depth
could leave other per-CPU queues idle while one queue handles all
traffic.

Prefer the local queue when it has available ring entries. If the local
queue is full, try other queues on the same NUMA node before falling
back to use the local queue.

Signed-off-by: Li Wang <[email protected]>
Tested-by: Jinxu Du <[email protected]>
---
Benchmark results

Machine: dual 64-core XEON GOLD 6548Y CPUs, 512GB RAM, 4TB NVME
Program: fio ioengine=libaio numjobs=1 direct=1 iodepth=256

Pattern	                  Baseline    Patched    Diff
read bs=512K (MB/s)       1146        1910       66.67%
write bs=512K (MB/s)      1929        2486       28.88%
randread bs=4K (KIOPS)    21.5        29.7       38.14%
randwrite bs=4K (KIOPS)   92.1        134        45.49%

 fs/fuse/dev_uring.c | 47 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index 77c8cec43d9c..c36438e79f8d 100644
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -11,6 +11,7 @@
 
 #include <linux/fs.h>
 #include <linux/io_uring/cmd.h>
+#include <linux/topology.h>
 
 static bool __read_mostly enable_uring;
 module_param(enable_uring, bool, 0644);
@@ -1318,10 +1319,46 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw)
 	}
 }
 
+static inline bool fuse_uring_queue_has_avail_ent(struct fuse_ring_queue *queue)
+{
+	bool has;
+
+	if (!queue)
+		return false;
+
+	spin_lock(&queue->lock);
+	has = !list_empty(&queue->ent_avail_queue);
+	spin_unlock(&queue->lock);
+
+	return has;
+}
+
+static struct fuse_ring_queue *
+fuse_uring_find_avail_queue(struct fuse_ring *ring, int node, unsigned int skip)
+{
+	unsigned int i;
+	struct fuse_ring_queue *queue;
+
+	for (i = 0; i < ring->nr_queues; i++) {
+
+		if (i == skip)
+			continue;
+		if (node >= 0 && cpu_to_node(i) != node)
+			continue;
+
+		queue = ring->queues[i];
+		if (fuse_uring_queue_has_avail_ent(queue))
+			return queue;
+	}
+
+	return NULL;
+}
+
 static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring)
 {
 	unsigned int qid;
 	struct fuse_ring_queue *queue;
+	int node;
 
 	qid = task_cpu(current);
 
@@ -1331,7 +1368,15 @@ static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring)
 		qid = 0;
 
 	queue = ring->queues[qid];
-	WARN_ONCE(!queue, "Missing queue for qid %d\n", qid);
+	if (fuse_uring_queue_has_avail_ent(queue))
+		return queue;
+
+	node = cpu_to_node(qid);
+	queue = fuse_uring_find_avail_queue(ring, node, qid);
+	if (!queue) {
+		queue = ring->queues[qid];
+		WARN_ONCE(!queue, "Missing queue for qid %d\n", qid);
+	}
 
 	return queue;
 }
-- 
2.34.1