On 2/8/2026 10:59 am, Tejun Heo wrote:
Hello Tejun,
Thanks for taking a look into this.
> Hello,
>
> On Tue, Jul 28, 2026 at 02:55:19PM +0800, Imran Khan wrote:
>> Nearly every subsystem defers work to workqueues, and their state can
>> already be observed on a live system, just not at a granularity that is
>> convenient to consume. sysrq dumps every workqueue and pool to the kernel
>> log in a fixed format, with no way to select or aggregate; the WQ_SYSFS
>> interface only covers workqueues that ask for it and exposes attributes
>> rather than runtime state; and the drgn scripts under tools/workqueue/ need
>> a debuginfo-equipped userspace and read the state from the outside, without
>> the locks that protect it.
>>
>> This series adds BPF iterators for workqueues, worker pools and pending
>> work items, so that this state can be walked from inside the kernel, under
>> the right locking, with filtering and aggregation done in place and only
>> the interesting part copied to userspace.
>
> I'm not necessarily against it but what are the use cases here? If for
> debugging, isn't drgn + hooking into tracepoints mostly enough? Can you give
> concrete examples where bpf iterators are essential?
>
The main use case is being able to peek at workqueue state right when a problem
(where checking workqueue state makes sense) is detected.
One very common case for us is getting RDS TX timeouts because of the the stuck work items.
At the moment I don't have a system where I can both use the patched kernel and
reproduce the Tx timeout issue, so I have tried to explain the same using block
layer and dm-device as affected subsystem.
The bpf program shown below measures block_bio_queue -> block_bio_complete on the dm
device:
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
char _license[] SEC("license") = "GPL";
#define MIN_LAT_MS 5
#define HZ_HINT 1000
void bpf_rcu_read_lock(void) __ksym;
void bpf_rcu_read_unlock(void) __ksym;
extern int bpf_iter_workqueue_new(struct bpf_iter_workqueue *it) __weak __ksym;
extern struct workqueue_struct *bpf_iter_workqueue_next(struct bpf_iter_workqueue *it) __weak __ksym;
extern void bpf_iter_workqueue_destroy(struct bpf_iter_workqueue *it) __weak __ksym;
extern int bpf_iter_worker_pool_new(struct bpf_iter_worker_pool *it) __weak __ksym;
extern struct worker_pool *bpf_iter_worker_pool_next(struct bpf_iter_worker_pool *it) __weak __ksym;
extern void bpf_iter_worker_pool_destroy(struct bpf_iter_worker_pool *it) __weak __ksym;
SEC("iter/workqueue_pending_work")
int dump_pending(struct bpf_iter__workqueue_pending_work *ctx)
{
struct seq_file *seq = ctx->meta->seq;
struct wq_pending_work_info *info = ctx->info;
if (!info)
return 0;
BPF_SEQ_PRINTF(seq, "pool=%-3llu work=0x%llx func=%pS\n",
info->pool_id, info->work, (void *)info->func);
return 0;
}
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 65536);
__type(key, __u64);
__type(value, __u64);
} inflight SEC(".maps");
static __always_inline int is_dm_bio(struct bio *bio)
{
struct gendisk *disk;
char name[8] = {};
disk = BPF_CORE_READ(bio, bi_bdev, bd_disk);
if (!disk)
return 0;
bpf_probe_read_kernel_str(name, sizeof(name), disk->disk_name);
return name[0] == 'd' && name[1] == 'm' && name[2] == '-';
}
__u64 pending_report_us;
__u32 nr_slow;
__u32 nr_reports;
static __always_inline void dump_wq_state(__u64 lat_us)
{
struct bpf_iter_worker_pool pit;
struct bpf_iter_workqueue wit;
struct workqueue_struct *wq;
struct worker_pool *pool;
__u64 now_j = bpf_jiffies64();
__u32 this_cpu = bpf_get_smp_processor_id();
bpf_printk("STALL: dm-crypt bio took %llu us; sampling workqueue state now (cpu%d)",
lat_us, this_cpu);
bpf_rcu_read_lock();
bpf_iter_worker_pool_new(&pit);
while ((pool = bpf_iter_worker_pool_next(&pit))) {
__u64 stale_ms = (now_j - pool->last_progress_ts) * 1000 / HZ_HINT;
int has_work = pool->worklist.next != &pool->worklist;
int is_this_cpu = pool->cpu == (int)this_cpu;
if (stale_ms > 60000)
continue;
if (!has_work && !is_this_cpu)
continue;
bpf_printk(" %s pool %d cpu %d: no progress for %llu ms, work_waiting=%d, running=%d idle=%d/%d",
is_this_cpu ? "->" : " ",
pool->id, pool->cpu, stale_ms, has_work,
pool->nr_running, pool->nr_idle, pool->nr_workers);
}
bpf_iter_worker_pool_destroy(&pit);
bpf_iter_workqueue_new(&wit);
while ((wq = bpf_iter_workqueue_next(&wit))) {
char name[10];
bpf_probe_read_kernel_str(name, sizeof(name), wq->name);
if (name[0] != 'k' || name[1] != 'c' || name[2] != 'r')
continue;
bpf_printk(" wq %s: max_active=%d flags=0x%x",
name, wq->max_active, wq->flags);
}
bpf_iter_workqueue_destroy(&wit);
bpf_rcu_read_unlock();
}
SEC("tp_btf/block_bio_queue")
int BPF_PROG(on_bio_queue, struct bio *bio)
{
__u64 key = (__u64)(long)bio;
__u64 now = bpf_ktime_get_ns();
__u64 lat_us;
if (!is_dm_bio(bio))
return 0;
bpf_map_update_elem(&inflight, &key, &now, BPF_ANY);
lat_us = pending_report_us;
if (lat_us) {
pending_report_us = 0;
nr_reports++;
dump_wq_state(lat_us);
}
return 0;
}
SEC("tp_btf/block_bio_complete")
int BPF_PROG(on_bio_complete, struct request_queue *q, struct bio *bio)
{
__u64 key = (__u64)(long)bio;
__u64 *tsp, lat;
tsp = bpf_map_lookup_elem(&inflight, &key);
if (!tsp)
return 0;
lat = bpf_ktime_get_ns() - *tsp;
bpf_map_delete_elem(&inflight, &key);
if (lat < (__u64)MIN_LAT_MS * 1000000ULL)
return 0;
nr_slow++;
pending_report_us = lat / 1000;
return 0;
}
The above bpf programs can be attached using following 2 commands:
bpftool prog loadall sample.bpf.o /sys/fs/bpf/dm autoattach
bpftool iter pin sample.bpf.o /sys/fs/bpf/dm_pending
(sample.bpf.c is the name of file containing above code)
When a bio exceeds the threshold it walks the open-coded
worker_pool and workqueue iterators and prints what it finds:
fio-2409 [003] 1246.736278: STALL: dm-crypt bio took 950434 us; sampling workqueue state now (cpu3)
fio-2409 [003] 1246.736282: pool 10 cpu 2: no progress for 935 ms, work_waiting=1, running=0 idle=3/3
fio-2409 [003] 1246.736285: pool 22 cpu 5: no progress for 934 ms, work_waiting=1, running=0 idle=3/3
fio-2409 [003] 1246.736299: wq kcryptd_i: max_active=1 flags=0x148
fio-2409 [003] 1246.736300: wq kcryptd-2: max_active=1 flags=0x168
In order to reproduce the issue easily I had RT threads hogging the CPUs and
thus preventing timely run of workers.
Another limitation with drgn and traces is that often the production systems
don't have drgn and/or debuginfo installed and sometimes the systems are itself
in such a bad shape that running drgn becomes challenging.
For such cases as well, a quick look into the bpffs to find pending works
(like shown below) helps:
cat /sys/fs/bpf/dm_pending
pool=2 work=0xffff9987afc27c48 func=delayed_vfree_work+0x0/0x50
pool=6 work=0xffff9987afc67560 func=vmstat_update+0x0/0x50
pool=14 work=0xffff998484b39e20 func=kcryptd_crypt+0x0/0x310 [dm_crypt]
pool=22 work=0xffff9987afd67560 func=vmstat_update+0x0/0x50
pool=26 work=0xffff9987afda7560 func=vmstat_update+0x0/0x50
pool=30 work=0xffff9987afde7560 func=vmstat_update+0x0/0x50
If the issue happens randomly in short windows of few secs, collecting
the traces for long intervals and looking for data of that short window is
not easy.
These are the use cases/limitation I had in mind. Could you please let
me know your thoughts?
Thanks,
Imran
> Thanks.
>
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.