Re: powerpc64/pmap.c trouble report

Is the large (greater than 2^32) value of pr_nget below
exceptional? My crashes only happen for long-running
heavy workloads so a big value seems plausible but
maybe there is some limit I'm supposed to reconfigure
for such workloads?

panic: pmap_enter: failed to allocate pted
Stopped at      panic+0x134:    ori r0,r0,0x0
    TID    PID    UID     PRFLAGS     PFLAGS  CPU  COMMAND

 234577  13111   8889  0x18000001          0    3  go

 491287  13050   8889   0xa001007          0    2  asm
 331460  14455   8889  0x1a000003          0    5  asm
  66317  80047   8889  0x1a000003          0    1  compile
 331134  57997   8889  0x1a000003  0x4000000    0  go
 240615  91938   8889  0x1a000003  0x4000000    6  go
*247303  89969   8889  0x1a000003  0x4000000    4K go
  28756  33586      0     0x14000      0x200    7  pagedaemon
panic+0x134

pmap_enter+0x218

uvm_km_kmemalloc_pla+0x1f4
uvm_uarea_alloc+0x70

thread_fork+0xd8

sys___tfork+0xc4
syscall+0x530

trap+0x5dc

trapagain+0x4
--- syscall (number 8) ---
End of kernel: 0x457c99db0 lr 0x4bddd2604


ddb{4}>show struct pool pmap_pted_pool

struct pmap_pted_pool at 0x1089a98 (424 bytes) {pr_lock =3D {prl_mtx =3D {m=
tx_owner
 =3D (void *)0x0, mtx_wantipl =3D 7, mtx_oldipl =3D 0}, prl_rwlock =3D {rwl=
_owner =3D 0, r
wl_name =3D (const unsigned char *)0x700000000}}, pr_lock_ops =3D (const po=
ol_lock_

ops *)0xfc4950, pr_poollist =3D {sqe_next =3D (struct pool *)0x10898f0}, pr=
_emptypa
ges =3D {tqh_first =3D (struct pool_page_header *)0x0, tqh_last =3D 0x1089a=
b8}, pr_fu
llpages =3D {tqh_first =3D (struct pool_page_header *)0xc00000003e3b2f90, t=
qh_last =3D
 0xc00000014b344f90}, pr_partpages =3D {tqh_first =3D (struct pool_page_hea=
der *)0x
c000000140250f90, tqh_last =3D 0xc000000146821f90}, pr_curpage =3D (struct =
pool_pag
e_header *)0x0, pr_size =3D 56, pr_minitems =3D 20, pr_minpages =3D 1, pr_m=
axpages =3D 8
, pr_npages =3D 4632, pr_itemsperpage =3D 71, pr_slack =3D 0, pr_nitems =3D=
 328872, pr_
nout =3D 328870, pr_hardlimit =3D 4294967295, pr_serial =3D 14, pr_pgsize =
=3D 4096, pr_
pgmask =3D 18446744073709547520, pr_alloc =3D (struct pool_allocator *)0xfe=
9b40, pr
_wchan =3D (const unsigned char *)0xeb08e4, pr_flags =3D 0, pr_ipl =3D 7, p=
r_phtree =3D
 {rbh_root =3D {rbt_root =3D (struct rb_entry *)0x0}}, pr_cache =3D (struct=
 cpumem *)
0x0, pr_cache_magic =3D [0,0], pr_cache_lock =3D {prl_mtx =3D {mtx_owner =
=3D (void *)0x
0, mtx_wantipl =3D 0, mtx_oldipl =3D 0}, prl_rwlock =3D {rwl_owner =3D 0, r=
wl_name =3D (c
onst unsigned char *)0x0}}, pr_cache_lists =3D {tqh_first =3D (struct pool_=
cache_it
em *)0x0, tqh_last =3D 0x0}, pr_cache_nitems =3D 0, pr_cache_items =3D 0, p=
r_cache_co
ntention =3D 0, pr_cache_contention_prev =3D 0, pr_cache_timestamp =3D 0, p=
r_cache_ng
c =3D 0, pr_cache_nout =3D 0, pr_align =3D 8, pr_maxcolors =3D 2, pr_phoffs=
et =3D 3984, p
r_hardlimit_warning =3D (const unsigned char *)0x0, pr_hardlimit_ratecap =
=3D {tv_se

c =3D 0, tv_usec =3D 0}, pr_hardlimit_warning_last =3D {tv_sec =3D 0, tv_us=
ec =3D 0}, pr_
requests_lock =3D {prl_mtx =3D {mtx_owner =3D (void *)0x0, mtx_wantipl =3D =
7, mtx_oldip
l =3D 0}, prl_rwlock =3D {rwl_owner =3D 0, rwl_name =3D (const unsigned cha=
r *)0x700000
000}}, pr_requests =3D {tqh_first =3D (struct pool_request *)0x0, tqh_last =
=3D 0x1089
be8}, pr_requesting =3D 0, pr_nget =3D 5523479116, pr_nfail =3D 13, pr_nput=
 =3D 5523150
246, pr_npagealloc =3D 1387529, pr_npagefree =3D 1382897, pr_hiwat =3D 1007=
2, pr_nidl
e =3D 0, pr_crange =3D (const kmem_pa_mode *)0xfc60b0}


ddb{4}>show struct pool pmap_vp_pool

struct pmap_vp_pool at 0x10898f0 (424 bytes) {pr_lock =3D {prl_mtx =3D {mtx=
_owner =3D
 (void *)0x0, mtx_wantipl =3D 7, mtx_oldipl =3D 7}, prl_rwlock =3D {rwl_own=
er =3D 0, rw
l_name =3D (const unsigned char *)0x700000007}}, pr_lock_ops =3D (const poo=
l_lock_o

ps *)0xfc4950, pr_poollist =3D {sqe_next =3D (struct pool *)0x1089c40}, pr_=
emptypag
es =3D {tqh_first =3D (struct pool_page_header *)0xc0000001449f7af0, tqh_la=
st =3D 0xc
000000142a861d8}, pr_fullpages =3D {tqh_first =3D (struct pool_page_header =
*)0xc000
00003e3bbd20, tqh_last =3D 0xc00000014d3f9248}, pr_partpages =3D {tqh_first=
 =3D (stru
ct pool_page_header *)0xc00000003c3e9250, tqh_last =3D 0xc00000014a4e4d28},=
 pr_cu
rpage =3D (struct pool_page_header *)0xc00000014a4e4d28, pr_size =3D 2048, =
pr_minit
ems =3D 10, pr_minpages =3D 5, pr_maxpages =3D 8, pr_npages =3D 17150, pr_i=
temsperpage =3D
 2, pr_slack =3D 0, pr_nitems =3D 34300, pr_nout =3D 34255, pr_hardlimit =
=3D 4294967295
, pr_serial =3D 13, pr_pgsize =3D 4096, pr_pgmask =3D 18446744073709547520,=
 pr_alloc =3D

 (struct pool_allocator *)0xfe9b40, pr_wchan =3D (const unsigned char *)0xe=
e10ef,
 pr_flags =3D 0, pr_ipl =3D 7, pr_phtree =3D {rbh_root =3D {rbt_root =3D (s=
truct rb_entry
 *)0xc00000013f618a08}}, pr_cache =3D (struct cpumem *)0x0, pr_cache_magic =
=3D [0,0
], pr_cache_lock =3D {prl_mtx =3D {mtx_owner =3D (void *)0x0, mtx_wantipl =
=3D 0, mtx_ol
dipl =3D 0}, prl_rwlock =3D {rwl_owner =3D 0, rwl_name =3D (const unsigned =
char *)0x0}}
, pr_cache_lists =3D {tqh_first =3D (struct pool_cache_item *)0x0, tqh_last=
 =3D 0x0},
 pr_cache_nitems =3D 0, pr_cache_items =3D 0, pr_cache_contention =3D 0, pr=
_cache_con
tention_prev =3D 0, pr_cache_timestamp =3D 0, pr_cache_ngc =3D 0, pr_cache_=
nout =3D 0, p
r_align =3D 8, pr_maxcolors =3D 1, pr_phoffset =3D 0, pr_hardlimit_warning =
=3D (const u
nsigned char *)0x0, pr_hardlimit_ratecap =3D {tv_sec =3D 0, tv_usec =3D 0},=
 pr_hardli

mit_warning_last =3D {tv_sec =3D 0, tv_usec =3D 0}, pr_requests_lock =3D {p=
rl_mtx =3D {mt
x_owner =3D (void *)0x0, mtx_wantipl =3D 7, mtx_oldipl =3D 0}, prl_rwlock =
=3D {rwl_owne
r =3D 0, rwl_name =3D (const unsigned char *)0x700000000}}, pr_requests =3D=
 {tqh_firs
t =3D (struct pool_request *)0x0, tqh_last =3D 0x1089a40}, pr_requesting =
=3D 0, pr_ng

et =3D 147054103, pr_nfail =3D 0, pr_nput =3D 147019848, pr_npagealloc =3D =
6817420, pr_
npagefree =3D 6800270, pr_hiwat =3D 20556, pr_nidle =3D 17, pr_crange =3D (=
const kmem_p

a_mode *)0xfc60b0}


ddb{4}> show struct pool pmap_pmap_pool

struct pmap_pmap_pool at 0x1089c40 (424 bytes) {pr_lock =3D {prl_mtx =3D {m=
tx_owner
 =3D (void *)0x0, mtx_wantipl =3D 7, mtx_oldipl =3D 0}, prl_rwlock =3D {rwl=
_owner =3D 0, r

wl_name =3D (const unsigned char *)0x700000000}}, pr_lock_ops =3D (const po=
ol_lock_
ops *)0xfc4950, pr_poollist =3D {sqe_next =3D (struct pool *)0x104b5c0}, pr=
_emptypa

ges =3D {tqh_first =3D (struct pool_page_header *)0xc00000014ef78f90, tqh_l=
ast =3D 0x
c00000014ef78f90}, pr_fullpages =3D {tqh_first =3D (struct pool_page_header=
 *)0x0, t
qh_last =3D 0x1089c70}, pr_partpages =3D {tqh_first =3D (struct pool_page_h=
eader *)0x
c00000003e3aef90, tqh_last =3D 0xc000000149c6ef90}, pr_curpage =3D (struct =
pool_pag
e_header *)0xc00000003e3aef90, pr_size =3D 48, pr_minitems =3D 2, pr_minpag=
es =3D 1, p
r_maxpages =3D 8, pr_npages =3D 3, pr_itemsperpage =3D 83, pr_slack =3D 0, =
pr_nitems =3D 2
49, pr_nout =3D 138, pr_hardlimit =3D 4294967295, pr_serial =3D 12, pr_pgsi=
ze =3D 4096,
 pr_pgmask =3D 18446744073709547520, pr_alloc =3D (struct pool_allocator *)=
0xfe9b40
, pr_wchan =3D (const unsigned char *)0xe91044, pr_flags =3D 0, pr_ipl =3D =
7, pr_phtr
ee =3D {rbh_root =3D {rbt_root =3D (struct rb_entry *)0x0}}, pr_cache =3D (=
struct cpume
m *)0x0, pr_cache_magic =3D [0,0], pr_cache_lock =3D {prl_mtx =3D {mtx_owne=
r =3D (void *
)0x0, mtx_wantipl =3D 0, mtx_oldipl =3D 0}, prl_rwlock =3D {rwl_owner =3D 0=
, rwl_name =3D
 (const unsigned char *)0x0}}, pr_cache_lists =3D {tqh_first =3D (struct po=
ol_cache
_item *)0x0, tqh_last =3D 0x0}, pr_cache_nitems =3D 0, pr_cache_items =3D 0=
, pr_cache
_contention =3D 0, pr_cache_contention_prev =3D 0, pr_cache_timestamp =3D 0=
, pr_cache
_ngc =3D 0, pr_cache_nout =3D 0, pr_align =3D 8, pr_maxcolors =3D 1, pr_pho=
ffset =3D 3984
, pr_hardlimit_warning =3D (const unsigned char *)0x0, pr_hardlimit_ratecap=
 =3D {tv
_sec =3D 0, tv_usec =3D 0}, pr_hardlimit_warning_last =3D {tv_sec =3D 0, tv=
_usec =3D 0}, p
r_requests_lock =3D {prl_mtx =3D {mtx_owner =3D (void *)0x0, mtx_wantipl =
=3D 7, mtx_old
ipl =3D 0}, prl_rwlock =3D {rwl_owner =3D 0, rwl_name =3D (const unsigned c=
har *)0x7000
00000}}, pr_requests =3D {tqh_first =3D (struct pool_request *)0x0, tqh_las=
t =3D 0x10
89d90}, pr_requesting =3D 0, pr_nget =3D 1113457, pr_nfail =3D 0, pr_nput =
=3D 1113319, p
r_npagealloc =3D 3, pr_npagefree =3D 0, pr_hiwat =3D 3, pr_nidle =3D 1, pr_=
crange =3D (co
nst kmem_pa_mode *)0xfc60b0}

ddb{4}>

On Wed, Jun 5, 2024 at 12:16=E2=80=AFPM Miod Vallat <[email protected]> wrote:
>
> > There's a corruption...
> >
> > > ddb{7}> show panic
> > >  cpu6: kernel diagnostic assertion "((flags & PGO_LOCKED) !=3D 0 && r=
w_lock_held(
> > > uobj->vmobjlock)) || (flags & PGO_LOCKED) =3D=3D 0" failed: file "/sy=
s/uvm/uvm_vnod
> > > e.c", line 953
> > >
> > > *cpu7: assertwaitok: non-zero mutex count: 1
> > > ddb{7}> trace
> > > panic+0x134
> > > assertwaitok+0xf8
> > > mi_switch+0x5c
> > > sleep_finish+0x160
> > > rw_enter+0x1cc
> > > vm_map_lock_read_ln+0x38
> > > uvmfault_lookup+0x114
> > > uvm_fault_check+0x68
> > > uvm_fault+0x12c
> > > trap+0x7a4
> > > trapagain+0x4
> > > --- trap (type 0x300) ---
> > > phtree_RBT_COMPARE+0x28
> > > pool_do_put+0x94
> > > pool_put+0x94
> >            ^^^^
> > ...inside this pool.  Which of the 3 is it?  Can someone with a ppc64
> > figure out?
>
> It's pmap_vp_pool.