Re: powerpc64/pmap.c trouble report

Disregard this speculation about a possible 32bit int issue. I've
reproduced the panic with a smaller pr_nget.

On Thu, Jun 6, 2024 at 11:37=E2=80=AFPM Eric Grosse <[email protected]> wrot=
e:
>
> Is the large (greater than 2^32) value of pr_nget below
> exceptional? My crashes only happen for long-running
> heavy workloads so a big value seems plausible but
> maybe there is some limit I'm supposed to reconfigure
> for such workloads?
>
> panic: pmap_enter: failed to allocate pted
> Stopped at      panic+0x134:    ori r0,r0,0x0
>     TID    PID    UID     PRFLAGS     PFLAGS  CPU  COMMAND
>
>  234577  13111   8889  0x18000001          0    3  go
>
>  491287  13050   8889   0xa001007          0    2  asm
>  331460  14455   8889  0x1a000003          0    5  asm
>   66317  80047   8889  0x1a000003          0    1  compile
>  331134  57997   8889  0x1a000003  0x4000000    0  go
>  240615  91938   8889  0x1a000003  0x4000000    6  go
> *247303  89969   8889  0x1a000003  0x4000000    4K go
>   28756  33586      0     0x14000      0x200    7  pagedaemon
> panic+0x134
>
> pmap_enter+0x218
>
> uvm_km_kmemalloc_pla+0x1f4
> uvm_uarea_alloc+0x70
>
> thread_fork+0xd8
>
> sys___tfork+0xc4
> syscall+0x530
>
> trap+0x5dc
>
> trapagain+0x4
> --- syscall (number 8) ---
> End of kernel: 0x457c99db0 lr 0x4bddd2604
>
>
> ddb{4}>show struct pool pmap_pted_pool
>
> struct pmap_pted_pool at 0x1089a98 (424 bytes) {pr_lock =3D {prl_mtx =3D =
{mtx_owner
>  =3D (void *)0x0, mtx_wantipl =3D 7, mtx_oldipl =3D 0}, prl_rwlock =3D {r=
wl_owner =3D 0, r
> wl_name =3D (const unsigned char *)0x700000000}}, pr_lock_ops =3D (const =
pool_lock_
>
> ops *)0xfc4950, pr_poollist =3D {sqe_next =3D (struct pool *)0x10898f0}, =
pr_emptypa
> ges =3D {tqh_first =3D (struct pool_page_header *)0x0, tqh_last =3D 0x108=
9ab8}, pr_fu
> llpages =3D {tqh_first =3D (struct pool_page_header *)0xc00000003e3b2f90,=
 tqh_last =3D
>  0xc00000014b344f90}, pr_partpages =3D {tqh_first =3D (struct pool_page_h=
eader *)0x
> c000000140250f90, tqh_last =3D 0xc000000146821f90}, pr_curpage =3D (struc=
t pool_pag
> e_header *)0x0, pr_size =3D 56, pr_minitems =3D 20, pr_minpages =3D 1, pr=
_maxpages =3D 8
> , pr_npages =3D 4632, pr_itemsperpage =3D 71, pr_slack =3D 0, pr_nitems =
=3D 328872, pr_
> nout =3D 328870, pr_hardlimit =3D 4294967295, pr_serial =3D 14, pr_pgsize=
 =3D 4096, pr_
> pgmask =3D 18446744073709547520, pr_alloc =3D (struct pool_allocator *)0x=
fe9b40, pr
> _wchan =3D (const unsigned char *)0xeb08e4, pr_flags =3D 0, pr_ipl =3D 7,=
 pr_phtree =3D
>  {rbh_root =3D {rbt_root =3D (struct rb_entry *)0x0}}, pr_cache =3D (stru=
ct cpumem *)
> 0x0, pr_cache_magic =3D [0,0], pr_cache_lock =3D {prl_mtx =3D {mtx_owner =
=3D (void *)0x
> 0, mtx_wantipl =3D 0, mtx_oldipl =3D 0}, prl_rwlock =3D {rwl_owner =3D 0,=
 rwl_name =3D (c
> onst unsigned char *)0x0}}, pr_cache_lists =3D {tqh_first =3D (struct poo=
l_cache_it
> em *)0x0, tqh_last =3D 0x0}, pr_cache_nitems =3D 0, pr_cache_items =3D 0,=
 pr_cache_co
> ntention =3D 0, pr_cache_contention_prev =3D 0, pr_cache_timestamp =3D 0,=
 pr_cache_ng
> c =3D 0, pr_cache_nout =3D 0, pr_align =3D 8, pr_maxcolors =3D 2, pr_phof=
fset =3D 3984, p
> r_hardlimit_warning =3D (const unsigned char *)0x0, pr_hardlimit_ratecap =
=3D {tv_se
>
> c =3D 0, tv_usec =3D 0}, pr_hardlimit_warning_last =3D {tv_sec =3D 0, tv_=
usec =3D 0}, pr_
> requests_lock =3D {prl_mtx =3D {mtx_owner =3D (void *)0x0, mtx_wantipl =
=3D 7, mtx_oldip
> l =3D 0}, prl_rwlock =3D {rwl_owner =3D 0, rwl_name =3D (const unsigned c=
har *)0x700000
> 000}}, pr_requests =3D {tqh_first =3D (struct pool_request *)0x0, tqh_las=
t =3D 0x1089
> be8}, pr_requesting =3D 0, pr_nget =3D 5523479116, pr_nfail =3D 13, pr_np=
ut =3D 5523150
> 246, pr_npagealloc =3D 1387529, pr_npagefree =3D 1382897, pr_hiwat =3D 10=
072, pr_nidl
> e =3D 0, pr_crange =3D (const kmem_pa_mode *)0xfc60b0}
>
>
> ddb{4}>show struct pool pmap_vp_pool
>
> struct pmap_vp_pool at 0x10898f0 (424 bytes) {pr_lock =3D {prl_mtx =3D {m=
tx_owner =3D
>  (void *)0x0, mtx_wantipl =3D 7, mtx_oldipl =3D 7}, prl_rwlock =3D {rwl_o=
wner =3D 0, rw
> l_name =3D (const unsigned char *)0x700000007}}, pr_lock_ops =3D (const p=
ool_lock_o
>
> ps *)0xfc4950, pr_poollist =3D {sqe_next =3D (struct pool *)0x1089c40}, p=
r_emptypag
> es =3D {tqh_first =3D (struct pool_page_header *)0xc0000001449f7af0, tqh_=
last =3D 0xc
> 000000142a861d8}, pr_fullpages =3D {tqh_first =3D (struct pool_page_heade=
r *)0xc000
> 00003e3bbd20, tqh_last =3D 0xc00000014d3f9248}, pr_partpages =3D {tqh_fir=
st =3D (stru
> ct pool_page_header *)0xc00000003c3e9250, tqh_last =3D 0xc00000014a4e4d28=
}, pr_cu
> rpage =3D (struct pool_page_header *)0xc00000014a4e4d28, pr_size =3D 2048=
, pr_minit
> ems =3D 10, pr_minpages =3D 5, pr_maxpages =3D 8, pr_npages =3D 17150, pr=
_itemsperpage =3D
>  2, pr_slack =3D 0, pr_nitems =3D 34300, pr_nout =3D 34255, pr_hardlimit =
=3D 4294967295
> , pr_serial =3D 13, pr_pgsize =3D 4096, pr_pgmask =3D 1844674407370954752=
0, pr_alloc =3D
>
>  (struct pool_allocator *)0xfe9b40, pr_wchan =3D (const unsigned char *)0=
xee10ef,
>  pr_flags =3D 0, pr_ipl =3D 7, pr_phtree =3D {rbh_root =3D {rbt_root =3D =
(struct rb_entry
>  *)0xc00000013f618a08}}, pr_cache =3D (struct cpumem *)0x0, pr_cache_magi=
c =3D [0,0
> ], pr_cache_lock =3D {prl_mtx =3D {mtx_owner =3D (void *)0x0, mtx_wantipl=
 =3D 0, mtx_ol
> dipl =3D 0}, prl_rwlock =3D {rwl_owner =3D 0, rwl_name =3D (const unsigne=
d char *)0x0}}
> , pr_cache_lists =3D {tqh_first =3D (struct pool_cache_item *)0x0, tqh_la=
st =3D 0x0},
>  pr_cache_nitems =3D 0, pr_cache_items =3D 0, pr_cache_contention =3D 0, =
pr_cache_con
> tention_prev =3D 0, pr_cache_timestamp =3D 0, pr_cache_ngc =3D 0, pr_cach=
e_nout =3D 0, p
> r_align =3D 8, pr_maxcolors =3D 1, pr_phoffset =3D 0, pr_hardlimit_warnin=
g =3D (const u
> nsigned char *)0x0, pr_hardlimit_ratecap =3D {tv_sec =3D 0, tv_usec =3D 0=
}, pr_hardli
>
> mit_warning_last =3D {tv_sec =3D 0, tv_usec =3D 0}, pr_requests_lock =3D =
{prl_mtx =3D {mt
> x_owner =3D (void *)0x0, mtx_wantipl =3D 7, mtx_oldipl =3D 0}, prl_rwlock=
 =3D {rwl_owne
> r =3D 0, rwl_name =3D (const unsigned char *)0x700000000}}, pr_requests =
=3D {tqh_firs
> t =3D (struct pool_request *)0x0, tqh_last =3D 0x1089a40}, pr_requesting =
=3D 0, pr_ng
>
> et =3D 147054103, pr_nfail =3D 0, pr_nput =3D 147019848, pr_npagealloc =
=3D 6817420, pr_
> npagefree =3D 6800270, pr_hiwat =3D 20556, pr_nidle =3D 17, pr_crange =3D=
 (const kmem_p
>
> a_mode *)0xfc60b0}
>
>
> ddb{4}> show struct pool pmap_pmap_pool
>
> struct pmap_pmap_pool at 0x1089c40 (424 bytes) {pr_lock =3D {prl_mtx =3D =
{mtx_owner
>  =3D (void *)0x0, mtx_wantipl =3D 7, mtx_oldipl =3D 0}, prl_rwlock =3D {r=
wl_owner =3D 0, r
>
> wl_name =3D (const unsigned char *)0x700000000}}, pr_lock_ops =3D (const =
pool_lock_
> ops *)0xfc4950, pr_poollist =3D {sqe_next =3D (struct pool *)0x104b5c0}, =
pr_emptypa
>
> ges =3D {tqh_first =3D (struct pool_page_header *)0xc00000014ef78f90, tqh=
_last =3D 0x
> c00000014ef78f90}, pr_fullpages =3D {tqh_first =3D (struct pool_page_head=
er *)0x0, t
> qh_last =3D 0x1089c70}, pr_partpages =3D {tqh_first =3D (struct pool_page=
_header *)0x
> c00000003e3aef90, tqh_last =3D 0xc000000149c6ef90}, pr_curpage =3D (struc=
t pool_pag
> e_header *)0xc00000003e3aef90, pr_size =3D 48, pr_minitems =3D 2, pr_minp=
ages =3D 1, p
> r_maxpages =3D 8, pr_npages =3D 3, pr_itemsperpage =3D 83, pr_slack =3D 0=
, pr_nitems =3D 2
> 49, pr_nout =3D 138, pr_hardlimit =3D 4294967295, pr_serial =3D 12, pr_pg=
size =3D 4096,
>  pr_pgmask =3D 18446744073709547520, pr_alloc =3D (struct pool_allocator =
*)0xfe9b40
> , pr_wchan =3D (const unsigned char *)0xe91044, pr_flags =3D 0, pr_ipl =
=3D 7, pr_phtr
> ee =3D {rbh_root =3D {rbt_root =3D (struct rb_entry *)0x0}}, pr_cache =3D=
 (struct cpume
> m *)0x0, pr_cache_magic =3D [0,0], pr_cache_lock =3D {prl_mtx =3D {mtx_ow=
ner =3D (void *
> )0x0, mtx_wantipl =3D 0, mtx_oldipl =3D 0}, prl_rwlock =3D {rwl_owner =3D=
 0, rwl_name =3D
>  (const unsigned char *)0x0}}, pr_cache_lists =3D {tqh_first =3D (struct =
pool_cache
> _item *)0x0, tqh_last =3D 0x0}, pr_cache_nitems =3D 0, pr_cache_items =3D=
 0, pr_cache
> _contention =3D 0, pr_cache_contention_prev =3D 0, pr_cache_timestamp =3D=
 0, pr_cache
> _ngc =3D 0, pr_cache_nout =3D 0, pr_align =3D 8, pr_maxcolors =3D 1, pr_p=
hoffset =3D 3984
> , pr_hardlimit_warning =3D (const unsigned char *)0x0, pr_hardlimit_ratec=
ap =3D {tv
> _sec =3D 0, tv_usec =3D 0}, pr_hardlimit_warning_last =3D {tv_sec =3D 0, =
tv_usec =3D 0}, p
> r_requests_lock =3D {prl_mtx =3D {mtx_owner =3D (void *)0x0, mtx_wantipl =
=3D 7, mtx_old
> ipl =3D 0}, prl_rwlock =3D {rwl_owner =3D 0, rwl_name =3D (const unsigned=
 char *)0x7000
> 00000}}, pr_requests =3D {tqh_first =3D (struct pool_request *)0x0, tqh_l=
ast =3D 0x10
> 89d90}, pr_requesting =3D 0, pr_nget =3D 1113457, pr_nfail =3D 0, pr_nput=
 =3D 1113319, p
> r_npagealloc =3D 3, pr_npagefree =3D 0, pr_hiwat =3D 3, pr_nidle =3D 1, p=
r_crange =3D (co
> nst kmem_pa_mode *)0xfc60b0}
>
> ddb{4}>
>
> On Wed, Jun 5, 2024 at 12:16=E2=80=AFPM Miod Vallat <[email protected]> wrot=
e:
> >
> > > There's a corruption...
> > >
> > > > ddb{7}> show panic
> > > >  cpu6: kernel diagnostic assertion "((flags & PGO_LOCKED) !=3D 0 &&=
 rw_lock_held(
> > > > uobj->vmobjlock)) || (flags & PGO_LOCKED) =3D=3D 0" failed: file "/=
sys/uvm/uvm_vnod
> > > > e.c", line 953
> > > >
> > > > *cpu7: assertwaitok: non-zero mutex count: 1
> > > > ddb{7}> trace
> > > > panic+0x134
> > > > assertwaitok+0xf8
> > > > mi_switch+0x5c
> > > > sleep_finish+0x160
> > > > rw_enter+0x1cc
> > > > vm_map_lock_read_ln+0x38
> > > > uvmfault_lookup+0x114
> > > > uvm_fault_check+0x68
> > > > uvm_fault+0x12c
> > > > trap+0x7a4
> > > > trapagain+0x4
> > > > --- trap (type 0x300) ---
> > > > phtree_RBT_COMPARE+0x28
> > > > pool_do_put+0x94
> > > > pool_put+0x94
> > >            ^^^^
> > > ...inside this pool.  Which of the 3 is it?  Can someone with a ppc64
> > > figure out?
> >
> > It's pmap_vp_pool.