ICE + XSK ZC - page faults on 6.1 LTS when process exits?

Hi,=0A=
=0A=
I've been testing apps that use XSK+ZC on ICE with newer builds of the 6.1 =
LTS kernel in preparation for some production upgrades, and I've started to=
 notice some instability on newer versions. I can reproduce the issue easil=
y in the lab.=0A=
=0A=
Config:=0A=
- Known good multi-threaded application (i.e. production grade)=0A=
- Uses eBPF and AF_XDP with zero copy to act as 'bump in wire' in network=
=0A=
- Xeon's with Intel E810-CQDA2 (firmware: 3.20 0x8000d83e 1.3146.0)=0A=
- Effectively a vanilla rebuild of 6.1 using configs from el-repo project=
=0A=
=0A=
Scenario:=0A=
- Noticing hard kernel faults when shutting down application=0A=
- Can happen if the process is shut down via systemctl stop=0A=
- Can even happen with a simple kill -9 command to the PID=0A=
- Appears in builds after 6.1.87=0A=
=0A=
Tested kernels:=0A=
- 6.1.84: process exits smoothly=0A=
- 6.1.87: process exits smoothly=0A=
- 6.1.97: BUG: unable to handle page fault for address=0A=
- 6.1.106: BUG: unable to handle page fault for address=0A=
=0A=
Kdump log is below [1] from 6.1.106 but does seem to be the same in the ear=
lier version.=0A=
=0A=
Can anyone advise if this is a known issue?=0A=
=0A=
I don't have any builds between 6.1.87 and 6.1.97 but I can spend some time=
 trying to pinpoint the exact version things start to go wrong in, if it wo=
uld help anyone better equipped than me to debug!=0A=
=0A=
Kind regards=0A=
Alasdair=0A=
=0A=
[1] kdump log=0A=
=0A=
[  158.666867] BUG: unable to handle page fault for address: ffffa6510e5580=
c0=0A=
[  158.666887] #PF: supervisor read access in kernel mode=0A=
[  158.666896] #PF: error_code(0x0000) - not-present page=0A=
[  158.666903] PGD 100000067 P4D 100000067 PUD 106dc4067 PMD 0=0A=
[  158.666914] Oops: 0000 [#1] PREEMPT SMP PTI=0A=
[  158.666922] CPU: 7 PID: 1808 Comm: tlndd.bin Kdump: loaded Tainted: G   =
         E      6.1.106-1.X.el9.x86_64 #1=0A=
[  158.666940] Hardware name: Supermicro SYS-1028R-TDW/X10DDW-i, BIOS 3.2 1=
2/16/2019=0A=
[  158.666950] RIP: 0010:xp_free+0x11/0x80=0A=
[  158.666962] Code: 8b 04 d0 48 83 e0 fe 48 01 f0 c3 cc cc cc cc 66 2e 0f =
1f 84 00 00 00 00 00 0f 1f 44 00 00 41 56 41 55 41 54 55 48 8d 6f 58 53 <48=
> 8b 47 58 48 39 c5 74 0d 5b 5d 41 5c 41 5d 41 5e c3 cc cc cc cc=0A=
[  158.666985] RSP: 0018:ffffa65089e8b760 EFLAGS: 00010202=0A=
[  158.666993] RAX: ffff8fcf077c0000 RBX: 0000000000000001 RCX: 00000000000=
00000=0A=
[  158.667003] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffa6510e5=
58068=0A=
[  158.667012] RBP: ffffa6510e5580c0 R08: fffff8c50415a108 R09: ffff8fc7cac=
60000=0A=
[  158.667022] R10: 0000000000000219 R11: ffffffffffffffff R12: 00000000000=
00fff=0A=
[  158.667031] R13: 0000000000000000 R14: 0000000000000000 R15: ffff8fc7c13=
9d340=0A=
[  158.667040] FS:  00007f8504996880(0000) GS:ffff8fcedfdc0000(0000) knlGS:=
0000000000000000=0A=
[  158.667050] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033=0A=
[  158.667058] CR2: ffffa6510e5580c0 CR3: 00000001448e2002 CR4: 00000000001=
706e0=0A=
[  158.667068] Call Trace:=0A=
[  158.667075]  <TASK>=0A=
[  158.667082]  ? show_trace_log_lvl+0x1c4/0x2df=0A=
[  158.667094]  ? show_trace_log_lvl+0x1c4/0x2df=0A=
[  158.667103]  ? ice_xsk_clean_rx_ring+0x39/0x60 [ice]=0A=
[  158.667157]  ? __die_body.cold+0x8/0xd=0A=
[  158.667166]  ? page_fault_oops+0xac/0x150=0A=
[  158.667176]  ? fixup_exception+0x22/0x340=0A=
[  158.667185]  ? exc_page_fault+0xb2/0x150=0A=
[  158.667195]  ? asm_exc_page_fault+0x22/0x30=0A=
[  158.667206]  ? xp_free+0x11/0x80=0A=
[  158.667215]  ice_xsk_clean_rx_ring+0x39/0x60 [ice]=0A=
[  158.667250]  ice_clean_rx_ring+0x157/0x180 [ice]=0A=
[  158.667284]  ice_down+0x172/0x2b0 [ice]=0A=
[  158.667311]  ? ice_xdp_setup_prog+0x3b0/0x3b0 [ice]=0A=
[  158.667337]  ice_xdp_setup_prog+0xe3/0x3b0 [ice]=0A=
[  158.667364]  ? ice_xdp_setup_prog+0x3b0/0x3b0 [ice]=0A=
[  158.667391]  dev_xdp_install+0xc7/0x100=0A=
[  158.667402]  dev_xdp_attach+0x1e0/0x560=0A=
[  158.667412]  do_setlink+0x7a8/0xc10=0A=
[  158.667422]  ? __nla_validate_parse+0x12b/0x1b0=0A=
[  158.667436]  __rtnl_newlink+0x540/0x650=0A=
[  158.667446]  rtnl_newlink+0x44/0x70=0A=
[  158.667454]  rtnetlink_rcv_msg+0x15c/0x3d0=0A=
[  158.667477]  ? rtnl_calcit.isra.0+0x140/0x140=0A=
[  158.667485]  netlink_rcv_skb+0x51/0x100=0A=
[  158.667727]  netlink_unicast+0x246/0x360=0A=
[  158.667953]  netlink_sendmsg+0x24e/0x4b0=0A=
[  158.668173]  __sock_sendmsg+0x62/0x70=0A=
[  158.668389]  ____sys_sendmsg+0x247/0x2d0=0A=
[  158.668602]  ? copy_msghdr_from_user+0x6d/0xa0=0A=
[  158.668815]  ___sys_sendmsg+0x88/0xd0=0A=
[  158.669028]  ? __sk_destruct+0x156/0x230=0A=
[  158.669234]  ? kmem_cache_free+0x134/0x300=0A=
[  158.669437]  ? rcu_nocb_try_bypass+0x4a/0x440=0A=
[  158.669634]  ? __sk_destruct+0x156/0x230=0A=
[  158.669825]  ? _raw_spin_unlock_irqrestore+0x23/0x40=0A=
[  158.670010]  ? mod_objcg_state+0xc9/0x2f0=0A=
[  158.670186]  ? refill_obj_stock+0xae/0x160=0A=
[  158.670359]  ? rseq_get_rseq_cs.isra.0+0x16/0x220=0A=
[  158.670529]  ? rcu_nocb_try_bypass+0x4a/0x440=0A=
[  158.670696]  ? rseq_ip_fixup+0x72/0x1e0=0A=
[  158.670860]  __sys_sendmsg+0x59/0xa0=0A=
[  158.671021]  ? syscall_trace_enter.constprop.0+0x11e/0x190=0A=
[  158.671185]  do_syscall_64+0x35/0x80=0A=
[  158.671345]  entry_SYSCALL_64_after_hwframe+0x6e/0xd8=0A=
[  158.671503] RIP: 0033:0x7f850510f917=0A=
[  158.671658] Code: 0e 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b9 0f 1f =
00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 2e 00 00 00 0f 05 <48=
> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 89 54 24 1c 48 89 74 24 10=0A=
[  158.671993] RSP: 002b:00007ffcc805f238 EFLAGS: 00000246 ORIG_RAX: 000000=
000000002e=0A=
[  158.672171] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f85051=
0f917=0A=
[  158.672352] RDX: 0000000000000000 RSI: 000000000198e9e8 RDI: 00000000000=
00009=0A=
[  158.672534] RBP: 0000000001933c00 R08: 0000000001935980 R09: 00000000004=
60e48=0A=
[  158.672716] R10: 0000000000000011 R11: 0000000000000246 R12: 00000000019=
33c30=0A=
[  158.672899] R13: 0000000000515fd8 R14: 000000000198e9d0 R15: 00000000005=
13690=0A=
[  158.673086]  </TASK>=0A=
[  158.673269] Modules linked in: bonding(E) tls(E) nft_fib_inet(E) nft_fib=
_ipv4(E) nft_fib_ipv6(E) nft_fib(E) nft_reject_inet(E) nf_reject_ipv4(E) nf=
_reject_ipv6(E) nft_reject(E) nft_ct(E) nft_chain_nat(E) nf_nat(E) nf_connt=
rack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) rfkill(E) ip_set(E) nf_tables(E=
) libcrc32c(E) nfnetlink(E) vfat(E) fat(E) ipmi_ssif(E) intel_rapl_msr(E) i=
ntel_rapl_common(E) sb_edac(E) x86_pkg_temp_thermal(E) intel_powerclamp(E) =
coretemp(E) kvm_intel(E) iTCO_wdt(E) intel_pmc_bxt(E) iTCO_vendor_support(E=
) kvm(E) irqbypass(E) rapl(E) intel_cstate(E) ast(E) intel_uncore(E) drm_vr=
am_helper(E) drm_ttm_helper(E) ttm(E) pcspkr(E) mei_me(E) drm_kms_helper(E)=
 i2c_i801(E) lpc_ich(E) mei(E) i2c_smbus(E) mxm_wmi(E) ioatdma(E) acpi_ipmi=
(E) ipmi_si(E) ipmi_devintf(E) ipmi_msghandler(E) acpi_pad(E) acpi_power_me=
ter(E) joydev(E) drm(E) fuse(E) ext4(E) mbcache(E) jbd2(E) sd_mod(E) t10_pi=
(E) sg(E) ahci(E) crct10dif_pclmul(E) crc32_pclmul(E) libahci(E) crc32c_int=
el(E) ice(E)=0A=
[  158.673314]  polyval_clmulni(E) polyval_generic(E) igb(E) libata(E) ghas=
h_clmulni_intel(E) i2c_algo_bit(E) dca(E) wmi(E) dm_mirror(E) dm_region_has=
h(E) dm_log(E) dm_mod(E)=0A=
[  158.675578] CR2: ffffa6510e5580c0=