[PATCH] arm64: uaccess: Optimize copy_to/from_user with LSUI extension
Qinxin Xia <[email protected]>
| Newsgroups | org.infradead.lists.linux-arm-kernel,org.kernel.vger.linux-kernel |
|---|---|
| Message-ID | <[email protected]> |
Add support for the ARMv9.6/LSUI (Load/Store Unprivileged Immediate) extension to optimize copy_from_user operations. The implementation uses the alternative mechanism to select between the standard unprivileged load-pair sequence and the new ldtp/sttp instruction at runtime based on hardware capability. Co-developed-by: Qi Xi <[email protected]> Signed-off-by: Qi Xi <[email protected]> Signed-off-by: Qinxin Xia <[email protected]> --- arch/arm64/include/asm/asm-uaccess.h | 6 ++++++ arch/arm64/lib/copy_from_user.S | 11 +++++++++++ arch/arm64/lib/copy_to_user.S | 11 +++++++++++ 3 files changed, 28 insertions(+) diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h index 12aa6a283249..4c1b94b275e9 100644 --- a/arch/arm64/include/asm/asm-uaccess.h +++ b/arch/arm64/include/asm/asm-uaccess.h @@ -94,4 +94,10 @@ alternative_else_nop_endif _asm_extable_uaccess 8888b, \l; .endm + + .macro user_ldst_pair l, inst, reg1, reg2, addr, post_inc +8888: \inst \reg1, \reg2, [\addr], \post_inc; + + _asm_extable_uaccess 8888b, \l; + .endm #endif diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 400057d607ec..f561a9c07345 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -45,7 +45,18 @@ .endm .macro ldp1 reg1, reg2, ptr, val +#ifdef CONFIG_ARM64_LSUI + alternative_if_not ARM64_HAS_LSUI + user_ldp 9997f, \reg1, \reg2, \ptr, \val + alternative_else + .arch_extension lsui + user_ldst_pair 9997f, ldtp, \reg1, \reg2, \ptr, \val + nop + nop + alternative_endif +#else user_ldp 9997f, \reg1, \reg2, \ptr, \val +#endif .endm .macro stp1 reg1, reg2, ptr, val diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S index 819f2e3fc7a9..a77dd153566f 100644 --- a/arch/arm64/lib/copy_to_user.S +++ b/arch/arm64/lib/copy_to_user.S @@ -48,7 +48,18 @@ .endm .macro stp1 reg1, reg2, ptr, val +#ifdef CONFIG_ARM64_LSUI + alternative_if_not ARM64_HAS_LSUI + user_stp 9997f, \reg1, \reg2, \ptr, \val + alternative_else + .arch_extension lsui + user_ldst_pair 9997f, sttp, \reg1, \reg2, \ptr, \val + nop + nop + alternative_endif +#else user_stp 9997f, \reg1, \reg2, \ptr, \val +#endif .endm .macro cpy1 dst, src, count -- 2.33.0