[glibc] x86_64: Prefer EVEX512 code-path on AMD Zen5 CPUs
"H.J. Lu via Glibc-cvs" <[email protected]>
| Newsgroups | gmane.comp.lib.glibc.cvs |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=cd5fda114ece002945ace3d54a8f80a4f67d1fbb commit cd5fda114ece002945ace3d54a8f80a4f67d1fbb Author: Sajan Karumanchi <[email protected]> Date: Thu Mar 26 09:21:30 2026 +0000 x86_64: Prefer EVEX512 code-path on AMD Zen5 CPUs Introduced a synthetic architecture preference flag (Prefer_EVEX512) and enabled it for AMD Zen5 (CPUID Family 0x1A) when AVX-512 is supported. This flag modifies IFUNC dispatch to prefer 512-bit EVEX variants over 256-bit EVEX variants for string and memory functions on Zen5 processors, leveraging their native 512-bit execution units for improved throughput. When Prefer_EVEX512 is set, the dispatcher selects evex512 implementations; otherwise, it falls back to evex (256-bit) variants. The implementation updates the IFUNC selection logic in ifunc-avx2.h and ifunc-evex.h to check for the Prefer_EVEX512 flag before dispatching to EVEX512 implementations. This change affects six string/memory functions: - strchr - strlen - strnlen - strrchr - strchrnul - memchr Benchmarks conducted on AMD Zen5 hardware demonstrate significant performance improvements across all affected functions: Function Baseline Patched Avg Avg Avg Max Variant Variant Baseline Patched Change Improve (ns) (ns) % % ------------+----------+----------+-----------+----------+--------+-------- STRCHR evex evex512 16.408 12.293 25.08% 37.69% STRLEN evex evex512 16.862 11.436 32.18% 56.74% STRNLEN evex evex512 18.493 11.762 36.40% 64.40% STRRCHR evex evex512 15.154 10.874 28.24% 44.38% STRCHRNUL evex evex512 16.464 12.605 23.44% 45.56% MEMCHR evex evex512 9.984 8.268 17.19% 39.99% Additionally, a tunable option (glibc.cpu.x86_cpu_features.preferred) is provided to allow runtime control of the Prefer_EVEX512 flag for testing and compatibility. Reviewed-by: Ganesh Gopalasubramanian <[email protected]> Reviewed-by: H.J. Lu <[email protected]> Diff: --- sysdeps/x86/cpu-features.c | 6 ++++++ sysdeps/x86/cpu-tunables.c | 6 ++++++ .../include/cpu-features-preferred_feature_index_1.def | 1 + sysdeps/x86_64/multiarch/ifunc-avx2.h | 15 ++++++++++++--- sysdeps/x86_64/multiarch/ifunc-evex.h | 11 ++++++++++- sysdeps/x86_64/multiarch/memchr.c | 1 + sysdeps/x86_64/multiarch/strchr.c | 8 +++++++- sysdeps/x86_64/multiarch/strchrnul.c | 1 + sysdeps/x86_64/multiarch/strlen.c | 1 + sysdeps/x86_64/multiarch/strnlen.c | 1 + sysdeps/x86_64/multiarch/strrchr.c | 1 + 11 files changed, 47 insertions(+), 5 deletions(-) diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c index ccfef55652..73ee4f2bb8 100644 --- a/sysdeps/x86/cpu-features.c +++ b/sysdeps/x86/cpu-features.c @@ -1009,6 +1009,12 @@ disable_tsx: cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset] &= ~bit_arch_Avoid_Non_Temporal_Memset; + /* Prefer EVEX512 string/memory variants on AMD Zen5 (Family 0x1A) + when AVX-512 is usable. */ + if (family == 0x1A && CPU_FEATURE_USABLE_P (cpu_features, AVX512F)) + cpu_features->preferred[index_arch_Prefer_EVEX512] + |= bit_arch_Prefer_EVEX512; + if (CPU_FEATURE_USABLE_P (cpu_features, AVX)) { /* Since the FMA4 bit is in CPUID_INDEX_80000001 and diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c index 51769a5493..59dde5f1d7 100644 --- a/sysdeps/x86/cpu-tunables.c +++ b/sysdeps/x86/cpu-tunables.c @@ -203,6 +203,12 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp) 11); } break; + case 14: + { + CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH + (n, cpu_features, Prefer_EVEX512, AVX512F, 14); + } + break; case 15: { CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def index 8cab2ae248..74acb3fde1 100644 --- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def +++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def @@ -35,3 +35,4 @@ BIT (Prefer_FSRM) BIT (Avoid_Short_Distance_REP_MOVSB) BIT (Avoid_Non_Temporal_Memset) BIT (Avoid_STOSB) +BIT (Prefer_EVEX512) diff --git a/sysdeps/x86_64/multiarch/ifunc-avx2.h b/sysdeps/x86_64/multiarch/ifunc-avx2.h index bc7178d97e..474466ba93 100644 --- a/sysdeps/x86_64/multiarch/ifunc-avx2.h +++ b/sysdeps/x86_64/multiarch/ifunc-avx2.h @@ -1,4 +1,4 @@ -/* Common definition for ifunc selections optimized with SSE2 and AVX2. +/* Common definition for ifunc selections optimized with SSE2, AVX2 and EVEX512. All versions must be listed in ifunc-impl-list.c. Copyright (C) 2017-2026 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -25,6 +25,10 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; +#ifdef USE_EVEX512 +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden; +#endif + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; @@ -44,8 +48,13 @@ IFUNC_SELECTOR (void) { if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) - return OPTIMIZE (evex); - + { +#ifdef USE_EVEX512 + if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512)) + return OPTIMIZE (evex512); +#endif + return OPTIMIZE (evex); + } if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) return OPTIMIZE (avx2_rtm); diff --git a/sysdeps/x86_64/multiarch/ifunc-evex.h b/sysdeps/x86_64/multiarch/ifunc-evex.h index 973a5b3d15..02ca749a72 100644 --- a/sysdeps/x86_64/multiarch/ifunc-evex.h +++ b/sysdeps/x86_64/multiarch/ifunc-evex.h @@ -1,4 +1,4 @@ -/* Common definition for ifunc selection optimized with EVEX. +/* Common definition for ifunc selection optimized with EVEX and EVEX512. All versions must be listed in ifunc-impl-list.c. Copyright (C) 2017-2026 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -22,6 +22,10 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_rtm) attribute_hidden; +#ifdef USE_EVEX512 +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden; +#endif + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; @@ -42,6 +46,11 @@ IFUNC_SELECTOR (void) if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) { +#ifdef USE_EVEX512 + if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512)) + return OPTIMIZE (evex512); +#endif + if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) return OPTIMIZE (evex_rtm); diff --git a/sysdeps/x86_64/multiarch/memchr.c b/sysdeps/x86_64/multiarch/memchr.c index cea0518787..b2a6c666a9 100644 --- a/sysdeps/x86_64/multiarch/memchr.c +++ b/sysdeps/x86_64/multiarch/memchr.c @@ -24,6 +24,7 @@ # undef memchr # define SYMBOL_NAME memchr +# define USE_EVEX512 1 # include "ifunc-evex.h" libc_ifunc_redirected (__redirect_memchr, memchr, IFUNC_SELECTOR ()); diff --git a/sysdeps/x86_64/multiarch/strchr.c b/sysdeps/x86_64/multiarch/strchr.c index 1064bc8e5b..f6bd36ba73 100644 --- a/sysdeps/x86_64/multiarch/strchr.c +++ b/sysdeps/x86_64/multiarch/strchr.c @@ -27,6 +27,7 @@ # include <init-arch.h> extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden; extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; @@ -46,7 +47,12 @@ IFUNC_SELECTOR (void) { if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) - return OPTIMIZE (evex); + { + if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512)) + return OPTIMIZE (evex512); + + return OPTIMIZE (evex); + } if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) return OPTIMIZE (avx2_rtm); diff --git a/sysdeps/x86_64/multiarch/strchrnul.c b/sysdeps/x86_64/multiarch/strchrnul.c index f4bfc28c7b..3bb93bb812 100644 --- a/sysdeps/x86_64/multiarch/strchrnul.c +++ b/sysdeps/x86_64/multiarch/strchrnul.c @@ -26,6 +26,7 @@ # undef strchrnul # define SYMBOL_NAME strchrnul +# define USE_EVEX512 1 # include "ifunc-avx2.h" libc_ifunc_redirected (__redirect_strchrnul, __strchrnul, diff --git a/sysdeps/x86_64/multiarch/strlen.c b/sysdeps/x86_64/multiarch/strlen.c index 95aa48e5d3..2f93abf1c8 100644 --- a/sysdeps/x86_64/multiarch/strlen.c +++ b/sysdeps/x86_64/multiarch/strlen.c @@ -24,6 +24,7 @@ # undef strlen # define SYMBOL_NAME strlen +# define USE_EVEX512 1 # include "ifunc-avx2.h" libc_ifunc_redirected (__redirect_strlen, strlen, IFUNC_SELECTOR ()); diff --git a/sysdeps/x86_64/multiarch/strnlen.c b/sysdeps/x86_64/multiarch/strnlen.c index bfe4d7abf0..62e50a27e4 100644 --- a/sysdeps/x86_64/multiarch/strnlen.c +++ b/sysdeps/x86_64/multiarch/strnlen.c @@ -26,6 +26,7 @@ # undef strnlen # define SYMBOL_NAME strnlen +# define USE_EVEX512 1 # include "ifunc-avx2.h" libc_ifunc_redirected (__redirect_strnlen, __strnlen, IFUNC_SELECTOR ()); diff --git a/sysdeps/x86_64/multiarch/strrchr.c b/sysdeps/x86_64/multiarch/strrchr.c index 66d3f5b1d1..0965938561 100644 --- a/sysdeps/x86_64/multiarch/strrchr.c +++ b/sysdeps/x86_64/multiarch/strrchr.c @@ -23,6 +23,7 @@ # undef strrchr # define SYMBOL_NAME strrchr +# define USE_EVEX512 1 # include "ifunc-avx2.h" libc_ifunc_redirected (__redirect_strrchr, strrchr, IFUNC_SELECTOR ());