[glibc] powerpc64le: Add optimized __memcmpeq for POWER10
Sachin Monga via Glibc-cvs <[email protected]> Tue, 2 Jun 2026 06:08:17 +0000 (GMT)
| Newsgroups | gmane.comp.lib.glibc.cvs |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=7193069543b0883b667d7c14d4b83fa3e8b2e1de commit 7193069543b0883b667d7c14d4b83fa3e8b2e1de Author: Sachin Monga <[email protected]> Date: Tue Jun 2 00:44:41 2026 -0500 powerpc64le: Add optimized __memcmpeq for POWER10 __memcmpeq (added in glibc 2.35) was previously an alias to memcmp on POWER10 via strong_alias. However, in the multiarch IFUNC path, this caused __memcmpeq to resolve to the generic C memcmp.c implementation rather than the optimized POWER10 memcmp.S, leaving a significant performance gap. Unlike memcmp, __memcmpeq only needs to return zero or nonzero with no requirement on the sign or magnitude for unequal inputs, allowing a simpler and faster implementation. Performance on POWER10 : 1) __memcmpeq (generic) -> __memcmpeq_power10 The primary motivation - __memcmpeq was resolving to generic C in the multiarch path. - Small data (< 8B to < 512B) : ~52% - 82% improvement. - Bulk (< 16MB to < 256MB) : ~25% - 32% improvement. - Large (1GB) : ~33% improvement 2) memcmp_power10 (optimized .S) -> __memcmpeq_power10: Comparing dedicated __memcmpeq against the optimized memcmp it previously aliased to. - Small data (< 8B to < 256B) : No improvement observed. Real-world workloads predominantly operate on larger buffers - >= 512B : ~9% improvement. - 16MB - 128MB : ~25% - 32% improvement. - 256MB : ~3% improvement. - Large (1GB) : On par. Reviewed-by: Wilco Dijkstra <[email protected]> Diff: --- sysdeps/powerpc/powerpc64/le/power10/memcmp.S | 2 - sysdeps/powerpc/powerpc64/le/power10/memcmpeq.S | 158 +++++++++++++++++++++ sysdeps/powerpc/powerpc64/multiarch/Makefile | 2 +- .../powerpc/powerpc64/multiarch/ifunc-impl-list.c | 20 +++ sysdeps/powerpc/powerpc64/multiarch/memcmp-ppc64.c | 7 +- .../powerpc/powerpc64/multiarch/memcmpeq-power10.S | 28 ++++ sysdeps/powerpc/powerpc64/multiarch/memcmpeq.c | 57 ++++++++ 7 files changed, 268 insertions(+), 6 deletions(-) diff --git a/sysdeps/powerpc/powerpc64/le/power10/memcmp.S b/sysdeps/powerpc/powerpc64/le/power10/memcmp.S index 46a74dea4d..8915676e1b 100644 --- a/sysdeps/powerpc/powerpc64/le/power10/memcmp.S +++ b/sysdeps/powerpc/powerpc64/le/power10/memcmp.S @@ -161,5 +161,3 @@ L(tail8): END (MEMCMP) libc_hidden_builtin_def (memcmp) weak_alias (memcmp, bcmp) -strong_alias (memcmp, __memcmpeq) -libc_hidden_def (__memcmpeq) diff --git a/sysdeps/powerpc/powerpc64/le/power10/memcmpeq.S b/sysdeps/powerpc/powerpc64/le/power10/memcmpeq.S new file mode 100644 index 0000000000..6881e4df1d --- /dev/null +++ b/sysdeps/powerpc/powerpc64/le/power10/memcmpeq.S @@ -0,0 +1,158 @@ +/* Optimized __memcmpeq implementation for POWER10. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <sysdep.h> + +#define COMPARE_32(vr1,vr2,offset)\ + lxvp 32+vr1,offset(r3); \ + lxvp 32+vr2,offset(r4); \ + vxor v4,vr1,vr2; \ + vxor v5,vr1+1,vr2+1; \ + vor v19,v19,v4; \ + vor v19,v19,v5; + +/* int [r3] __memcmpeq (const char *s1 [r3], const char *s2 [r4], + size_t size [r5]) + Returns 0 if equal, 1 if not equal (no lexicographic comparison) */ + +#ifndef MEMCMPEQ +# define MEMCMPEQ __memcmpeq +#endif + .machine power10 +ENTRY_TOCLESS (MEMCMPEQ, 4) + CALL_MCOUNT 3 + + /* Fast path: size == 0 */ + cmpdi cr7,r5,0 + beq cr7,L(finish) + + /* Fast path: same pointer */ + cmpd cr7,r3,r4 + beq cr7,L(finish) + + cmpldi cr6,r5,64 + bgt cr6,L(loop_head) + +/* Compare 64 bytes. This section is used for lengths <= 64 and for the last + bytes for larger lengths. */ +L(last_compare): + li r8,16 + + sldi r9,r5,56 + sldi r8,r8,56 + addi r6,r3,16 + addi r7,r4,16 + + /* Align up to 16 bytes. */ + lxvl 32+v0,r3,r9 + lxvl 32+v2,r4,r9 + + /* Branch to not_equal if any bytes differ (CR6 set by vcmpneb.). + Branch to finish if no bytes remain (CR0.LT set when r9 went + negative after sub.). */ + sub. r9,r9,r8 + vcmpneb. v4,v0,v2 + bne cr6,L(not_equal) + bt 4*cr0+lt,L(finish) + + addi r3,r3,32 + addi r4,r4,32 + + lxvl 32+v1,r6,r9 + lxvl 32+v3,r7,r9 + sub. r9,r9,r8 + vcmpneb. v5,v1,v3 + bne cr6,L(not_equal) + bt 4*cr0+lt,L(finish) + + addi r6,r3,16 + addi r7,r4,16 + + lxvl 32+v6,r3,r9 + lxvl 32+v8,r4,r9 + sub. r9,r9,r8 + vcmpneb. v4,v6,v8 + bne cr6,L(not_equal) + bt 4*cr0+lt,L(finish) + + lxvl 32+v7,r6,r9 + lxvl 32+v9,r7,r9 + vcmpneb. v5,v7,v9 + bne cr6,L(not_equal) + +L(finish): + /* The contents are equal. */ + li r3,0 + blr + +L(not_equal): + li r3,1 + blr + +L(loop_head): + /* Calculate how many loops to run. */ + srdi. r8,r5,7 + beq L(loop_tail) + mtctr r8 + + vxor v18,v18,v18 + vxor v19,v19,v19 + .p2align 5 +L(loop_128): + COMPARE_32(v0,v2,0) + COMPARE_32(v6,v8,32) + vcmpneb. v17,v19,v18 + bne cr6,L(not_equal) + COMPARE_32(v10,v12,64) + COMPARE_32(v14,v16,96) + + vcmpneb. v17,v19,v18 + bne cr6,L(not_equal) + + addi r3,r3,128 + addi r4,r4,128 + bdnz L(loop_128) + + /* Account loop comparisons. */ + clrldi. r5,r5,57 + beq L(finish) + +/* Compares 64 bytes if length is still bigger than 64 bytes. */ + .p2align 5 +L(loop_tail): + /* Initialize accumulator for tail */ + vxor v18,v18,v18 + vxor v19,v19,v19 + + cmpldi r5,64 + ble L(last_compare) + + COMPARE_32(v0,v2,0) + COMPARE_32(v6,v8,32) + + vcmpneb. v17,v19,v18 + bne cr6,L(not_equal) + + addi r3,r3,64 + addi r4,r4,64 + subi r5,r5,64 + b L(last_compare) + +END (MEMCMPEQ) + +libc_hidden_def (MEMCMPEQ) diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile index c9178223a8..164aac9dca 100644 --- a/sysdeps/powerpc/powerpc64/multiarch/Makefile +++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile @@ -30,7 +30,7 @@ sysdep_routines += memcpy-power8-cached memcpy-power7 memcpy-a2 memcpy-power6 \ strncase-power8 ifneq (,$(filter %le,$(config-machine))) -sysdep_routines += memcmp-power10 memcpy-power10 memmove-power10 memset-power10 \ +sysdep_routines += memcmp-power10 memcpy-power10 memmove-power10 memset-power10 memcmpeq-power10 \ rawmemchr-power9 rawmemchr-power10 \ strcmp-power9 strcmp-power10 strncmp-power9 strncmp-power10 \ strcpy-power9 strcat-power10 stpcpy-power9 \ diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c index 1458b4575d..dc9724f083 100644 --- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c +++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c @@ -218,6 +218,26 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, __memcmp_power4) IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_ppc)) + /* Support sysdeps/powerpc/powerpc64/multiarch/memcmpeq.c. + Pre-POWER10 variants reuse __memcmp_* since memcmp's return value + satisfies __memcmpeq's zero/non-zero contract. */ + + IFUNC_IMPL (i, name, __memcmpeq, +#ifdef __LITTLE_ENDIAN__ + IFUNC_IMPL_ADD (array, i, __memcmpeq, + hwcap2 & PPC_FEATURE2_ARCH_3_1 + && hwcap & PPC_FEATURE_HAS_VSX, + __memcmpeq_power10) +#endif + IFUNC_IMPL_ADD (array, i, __memcmpeq, hwcap2 & PPC_FEATURE2_ARCH_2_07 + && hwcap & PPC_FEATURE_HAS_ALTIVEC, + __memcmp_power8) + IFUNC_IMPL_ADD (array, i, __memcmpeq, hwcap & PPC_FEATURE_ARCH_2_06, + __memcmp_power7) + IFUNC_IMPL_ADD (array, i, __memcmpeq, hwcap & PPC_FEATURE_POWER4, + __memcmp_power4) + IFUNC_IMPL_ADD (array, i, __memcmpeq, 1, __memcmp_ppc)) + /* Support sysdeps/powerpc/powerpc64/multiarch/mempcpy.c. */ IFUNC_IMPL (i, name, mempcpy, IFUNC_IMPL_ADD (array, i, mempcpy, diff --git a/sysdeps/powerpc/powerpc64/multiarch/memcmp-ppc64.c b/sysdeps/powerpc/powerpc64/multiarch/memcmp-ppc64.c index ef69cfe8da..f885d3fb55 100644 --- a/sysdeps/powerpc/powerpc64/multiarch/memcmp-ppc64.c +++ b/sysdeps/powerpc/powerpc64/multiarch/memcmp-ppc64.c @@ -22,14 +22,15 @@ #define weak_alias(name, aliasname) \ extern __typeof (__memcmp_ppc) aliasname \ __attribute__ ((weak, alias ("__memcmp_ppc"))); +/* __memcmpeq is now owned by the memcmpeq IFUNC selector (memcmpeq.os) */ #undef strong_alias -#define strong_alias(name, aliasname) \ - extern __typeof (__memcmp_ppc) aliasname \ - __attribute__ ((alias ("__memcmp_ppc"))); +#define strong_alias(name, aliasname) #if IS_IN (libc) && defined(SHARED) # undef libc_hidden_builtin_def # define libc_hidden_builtin_def(name) \ __hidden_ver1(__memcmp_ppc, __GI_memcmp, __memcmp_ppc); +# undef libc_hidden_def +# define libc_hidden_def(name) #endif extern __typeof (memcmp) __memcmp_ppc attribute_hidden; diff --git a/sysdeps/powerpc/powerpc64/multiarch/memcmpeq-power10.S b/sysdeps/powerpc/powerpc64/multiarch/memcmpeq-power10.S new file mode 100644 index 0000000000..ee4b433712 --- /dev/null +++ b/sysdeps/powerpc/powerpc64/multiarch/memcmpeq-power10.S @@ -0,0 +1,28 @@ +/* Wrapper for POWER10 __memcmpeq implementation. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#define MEMCMPEQ __memcmpeq_power10 + +#undef libc_hidden_builtin_def +#define libc_hidden_builtin_def(name) +#undef libc_hidden_def +#define libc_hidden_def(name) +#undef weak_alias +#define weak_alias(name, alias) + +#include <sysdeps/powerpc/powerpc64/le/power10/memcmpeq.S> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memcmpeq.c b/sysdeps/powerpc/powerpc64/multiarch/memcmpeq.c new file mode 100644 index 0000000000..d0c9fdc3d6 --- /dev/null +++ b/sysdeps/powerpc/powerpc64/multiarch/memcmpeq.c @@ -0,0 +1,57 @@ +/* Multiple versions of memcmpeq. PowerPC64 version. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for definition in libc. */ +#if IS_IN (libc) +# define __memcmpeq __redirect___memcmpeq +# include <string.h> +# include <shlib-compat.h> +# include "init-arch.h" + +/* Reuse the existing optimized memcmp variants for pre-POWER10 hardware + as memcmp is a superset */ +extern __typeof (memcmp) __memcmp_ppc attribute_hidden; +extern __typeof (memcmp) __memcmp_power4 attribute_hidden; +extern __typeof (memcmp) __memcmp_power7 attribute_hidden; +extern __typeof (memcmp) __memcmp_power8 attribute_hidden; +extern __typeof (__memcmpeq) __memcmpeq_power10 attribute_hidden; +# undef __memcmpeq + +/* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle + ifunc symbol properly. */ +libc_ifunc_redirected (__redirect___memcmpeq, __memcmpeq, +#ifdef __LITTLE_ENDIAN__ + (hwcap2 & PPC_FEATURE2_ARCH_3_1 + && hwcap & PPC_FEATURE_HAS_VSX) + ? __memcmpeq_power10 : +#endif + (hwcap2 & PPC_FEATURE2_ARCH_2_07 + && hwcap & PPC_FEATURE_HAS_ALTIVEC) + ? __memcmp_power8 : + (hwcap & PPC_FEATURE_ARCH_2_06) + ? __memcmp_power7 + : (hwcap & PPC_FEATURE_POWER4) + ? __memcmp_power4 + : __memcmp_ppc); +# ifdef SHARED +__hidden_ver1 (__memcmpeq, __GI___memcmpeq, __redirect___memcmpeq) + __attribute__ ((visibility ("hidden"))) __attribute_copy__ (__memcmpeq); +# endif +#else +#include <string/memcmp.c> +#endif