[glibc] AArch64: Optimize memcmp for Kunpeng 950 with SVE

https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=3237d63b8462764e282cd8de79a02558071d4348

commit 3237d63b8462764e282cd8de79a02558071d4348
Author: Weihong Ye <[email protected]>
Date:   Mon May 18 16:25:46 2026 +0000

    AArch64: Optimize memcmp for Kunpeng 950 with SVE
    
    Key optimizations:
    - Use SVE predication for branch-free handling of short inputs and tails
    - Use 4-way loop unrolling to maximize pipeline utilization
    - Optimize mismatch detection with early exit logic
    
    Benchmark (bench-memcmp, generic -> this patch):
    - Small (0-128B): 15% - 50% speedup
    - Medium (129-1024B): 21% - 50% speedup
    - Large (2048-4096B): 28% - 50% speedup
    
    Note: regressions may be observed in edge cases where offsets
    are near 4K boundaries. These instances are rare and the overall
    performance gain remains significantly positive.
    
    Also add IFUNC support for memcmp and correct the first-line
    comment in memcpy_kunpeng950.S.

Diff:
---
 sysdeps/aarch64/memcmp.S                      |  13 ++-
 sysdeps/aarch64/multiarch/Makefile            |   2 +
 sysdeps/aarch64/multiarch/ifunc-impl-list.c   |   5 +-
 sysdeps/aarch64/multiarch/memcmp.c            |  54 ++++++++++
 sysdeps/aarch64/multiarch/memcmp_generic.S    |  42 ++++++++
 sysdeps/aarch64/multiarch/memcmp_kunpeng950.S | 145 ++++++++++++++++++++++++++
 sysdeps/aarch64/multiarch/memcpy_kunpeng950.S |   2 +-
 7 files changed, 256 insertions(+), 7 deletions(-)

diff --git a/sysdeps/aarch64/memcmp.S b/sysdeps/aarch64/memcmp.S
index f177520d63..e33086b4eb 100644
--- a/sysdeps/aarch64/memcmp.S
+++ b/sysdeps/aarch64/memcmp.S
@@ -42,8 +42,11 @@
 #define src1end	x7
 #define src2end	x8
 
+#ifndef MEMCMP
+# define MEMCMP memcmp
+#endif
 
-ENTRY (memcmp)
+ENTRY (MEMCMP)
 	cmp	limit, 16
 	b.lo	L(less16)
 	ldp	data1, data3, [src1]
@@ -197,10 +200,10 @@ L(loop64):
 	cneg	result, result, lo
 	ret
 
-END (memcmp)
+END (MEMCMP)
 #undef bcmp
-weak_alias (memcmp, bcmp)
+weak_alias (MEMCMP, bcmp)
 #undef __memcmpeq
-strong_alias (memcmp, __memcmpeq)
-libc_hidden_builtin_def (memcmp)
+strong_alias (MEMCMP, __memcmpeq)
+libc_hidden_builtin_def (MEMCMP)
 libc_hidden_def (__memcmpeq)
diff --git a/sysdeps/aarch64/multiarch/Makefile b/sysdeps/aarch64/multiarch/Makefile
index 988f7cec25..38952655b1 100644
--- a/sysdeps/aarch64/multiarch/Makefile
+++ b/sysdeps/aarch64/multiarch/Makefile
@@ -1,5 +1,7 @@
 ifeq ($(subdir),string)
 sysdep_routines += \
+  memcmp_generic \
+  memcmp_kunpeng950 \
   memcpy_a64fx \
   memcpy_generic \
   memcpy_kunpeng950 \
diff --git a/sysdeps/aarch64/multiarch/ifunc-impl-list.c b/sysdeps/aarch64/multiarch/ifunc-impl-list.c
index ea5f5853c3..d43f6b58ee 100644
--- a/sysdeps/aarch64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/aarch64/multiarch/ifunc-impl-list.c
@@ -33,7 +33,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 
   INIT_ARCH ();
 
-  /* Support sysdeps/aarch64/multiarch/memcpy.c, memmove.c and memset.c.  */
+  /* Support sysdeps/aarch64/multiarch/memcmp.c, memcpy.c, memmove.c and memset.c.  */
+  IFUNC_IMPL (i, name, memcmp,
+	      IFUNC_IMPL_ADD (array, i, memcmp, sve, __memcmp_kunpeng950)
+	      IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_generic))
   IFUNC_IMPL (i, name, memcpy,
 	      IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_oryon1)
 	      IFUNC_IMPL_ADD (array, i, memcpy, sve, __memcpy_a64fx)
diff --git a/sysdeps/aarch64/multiarch/memcmp.c b/sysdeps/aarch64/multiarch/memcmp.c
new file mode 100644
index 0000000000..5c3dc63068
--- /dev/null
+++ b/sysdeps/aarch64/multiarch/memcmp.c
@@ -0,0 +1,54 @@
+/* Multiple versions of memcmp. AARCH64 version.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   Copyright The GNU Toolchain Authors.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in libc.  */
+
+#if IS_IN (libc)
+/* Redefine memcmp so that the compiler won't complain about the type
+   mismatch with the IFUNC selector in strong_alias, below.  */
+# undef memcmp
+# define memcmp __redirect_memcmp
+# include <string.h>
+# include <init-arch.h>
+
+extern __typeof (__redirect_memcmp) __libc_memcmp;
+
+extern __typeof (__redirect_memcmp) __memcmp_generic attribute_hidden;
+extern __typeof (__redirect_memcmp) __memcmp_kunpeng950 attribute_hidden;
+
+static inline __typeof (__redirect_memcmp) *
+select_memcmp_ifunc (void)
+{
+  INIT_ARCH ();
+
+  if (sve)
+  {
+    if (IS_KUNPENG950 (midr))
+    {
+      return __memcmp_kunpeng950;
+    }
+  }
+  return __memcmp_generic;
+}
+
+libc_ifunc (__libc_memcmp, select_memcmp_ifunc ());
+
+# undef memcmp
+strong_alias (__libc_memcmp, memcmp);
+#endif
diff --git a/sysdeps/aarch64/multiarch/memcmp_generic.S b/sysdeps/aarch64/multiarch/memcmp_generic.S
new file mode 100644
index 0000000000..9b24610814
--- /dev/null
+++ b/sysdeps/aarch64/multiarch/memcmp_generic.S
@@ -0,0 +1,42 @@
+/* A Generic Optimized memcmp implementation for AARCH64.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* The actual memcmp code is in ../memcmp.S.  If we are
+   building libc this file defines __memcmp_generic. Otherwise
+   the include of ../memcmp.S will define the normal __memcmp
+   entry points.  */
+
+#include <sysdep.h>
+
+#if IS_IN (libc)
+
+# define MEMCMP __memcmp_generic
+
+/* Do not hide the generic versions of memcmp, we use them
+   internally.  */
+# undef libc_hidden_builtin_def
+# define libc_hidden_builtin_def(name)
+
+# ifdef SHARED
+/* It doesn't make sense to send libc-internal memcmp calls through a PLT. */
+	.globl __GI_memcmp; __GI_memcmp = __memcmp_generic
+# endif
+
+#endif
+
+#include "../memcmp.S"
diff --git a/sysdeps/aarch64/multiarch/memcmp_kunpeng950.S b/sysdeps/aarch64/multiarch/memcmp_kunpeng950.S
new file mode 100644
index 0000000000..b5fb59ebcf
--- /dev/null
+++ b/sysdeps/aarch64/multiarch/memcmp_kunpeng950.S
@@ -0,0 +1,145 @@
+/* Optimized memcmp for Huawei Kunpeng 950 processor.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library.  If not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <sysdep.h>
+
+/* Assumptions:
+ *
+ * ARMv8.2-a, AArch64, Advanced SIMD, SVE, unaligned accesses
+ *
+ */
+
+.arch armv8.2-a+sve
+
+#define src1    x0
+#define src2    x1
+#define cnt     x2
+#define result  w0
+#define off_vl      x3
+#define off_vlx2    x4
+#define off_vlx3    x5
+#define off_vlx4    x6
+
+ENTRY (__memcmp_kunpeng950)
+    whilelo p0.b, xzr, cnt
+    b.none  L(equal)
+    cntb    off_vl
+    ld1b    z0.b, p0/z, [src1]
+    ld1b    z1.b, p0/z, [src2]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    whilelo p0.b, off_vl, cnt
+    b.none  L(equal)
+    cntb    off_vlx2, all, mul #2
+    ld1b    z0.b, p0/z, [src1, 1, mul vl]
+    ld1b    z1.b, p0/z, [src2, 1, mul vl]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    whilelo p0.b, off_vlx2, cnt
+    b.none  L(equal)
+    cntb    off_vlx3, all, mul #3
+    ld1b    z0.b, p0/z, [src1, 2, mul vl]
+    ld1b    z1.b, p0/z, [src2, 2, mul vl]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    whilelo p0.b, off_vlx3, cnt
+    b.none  L(equal)
+    cntb    off_vlx4, all, mul #4
+    ld1b    z0.b, p0/z, [src1, 3, mul vl]
+    ld1b    z1.b, p0/z, [src2, 3, mul vl]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    subs    cnt, cnt, off_vlx4
+    b.ls    L(equal)
+    add     src1, src1, off_vlx4
+    add     src2, src2, off_vlx4
+    cmp     cnt, off_vlx4
+    b.lo    L(tail_4xvl)
+
+    .p2align 4
+L(loop_full):
+    ld1b    z0.b, p0/z, [src1]
+    ld1b    z1.b, p0/z, [src2]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    ld1b    z0.b, p0/z, [src1, off_vl]
+    ld1b    z1.b, p0/z, [src2, off_vl]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    ld1b    z0.b, p0/z, [src1, off_vlx2]
+    ld1b    z1.b, p0/z, [src2, off_vlx2]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    ld1b    z0.b, p0/z, [src1, off_vlx3]
+    ld1b    z1.b, p0/z, [src2, off_vlx3]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    add     src1, src1, off_vlx4
+    add     src2, src2, off_vlx4
+    subs    cnt, cnt, off_vlx4
+    cmp     cnt, off_vlx4
+    b.hs    L(loop_full)
+
+L(tail_4xvl):
+    whilelo p0.b, xzr, cnt
+    b.none  L(equal)
+    ld1b    z0.b, p0/z, [src1]
+    ld1b    z1.b, p0/z, [src2]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    whilelo p0.b, off_vl, cnt
+    b.none  L(equal)
+    ld1b    z0.b, p0/z, [src1, off_vl]
+    ld1b    z1.b, p0/z, [src2, off_vl]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    whilelo p0.b, off_vlx2, cnt
+    b.none  L(equal)
+    ld1b    z0.b, p0/z, [src1, off_vlx2]
+    ld1b    z1.b, p0/z, [src2, off_vlx2]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+    whilelo p0.b, off_vlx3, cnt
+    b.none  L(equal)
+    ld1b    z0.b, p0/z, [src1, off_vlx3]
+    ld1b    z1.b, p0/z, [src2, off_vlx3]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch)
+
+L(equal):
+    mov     result, #0
+    ret
+
+L(mismatch):
+    mov     result, #1
+    cmphi   p2.b, p1/z, z1.b, z0.b
+    cneg    result, result, mi
+    ret
+END (__memcmp_kunpeng950)
diff --git a/sysdeps/aarch64/multiarch/memcpy_kunpeng950.S b/sysdeps/aarch64/multiarch/memcpy_kunpeng950.S
index 82534f9c18..38a56303de 100644
--- a/sysdeps/aarch64/multiarch/memcpy_kunpeng950.S
+++ b/sysdeps/aarch64/multiarch/memcpy_kunpeng950.S
@@ -1,4 +1,4 @@
-/* Optimized memcpy for Huawei Kupeng 950 processor.
+/* Optimized memcpy for Huawei Kunpeng 950 processor.
    Copyright (C) 2026 Free Software Foundation, Inc.
 
    This file is part of the GNU C Library.