[PATCH 1/2] AArch64: Add SVE2 strchr
Wilco Dijkstra <[email protected]> Mon, 3 Aug 2026 15:48:14 +0000
| Newsgroups | gmane.comp.lib.glibc.alpha |
|---|---|
| Message-ID | <PAWPR08MB898200881AE13F4E6B3DA67083D52@PAWPR08MB8982.eurprd08.prod.outlook.com> |
=0A=
Add an SVE2 strchr implementation and ifunc support. Use MATCH to check fo=
r=0A=
both the input char and NUL. Performance of bench-strchr improves ~28% on=
=0A=
Neoverse V2.=0A=
=0A=
---=0A=
=0A=
diff --git a/sysdeps/aarch64/multiarch/Makefile b/sysdeps/aarch64/multiarch=
/Makefile=0A=
index 6294a70c252af39964a8750749db34da413e6656..9616231c042c03088384b07ea8c=
5465d413048d1 100644=0A=
--- a/sysdeps/aarch64/multiarch/Makefile=0A=
+++ b/sysdeps/aarch64/multiarch/Makefile=0A=
@@ -16,6 +16,8 @@ sysdep_routines +=3D \=0A=
memset_oryon1 \=0A=
memset_sve_zva64 \=0A=
memset_zva64 \=0A=
+ strchr_generic \=0A=
+ strchr_sve2 \=0A=
strlen_asimd \=0A=
strlen_generic \=0A=
# sysdep_routines=0A=
diff --git a/sysdeps/aarch64/multiarch/dl-symbol-redir-ifunc.h b/sysdeps/aa=
rch64/multiarch/dl-symbol-redir-ifunc.h=0A=
index 0910e321d24d84db951cf498c3e883c2cfdd01f5..50b2581455c94dc2bd38f8e5ba9=
a357d1b79e89f 100644=0A=
--- a/sysdeps/aarch64/multiarch/dl-symbol-redir-ifunc.h=0A=
+++ b/sysdeps/aarch64/multiarch/dl-symbol-redir-ifunc.h=0A=
@@ -22,6 +22,7 @@=0A=
asm ("memset =3D __memset_generic");=0A=
asm ("strlen =3D __strlen_generic");=0A=
#ifndef SHARED=0A=
+asm ("strchr =3D __strchr_generic");=0A=
asm ("memcpy =3D __memcpy_generic");=0A=
asm ("memmove =3D __memmove_generic");=0A=
asm ("memcmp =3D __memcmp_generic");=0A=
diff --git a/sysdeps/aarch64/multiarch/ifunc-impl-list.c b/sysdeps/aarch64/=
multiarch/ifunc-impl-list.c=0A=
index d43f6b58ee075c815b93ab2f42735e7e9f315d32..a0c93147c532d935d562a66ac67=
ebaa5326dd3bd 100644=0A=
--- a/sysdeps/aarch64/multiarch/ifunc-impl-list.c=0A=
+++ b/sysdeps/aarch64/multiarch/ifunc-impl-list.c=0A=
@@ -33,7 +33,6 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifu=
nc_impl *array,=0A=
=0A=
INIT_ARCH ();=0A=
=0A=
- /* Support sysdeps/aarch64/multiarch/memcmp.c, memcpy.c, memmove.c and m=
emset.c. */=0A=
IFUNC_IMPL (i, name, memcmp,=0A=
IFUNC_IMPL_ADD (array, i, memcmp, sve, __memcmp_kunpeng950)=0A=
IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_generic))=0A=
@@ -61,6 +60,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifu=
nc_impl *array,=0A=
IFUNC_IMPL (i, name, strlen,=0A=
IFUNC_IMPL_ADD (array, i, strlen, !mte, __strlen_asimd)=0A=
IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_generic))=0A=
-=0A=
+ IFUNC_IMPL (i, name, strchr,=0A=
+ IFUNC_IMPL_ADD (array, i, strchr, sve2 && !mte, __strchr_sve2)=0A=
+ IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_generic))=0A=
return 0;=0A=
}=0A=
diff --git a/sysdeps/aarch64/multiarch/strchr.c b/sysdeps/aarch64/multiarch=
/strchr.c=0A=
new file mode 100644=0A=
index 0000000000000000000000000000000000000000..12f4e10bbe034ab462a4e578c69=
fd1d9b4afd9ed=0A=
--- /dev/null=0A=
+++ b/sysdeps/aarch64/multiarch/strchr.c=0A=
@@ -0,0 +1,38 @@=0A=
+/* Multiple versions of strchr. AArch64 version.=0A=
+ Copyright (C) 2026 Free Software Foundation, Inc.=0A=
+ This file is part of the GNU C Library.=0A=
+=0A=
+ The GNU C Library is free software; you can redistribute it and/or=0A=
+ modify it under the terms of the GNU Lesser General Public=0A=
+ License as published by the Free Software Foundation; either=0A=
+ version 2.1 of the License, or (at your option) any later version.=0A=
+=0A=
+ The GNU C Library is distributed in the hope that it will be useful,=0A=
+ but WITHOUT ANY WARRANTY; without even the implied warranty of=0A=
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU=0A=
+ Lesser General Public License for more details.=0A=
+=0A=
+ You should have received a copy of the GNU Lesser General Public=0A=
+ License along with the GNU C Library; if not, see=0A=
+ <https://www.gnu.org/licenses/>. */=0A=
+=0A=
+/* Define multiple versions only for the definition in libc. */=0A=
+=0A=
+#if IS_IN (libc)=0A=
+/* Redefine strchr so that the compiler won't complain about the type=0A=
+ mismatch with the IFUNC selector in strong_alias, below. */=0A=
+# undef strchr=0A=
+# define strchr __redirect_strchr=0A=
+# include <string.h>=0A=
+# include <init-arch.h>=0A=
+=0A=
+extern __typeof (__redirect_strchr) __libc_strchr;=0A=
+=0A=
+extern __typeof (__redirect_strchr) __strchr_generic attribute_hidden;=0A=
+extern __typeof (__redirect_strchr) __strchr_sve2 attribute_hidden;=0A=
+=0A=
+libc_ifunc (__libc_strchr, (sve2 && !mte ? __strchr_sve2 : __strchr_generi=
c));=0A=
+=0A=
+# undef strchr=0A=
+strong_alias (__libc_strchr, strchr);=0A=
+#endif=0A=
diff --git a/sysdeps/aarch64/multiarch/strchr_generic.S b/sysdeps/aarch64/m=
ultiarch/strchr_generic.S=0A=
new file mode 100644=0A=
index 0000000000000000000000000000000000000000..d72b0be62a89f4e5b090f9d5c19=
73088bad89704=0A=
--- /dev/null=0A=
+++ b/sysdeps/aarch64/multiarch/strchr_generic.S=0A=
@@ -0,0 +1,35 @@=0A=
+/* A generic optimized strchr implementation for AArch64.=0A=
+ Copyright (C) 2026 Free Software Foundation, Inc.=0A=
+ This file is part of the GNU C Library.=0A=
+=0A=
+ The GNU C Library is free software; you can redistribute it and/or=0A=
+ modify it under the terms of the GNU Lesser General Public=0A=
+ License as published by the Free Software Foundation; either=0A=
+ version 2.1 of the License, or (at your option) any later version.=0A=
+=0A=
+ The GNU C Library is distributed in the hope that it will be useful,=0A=
+ but WITHOUT ANY WARRANTY; without even the implied warranty of=0A=
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU=0A=
+ Lesser General Public License for more details.=0A=
+=0A=
+ You should have received a copy of the GNU Lesser General Public=0A=
+ License along with the GNU C Library; if not, see=0A=
+ <https://www.gnu.org/licenses/>. */=0A=
+=0A=
+#include <sysdep.h>=0A=
+=0A=
+#if IS_IN (libc)=0A=
+=0A=
+# define STRCHR __strchr_generic=0A=
+=0A=
+/* Do not hide the generic version of strchr, we use it internally. */=0A=
+# undef libc_hidden_builtin_def=0A=
+# define libc_hidden_builtin_def(name)=0A=
+=0A=
+# ifdef SHARED=0A=
+/* It doesn't make sense to send libc-internal strchr calls through a PLT.=
*/=0A=
+ .globl __GI_strchr; __GI_strchr =3D __strchr_generic=0A=
+# endif=0A=
+#endif=0A=
+=0A=
+#include "../strchr.S"=0A=
diff --git a/sysdeps/aarch64/multiarch/strchr_sve2.S b/sysdeps/aarch64/mult=
iarch/strchr_sve2.S=0A=
new file mode 100644=0A=
index 0000000000000000000000000000000000000000..4f24c65e2915a14286a0ea922ae=
8325bde3561c1=0A=
--- /dev/null=0A=
+++ b/sysdeps/aarch64/multiarch/strchr_sve2.S=0A=
@@ -0,0 +1,116 @@=0A=
+/* Optimized strchr for SVE2.=0A=
+ Copyright (C) 2026 Free Software Foundation, Inc.=0A=
+=0A=
+ This file is part of the GNU C Library.=0A=
+=0A=
+ The GNU C Library is free software; you can redistribute it and/or=0A=
+ modify it under the terms of the GNU Lesser General Public=0A=
+ License as published by the Free Software Foundation; either=0A=
+ version 2.1 of the License, or (at your option) any later version.=0A=
+=0A=
+ The GNU C Library is distributed in the hope that it will be useful,=0A=
+ but WITHOUT ANY WARRANTY; without even the implied warranty of=0A=
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU=0A=
+ Lesser General Public License for more details.=0A=
+=0A=
+ You should have received a copy of the GNU Lesser General Public=0A=
+ License along with the GNU C Library. If not, see=0A=
+ <https://www.gnu.org/licenses/>. */=0A=
+=0A=
+#include <sysdep.h>=0A=
+=0A=
+/* Assumptions:=0A=
+ *=0A=
+ * SVE2, unaligned accesses.=0A=
+ *=0A=
+ */=0A=
+=0A=
+.arch armv9-a+sve2=0A=
+=0A=
+#if defined (BUILD_STRCHRNUL)=0A=
+#define STRCHR __strchrnul_sve2=0A=
+#else=0A=
+#define STRCHR __strchr_sve2=0A=
+#endif=0A=
+=0A=
+#define chrin w1 // int chr.=0A=
+#define result x0 // Return.=0A=
+=0A=
+#define src x0 // Current data pointer.=0A=
+#define vlalign x2 // Alignment mask.=0A=
+#define tmp x3=0A=
+=0A=
+#define zdata z0 // Data.=0A=
+#define zchr_nul z1 // chr and NUL data for search.=0A=
+=0A=
+#define pall p0 // All active predicate.=0A=
+#define pchr_nul p1 // chr and NUL search result.=0A=
+#define pchr p2 // chr search result.=0A=
+#define psrc p3 // Mask of chars before srcin.=0A=
+#define pfirst p4 // First found.=0A=
+=0A=
+ENTRY (STRCHR)=0A=
+ ptrue pall.b=0A=
+ and chrin, chrin, 0xff // Set zeros after first byte.=0A=
+ dup zchr_nul.h, chrin // Initialize chr and NUL for search.=0A=
+=0A=
+ and tmp, src, 0xfff // Check for pagecross of 2VL reads.=0A=
+ incb tmp, all, mul #2=0A=
+ tbnz tmp, 12, L(pagecross)=0A=
+=0A=
+ ld1b {zdata.b}, pall/z, [src]=0A=
+ match pchr_nul.b, pall/z, zdata.b, zchr_nul.b=0A=
+ b.any L(firstmatch)=0A=
+ ld1b {zdata.b}, pall/z, [src, 1, mul vl]=0A=
+ match pchr_nul.b, pall/z, zdata.b, zchr_nul.b=0A=
+ b.none L(loop_start)=0A=
+ incb src=0A=
+L(firstmatch):=0A=
+ brkb pfirst.b, pall/z, pchr_nul.b // Find first element.=0A=
+ incp src, pfirst.b // Increment by element index.=0A=
+#if !defined (BUILD_STRCHRNUL)=0A=
+ cbz chrin, 1f // Special case NUL chrin.=0A=
+ cmpne pchr.b, pchr_nul/z, zdata.b, 0 // Matched chrin?=0A=
+ csel result, src, xzr, first // If it is NUL, return null.=0A=
+1: ret=0A=
+ nop=0A=
+#else=0A=
+ ret=0A=
+#endif=0A=
+L(loop_start):=0A=
+ mov vlalign, 0=0A=
+ decb vlalign=0A=
+ and src, src, vlalign // Align src to VL.=0A=
+L(loop):=0A=
+ ld1b {zdata.b}, pall/z, [src, 2, mul vl]=0A=
+ incb src, all, mul #2=0A=
+ match pchr_nul.b, pall/z, zdata.b, zchr_nul.b=0A=
+ b.any L(loop_end)=0A=
+L(loop2):=0A=
+ ld1b {zdata.b}, pall/z, [src, 1, mul vl]=0A=
+ match pchr_nul.b, pall/z, zdata.b, zchr_nul.b=0A=
+ b.none L(loop)=0A=
+ incb src=0A=
+L(loop_end):=0A=
+ brkb pfirst.b, pall/z, pchr_nul.b // Find first element.=0A=
+ incp src, pfirst.b // Increment by element index.=0A=
+#if !defined (BUILD_STRCHRNUL)=0A=
+ cbz chrin, 2f // Special case NUL chrin.=0A=
+ cmpne pchr.b, pchr_nul/z, zdata.b, 0 // Matched chrin?=0A=
+ csel result, src, xzr, first // If it is NUL, return null.=0A=
+#endif=0A=
+2: ret=0A=
+=0A=
+ .p2align 3=0A=
+L(pagecross):=0A=
+ mov vlalign, 0=0A=
+ decb vlalign=0A=
+ orn tmp, src, vlalign // Find end of VL aligned src.=0A=
+ whilels psrc.b, src, tmp // Mask off elements.=0A=
+ ld1b {zdata.b}, psrc/z, [src]=0A=
+ match pchr_nul.b, psrc/z, zdata.b, zchr_nul.b=0A=
+ b.any L(firstmatch)=0A=
+ and src, src, vlalign // Align src to VL.=0A=
+ b L(loop2)=0A=
+=0A=
+END (STRCHR)=0A=
diff --git a/sysdeps/aarch64/strchr.S b/sysdeps/aarch64/strchr.S=0A=
index e793fa92f15b63495eab94c83534c5830258bfec..daa28acbfc8b278897d47ec403d=
c945eb3ba3555 100644=0A=
--- a/sysdeps/aarch64/strchr.S=0A=
+++ b/sysdeps/aarch64/strchr.S=0A=
@@ -26,6 +26,10 @@=0A=
* MTE compatible.=0A=
*/=0A=
=0A=
+#ifndef STRCHR=0A=
+# define STRCHR strchr=0A=
+#endif=0A=
+=0A=
#define srcin x0=0A=
#define chrin w1=0A=
#define result x0=0A=
@@ -51,7 +55,7 @@=0A=
zeroes gives the position of the matching byte if it is a multiple of 4=
.=0A=
If it is not a multiple of 4, there was no match. */=0A=
=0A=
-ENTRY (strchr)=0A=
+ENTRY (STRCHR)=0A=
bic src, srcin, 15=0A=
dup vrepchr.16b, chrin=0A=
ld1 {vdata.16b}, [src]=0A=
@@ -109,6 +113,6 @@ L(end):=0A=
csel result, result, xzr, eq=0A=
ret=0A=
=0A=
-END (strchr)=0A=
-libc_hidden_builtin_def (strchr)=0A=
-weak_alias (strchr, index)=0A=
+END (STRCHR)=0A=
+libc_hidden_builtin_def (STRCHR)=0A=
+weak_alias (STRCHR, index)=0A=
=0A=