[glibc] riscv: Add RVV memcpy for both multiarch and non-multiarch builds
Peter Bergner via Glibc-cvs <[email protected]>
| Newsgroups | gmane.comp.lib.glibc.cvs |
|---|---|
| Message-ID | <[email protected]> |
https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=81eb2bb5ef4385fe792b506178fad92efc9908a3 commit 81eb2bb5ef4385fe792b506178fad92efc9908a3 Author: Yao Zihong <[email protected]> Date: Tue Apr 21 14:58:10 2026 -0500 riscv: Add RVV memcpy for both multiarch and non-multiarch builds This patch adds an RVV-optimized implementation of memcpy for RISC-V and enables it for both multiarch (IFUNC) and non-multiarch builds. The implementation integrates Hau Hsu's 2023 RVV work under a unified ifunc-based framework. A vectorized version (__memcpy_vector) is added alongside the generic fallback (__memcpy_generic). The runtime resolver selects the RVV variant when RISCV_HWPROBE_KEY_IMA_EXT_0 reports vector support (RVV). Currently, the resolver still selects the RVV variant even when the RVV extension is disabled via prctl(). As a consequence, any process that has RVV disabled via prctl() will receive SIGILL when calling memcpy(). Co-authored-by: Hau Hsu <[email protected]> Co-authored-by: Jerry Shih <[email protected]> Signed-off-by: Yao Zihong <[email protected]> Reviewed-by: Peter Bergner <[email protected]> Diff: --- sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h | 1 + sysdeps/riscv/multiarch/memcpy-vector.S | 24 ++++++++++ sysdeps/riscv/rvv/memcpy.S | 54 ++++++++++++++++++++++ sysdeps/unix/sysv/linux/riscv/multiarch/Makefile | 1 + .../sysv/linux/riscv/multiarch/ifunc-impl-list.c | 2 + sysdeps/unix/sysv/linux/riscv/multiarch/memcpy.c | 5 ++ 6 files changed, 87 insertions(+) diff --git a/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h b/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h index 69a0790838..a4ff47107b 100644 --- a/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h +++ b/sysdeps/riscv/multiarch/dl-symbol-redir-ifunc.h @@ -20,6 +20,7 @@ #define _DL_IFUNC_GENERIC_H #ifndef SHARED +asm ("memcpy = __memcpy_generic"); asm ("memset = __memset_generic"); asm ("memcpy = __memcpy_generic"); #endif diff --git a/sysdeps/riscv/multiarch/memcpy-vector.S b/sysdeps/riscv/multiarch/memcpy-vector.S new file mode 100644 index 0000000000..97b80b8069 --- /dev/null +++ b/sysdeps/riscv/multiarch/memcpy-vector.S @@ -0,0 +1,24 @@ +/* Re-include the RISC-V RVV based memcpy implementation. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#if IS_IN(libc) +# define MEMCPY __memcpy_vector +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(name) +# include <sysdeps/riscv/rvv/memcpy.S> +#endif diff --git a/sysdeps/riscv/rvv/memcpy.S b/sysdeps/riscv/rvv/memcpy.S new file mode 100644 index 0000000000..b4d600c59d --- /dev/null +++ b/sysdeps/riscv/rvv/memcpy.S @@ -0,0 +1,54 @@ +/* RISC-V RVV based memcpy. + Copyright (C) 2026 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <sys/asm.h> + +#ifndef MEMCPY +# define MEMCPY memcpy +#endif + +#define dst a0 +#define src a1 +#define num a2 + +#define ivl a3 +#define dst_ptr a4 + +#define ELEM_LMUL_SETTING m8 +#define vdata v0 + +ENTRY (MEMCPY) +.option push +.option arch, +v + mv dst_ptr, dst +L(loop): + vsetvli ivl, num, e8, ELEM_LMUL_SETTING, ta, ma + + vle8.v vdata, (src) + sub num, num, ivl + add src, src, ivl + vse8.v vdata, (dst_ptr) + add dst_ptr, dst_ptr, ivl + + bnez num, L(loop) + + ret +.option pop +END (MEMCPY) +libc_hidden_builtin_def (memcpy) diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile index c82693251a..32be517d21 100644 --- a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile @@ -2,6 +2,7 @@ ifeq ($(subdir),string) sysdep_routines += \ memcpy \ memcpy-generic \ + memcpy-vector \ memcpy_noalignment \ memset \ memset-generic \ diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c index f4154bb8f7..0eb55a5ad8 100644 --- a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c @@ -44,6 +44,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, } IFUNC_IMPL (i, name, memcpy, + IFUNC_IMPL_ADD (array, i, memcpy, rvv_enabled, + __memcpy_vector) IFUNC_IMPL_ADD (array, i, memcpy, fast_unaligned, __memcpy_noalignment) IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_generic)) diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/memcpy.c b/sysdeps/unix/sysv/linux/riscv/multiarch/memcpy.c index f096693431..11919d815a 100644 --- a/sysdeps/unix/sysv/linux/riscv/multiarch/memcpy.c +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/memcpy.c @@ -32,11 +32,16 @@ extern __typeof (__redirect_memcpy) __libc_memcpy; extern __typeof (__redirect_memcpy) __memcpy_generic attribute_hidden; extern __typeof (__redirect_memcpy) __memcpy_noalignment attribute_hidden; +extern __typeof (__redirect_memcpy) __memcpy_vector attribute_hidden; static inline __typeof (__redirect_memcpy) * select_memcpy_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func) { unsigned long long int v; + if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_IMA_EXT_0, &v) == 0 + && (v & RISCV_HWPROBE_IMA_V) == RISCV_HWPROBE_IMA_V) + return __memcpy_vector; + if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_CPUPERF_0, &v) == 0 && (v & RISCV_HWPROBE_MISALIGNED_MASK) == RISCV_HWPROBE_MISALIGNED_FAST) return __memcpy_noalignment;