[PATCH 1/4] riscv: use unaligned vector memory access when supported
Jussi Kivilinna <[email protected]> Wed, 29 Jul 2026 19:34:21 +0300
| Newsgroups | gmane.comp.encryption.gpg.libgcrypt.devel |
|---|---|
| Message-ID | <[email protected]> |
* cipher/Makefile.am: Use -mstrict-align only for plain RVV implementations; add Zvkb to RVV-crypto -march flags. * cipher/chacha20-riscv-v.c (unaligned_load_u32m1) (unaligned_store_u32m1): New. (chacha20_rvv_blocks): Use unaligned load/store helpers. * cipher/cipher-gcm-riscv-zvkg.c (unaligned_load_u32m1): Use element width access when RVV_UNALIGNED_NOT_ALLOWED is not defined. * cipher/rijndael-riscv-zvkned.c (unaligned_load_u64m1) (unaligned_load_u32m1, unaligned_store_u32m1, unaligned_load_u32m2) (unaligned_store_u32m2, unaligned_load_u32m4, unaligned_store_u32m4): Likewise. * cipher/sha256-riscv-zvknha-zvkb.c (load_and_swap): Likewise; drop 'vl_bytes' parameter. (sha256_transform_zvknha_zvkb): Remove 'vl_bytes'. * cipher/sha512-riscv-zvknhb-zvkb.c: Likewise. * cipher/simd-common-riscv.h (RVV_UNALIGNED_NOT_ALLOWED): New. * configure.ac: Replace -mstrict-align compiler support check with Zicclsm enabled check; add Zicclsm to MARCH_RVA23U64_BASE and Zvkb to MARCH_RVA23U64_WITH_VEC_CRYPTO; drop -mstrict-align from RVV-crypto check flags; add vrev8 to RVV-crypto intrinsics test. -- -mstrict-align was added for GCC-14 code generation bug, but is really needed for SpacemiT K1 which does not support unaligned vector memory access. K1 does not have vector cryptography extensions, so RVV-crypto implementations can expect unaligned vector access to work. Detect support through Zicclsm extension and keep -mstrict-align for plain RVV implementations. Signed-off-by: Jussi Kivilinna <[email protected]> --- cipher/Makefile.am | 22 +- cipher/chacha20-riscv-v.c | 258 ++++++++++----------- cipher/cipher-gcm-riscv-zvkg.c | 5 +- cipher/rijndael-riscv-zvkned.c | 366 +++++++++++++++--------------- cipher/sha256-riscv-zvknha-zvkb.c | 21 +- cipher/sha512-riscv-zvknhb-zvkb.c | 21 +- cipher/simd-common-riscv.h | 13 ++ configure.ac | 62 +++-- 8 files changed, 399 insertions(+), 369 deletions(-) diff --git a/cipher/Makefile.am b/cipher/Makefile.am index 021a7ab6..0c00d181 100644 --- a/cipher/Makefile.am +++ b/cipher/Makefile.am @@ -368,16 +368,15 @@ serpent-avx512-x86.o: $(srcdir)/serpent-avx512-x86.c Makefile serpent-avx512-x86.lo: $(srcdir)/serpent-avx512-x86.c Makefile `echo $(LTCOMPILE) $(avx512f_cflags) -c $< | $(instrumentation_munging) ` -# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads) if ENABLE_RISCV_VECTOR_INTRINSICS_EXTRA_CFLAGS +# Note: -mstrict-align needed for SpacemiT K1 RVV bug (K1 does not support +# unaligned vector memory access). This is enabled only for plain RVV +# implementations, RVV-crypto extension implementations expect CPU with proper +# support for unaligned vector memory access. riscv_vector_cflags = -O2 -march=@MARCH_RVA22U64_WITH_VEC@ -mstrict-align else -if SUPPORT_CC_RISCV_MSTRICT_ALIGN -riscv_vector_cflags = -O2 -mstrict-align -else riscv_vector_cflags = endif -endif chacha20-riscv-v.o: $(srcdir)/chacha20-riscv-v.c Makefile `echo $(COMPILE) $(riscv_vector_cflags) -c $< | $(instrumentation_munging) ` @@ -391,22 +390,15 @@ rijndael-vp-riscv.o: $(srcdir)/rijndael-vp-riscv.c Makefile rijndael-vp-riscv.lo: $(srcdir)/rijndael-vp-riscv.c Makefile `echo $(LTCOMPILE) $(riscv_vector_cflags) -c $< | $(instrumentation_munging) ` -# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads) if ENABLE_RISCV_VECTOR_CRYPTO_INTRINSICS_EXTRA_CFLAGS -riscv_vector_crypto_aes_cflags = -O2 -march=@MARCH_RVA23U64_BASE@_zvkned -mstrict-align -riscv_vector_crypto_sha_cflags = -O2 -march=@MARCH_RVA23U64_BASE@_zvknha_zvknhb_zvkb -mstrict-align -riscv_vector_crypto_gcm_cflags = -O2 -march=@MARCH_RVA23U64_BASE@_zvkg -mstrict-align -else -if SUPPORT_CC_RISCV_MSTRICT_ALIGN -riscv_vector_crypto_aes_cflags = -O2 -mstrict-align -riscv_vector_crypto_sha_cflags = -O2 -mstrict-align -riscv_vector_crypto_gcm_cflags = -O2 -mstrict-align +riscv_vector_crypto_aes_cflags = -O2 -march=@MARCH_RVA23U64_BASE@_zvkned_zvkb +riscv_vector_crypto_sha_cflags = -O2 -march=@MARCH_RVA23U64_BASE@_zvknha_zvknhb_zvkb +riscv_vector_crypto_gcm_cflags = -O2 -march=@MARCH_RVA23U64_BASE@_zvkg_zvkb else riscv_vector_crypto_aes_cflags = riscv_vector_crypto_sha_cflags = riscv_vector_crypto_gcm_cflags = endif -endif cipher-gcm-riscv-zvkg.o: $(srcdir)/cipher-gcm-riscv-zvkg.c Makefile `echo $(COMPILE) $(riscv_vector_crypto_gcm_cflags) -c $< | $(instrumentation_munging) ` diff --git a/cipher/chacha20-riscv-v.c b/cipher/chacha20-riscv-v.c index 1304a333..94026305 100644 --- a/cipher/chacha20-riscv-v.c +++ b/cipher/chacha20-riscv-v.c @@ -151,85 +151,113 @@ gen_indexes(size_t vl, size_t stride) return __riscv_vadd_vv_u16m2(idx_hi, idx_lo, vl * 4); } +static ASM_FUNC_ATTR_INLINE vuint32m1_t +unaligned_load_u32m1(const void *src, size_t vl) +{ +#ifdef RVV_UNALIGNED_NOT_ALLOWED + return __riscv_vreinterpret_v_u8m1_u32m1(__riscv_vle8_v_u8m1(src, vl * 4)); +#else + return __riscv_vle32_v_u32m1(src, vl); +#endif +} + +static ASM_FUNC_ATTR_INLINE void +unaligned_store_u32m1(void *dst, vuint32m1_t vec, size_t vl) +{ +#ifdef RVV_UNALIGNED_NOT_ALLOWED + __riscv_vse8_v_u8m1(dst, __riscv_vreinterpret_v_u32m1_u8m1(vec), vl * 4); +#else + __riscv_vse32_v_u32m1(dst, vec, vl); +#endif +} + static ASM_FUNC_ATTR_INLINE vuint32m1x8_t unaligned_vlsseg8e32_v_u32m1x8(const void *src, size_t vl) { - const byte *bsrc = src; - vuint16m2_t indexes; - vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7; - vuint32m1x8_t data; - - if (LIKELY(((uintptr_t)src & 3) == 0)) +#ifdef RVV_UNALIGNED_NOT_ALLOWED + if (UNLIKELY(((uintptr_t)src & 3) != 0)) + { + const byte *bsrc = src; + vuint16m2_t indexes; + vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7; + vuint32m1x8_t data; + + indexes = gen_indexes(4 * vl, 64); + + b0 = __riscv_vluxei16_v_u8m1(bsrc + 0 * 4, indexes, vl * 4); + b1 = __riscv_vluxei16_v_u8m1(bsrc + 1 * 4, indexes, vl * 4); + b2 = __riscv_vluxei16_v_u8m1(bsrc + 2 * 4, indexes, vl * 4); + b3 = __riscv_vluxei16_v_u8m1(bsrc + 3 * 4, indexes, vl * 4); + b4 = __riscv_vluxei16_v_u8m1(bsrc + 4 * 4, indexes, vl * 4); + b5 = __riscv_vluxei16_v_u8m1(bsrc + 5 * 4, indexes, vl * 4); + b6 = __riscv_vluxei16_v_u8m1(bsrc + 6 * 4, indexes, vl * 4); + b7 = __riscv_vluxei16_v_u8m1(bsrc + 7 * 4, indexes, vl * 4); + + data = __riscv_vundefined_u32m1x8(); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 0, __riscv_vreinterpret_v_u8m1_u32m1(b0)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 1, __riscv_vreinterpret_v_u8m1_u32m1(b1)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 2, __riscv_vreinterpret_v_u8m1_u32m1(b2)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 3, __riscv_vreinterpret_v_u8m1_u32m1(b3)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 4, __riscv_vreinterpret_v_u8m1_u32m1(b4)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 5, __riscv_vreinterpret_v_u8m1_u32m1(b5)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 6, __riscv_vreinterpret_v_u8m1_u32m1(b6)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 7, __riscv_vreinterpret_v_u8m1_u32m1(b7)); + + return data; + } + else +#endif { /* Fast path for 32-bit aligned loads. */ return __riscv_vlsseg8e32_v_u32m1x8(src, 64, vl); } - - indexes = gen_indexes(4 * vl, 64); - - b0 = __riscv_vluxei16_v_u8m1(bsrc + 0 * 4, indexes, vl * 4); - b1 = __riscv_vluxei16_v_u8m1(bsrc + 1 * 4, indexes, vl * 4); - b2 = __riscv_vluxei16_v_u8m1(bsrc + 2 * 4, indexes, vl * 4); - b3 = __riscv_vluxei16_v_u8m1(bsrc + 3 * 4, indexes, vl * 4); - b4 = __riscv_vluxei16_v_u8m1(bsrc + 4 * 4, indexes, vl * 4); - b5 = __riscv_vluxei16_v_u8m1(bsrc + 5 * 4, indexes, vl * 4); - b6 = __riscv_vluxei16_v_u8m1(bsrc + 6 * 4, indexes, vl * 4); - b7 = __riscv_vluxei16_v_u8m1(bsrc + 7 * 4, indexes, vl * 4); - - data = __riscv_vundefined_u32m1x8(); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 0, __riscv_vreinterpret_v_u8m1_u32m1(b0)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 1, __riscv_vreinterpret_v_u8m1_u32m1(b1)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 2, __riscv_vreinterpret_v_u8m1_u32m1(b2)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 3, __riscv_vreinterpret_v_u8m1_u32m1(b3)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 4, __riscv_vreinterpret_v_u8m1_u32m1(b4)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 5, __riscv_vreinterpret_v_u8m1_u32m1(b5)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 6, __riscv_vreinterpret_v_u8m1_u32m1(b6)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 7, __riscv_vreinterpret_v_u8m1_u32m1(b7)); - - return data; } static ASM_FUNC_ATTR_INLINE void unaligned_vssseg8e32_v_u32m1x8(void *dst, vuint32m1x8_t data, size_t vl) { - byte *bdst = dst; - vuint16m2_t indexes; - vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7; - - if (LIKELY(((uintptr_t)dst & 3) == 0)) +#ifdef RVV_UNALIGNED_NOT_ALLOWED + if (UNLIKELY(((uintptr_t)dst & 3) != 0)) + { + byte *bdst = dst; + vuint16m2_t indexes; + vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7; + + indexes = gen_indexes(4 * vl, 64); + + b0 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 0)); + b1 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 1)); + b2 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 2)); + b3 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 3)); + b4 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 4)); + b5 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 5)); + b6 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 6)); + b7 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 7)); + + __riscv_vsuxei16_v_u8m1(bdst + 0 * 4, indexes, b0, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 1 * 4, indexes, b1, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 2 * 4, indexes, b2, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 3 * 4, indexes, b3, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 4 * 4, indexes, b4, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 5 * 4, indexes, b5, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 6 * 4, indexes, b6, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 7 * 4, indexes, b7, vl * 4); + } + else +#endif { /* Fast path for 32-bit aligned stores. */ __riscv_vssseg8e32_v_u32m1x8(dst, 64, data, vl); return; } - - indexes = gen_indexes(4 * vl, 64); - - b0 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 0)); - b1 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 1)); - b2 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 2)); - b3 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 3)); - b4 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 4)); - b5 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 5)); - b6 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 6)); - b7 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 7)); - - __riscv_vsuxei16_v_u8m1(bdst + 0 * 4, indexes, b0, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 1 * 4, indexes, b1, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 2 * 4, indexes, b2, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 3 * 4, indexes, b3, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 4 * 4, indexes, b4, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 5 * 4, indexes, b5, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 6 * 4, indexes, b6, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 7 * 4, indexes, b7, vl * 4); } static ASM_FUNC_ATTR_INLINE unsigned int @@ -385,8 +413,8 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) vuint32m1_t v0, v1, v2, v3; vuint32m1_t v4, v5, v6, v7; vuint32m1_t state0, state1, state2, state3; - vuint8m1_t i0, i1, i2, i3; - vuint8m1_t i4, i5, i6, i7; + vuint32m1_t i0, i1, i2, i3; + vuint32m1_t i4, i5, i6, i7; vuint16m1_t rot16 = gen_rot16(vl); vuint8m1_t rot8 = gen_rot8(vl); @@ -417,10 +445,10 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) v7 = state3; v7 = ADD_U64(v7, one_u64); - i0 = __riscv_vle8_v_u8m1(src + 0 * 16, vl * 4); - i1 = __riscv_vle8_v_u8m1(src + 1 * 16, vl * 4); - i2 = __riscv_vle8_v_u8m1(src + 2 * 16, vl * 4); - i3 = __riscv_vle8_v_u8m1(src + 3 * 16, vl * 4); + i0 = unaligned_load_u32m1(src + 0 * 16, vl); + i1 = unaligned_load_u32m1(src + 1 * 16, vl); + i2 = unaligned_load_u32m1(src + 2 * 16, vl); + i3 = unaligned_load_u32m1(src + 3 * 16, vl); for (i = 20; i > 0; i -= 2) { @@ -434,14 +462,10 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) v3 = __riscv_vadd_vv_u32m1(v3, state3, vl); state3 = ADD_U64(state3, one_u64); - v0 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i0), - v0, vl); - v1 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i1), - v1, vl); - v2 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i2), - v2, vl); - v3 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i3), - v3, vl); + v0 = __riscv_vxor_vv_u32m1(i0, v0, vl); + v1 = __riscv_vxor_vv_u32m1(i1, v1, vl); + v2 = __riscv_vxor_vv_u32m1(i2, v2, vl); + v3 = __riscv_vxor_vv_u32m1(i3, v3, vl); v4 = __riscv_vadd_vv_u32m1(v4, state0, vl); v5 = __riscv_vadd_vv_u32m1(v5, state1, vl); @@ -449,37 +473,25 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) v7 = __riscv_vadd_vv_u32m1(v7, state3, vl); state3 = ADD_U64(state3, one_u64); - i4 = __riscv_vle8_v_u8m1(src + 4 * 16, vl * 4); - i5 = __riscv_vle8_v_u8m1(src + 5 * 16, vl * 4); - i6 = __riscv_vle8_v_u8m1(src + 6 * 16, vl * 4); - i7 = __riscv_vle8_v_u8m1(src + 7 * 16, vl * 4); - - __riscv_vse8_v_u8m1(dst + 0 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v0), vl * 4); - __riscv_vse8_v_u8m1(dst + 1 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v1), vl * 4); - __riscv_vse8_v_u8m1(dst + 2 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v2), vl * 4); - __riscv_vse8_v_u8m1(dst + 3 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v3), vl * 4); - - v4 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i4), - v4, vl); - v5 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i5), - v5, vl); - v6 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i6), - v6, vl); - v7 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i7), - v7, vl); - - __riscv_vse8_v_u8m1(dst + 4 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v4), vl * 4); - __riscv_vse8_v_u8m1(dst + 5 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v5), vl * 4); - __riscv_vse8_v_u8m1(dst + 6 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v6), vl * 4); - __riscv_vse8_v_u8m1(dst + 7 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v7), vl * 4); + i4 = unaligned_load_u32m1(src + 4 * 16, vl); + i5 = unaligned_load_u32m1(src + 5 * 16, vl); + i6 = unaligned_load_u32m1(src + 6 * 16, vl); + i7 = unaligned_load_u32m1(src + 7 * 16, vl); + + unaligned_store_u32m1(dst + 0 * 16, v0, vl); + unaligned_store_u32m1(dst + 1 * 16, v1, vl); + unaligned_store_u32m1(dst + 2 * 16, v2, vl); + unaligned_store_u32m1(dst + 3 * 16, v3, vl); + + v4 = __riscv_vxor_vv_u32m1(i4, v4, vl); + v5 = __riscv_vxor_vv_u32m1(i5, v5, vl); + v6 = __riscv_vxor_vv_u32m1(i6, v6, vl); + v7 = __riscv_vxor_vv_u32m1(i7, v7, vl); + + unaligned_store_u32m1(dst + 4 * 16, v4, vl); + unaligned_store_u32m1(dst + 5 * 16, v5, vl); + unaligned_store_u32m1(dst + 6 * 16, v6, vl); + unaligned_store_u32m1(dst + 7 * 16, v7, vl); src += 2 * 64; dst += 2 * 64; @@ -495,10 +507,10 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) v2 = state2; v3 = state3; - i0 = __riscv_vle8_v_u8m1(src + 0 * 16, vl * 4); - i1 = __riscv_vle8_v_u8m1(src + 1 * 16, vl * 4); - i2 = __riscv_vle8_v_u8m1(src + 2 * 16, vl * 4); - i3 = __riscv_vle8_v_u8m1(src + 3 * 16, vl * 4); + i0 = unaligned_load_u32m1(src + 0 * 16, vl); + i1 = unaligned_load_u32m1(src + 1 * 16, vl); + i2 = unaligned_load_u32m1(src + 2 * 16, vl); + i3 = unaligned_load_u32m1(src + 3 * 16, vl); for (i = 20; i > 0; i -= 2) { @@ -513,22 +525,14 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) state3 = ADD_U64(state3, one_u64); - v0 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i0), - v0, vl); - v1 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i1), - v1, vl); - v2 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i2), - v2, vl); - v3 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i3), - v3, vl); - __riscv_vse8_v_u8m1(dst + 0 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v0), vl * 4); - __riscv_vse8_v_u8m1(dst + 1 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v1), vl * 4); - __riscv_vse8_v_u8m1(dst + 2 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v2), vl * 4); - __riscv_vse8_v_u8m1(dst + 3 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v3), vl * 4); + v0 = __riscv_vxor_vv_u32m1(i0, v0, vl); + v1 = __riscv_vxor_vv_u32m1(i1, v1, vl); + v2 = __riscv_vxor_vv_u32m1(i2, v2, vl); + v3 = __riscv_vxor_vv_u32m1(i3, v3, vl); + unaligned_store_u32m1(dst + 0 * 16, v0, vl); + unaligned_store_u32m1(dst + 1 * 16, v1, vl); + unaligned_store_u32m1(dst + 2 * 16, v2, vl); + unaligned_store_u32m1(dst + 3 * 16, v3, vl); src += 64; dst += 64; diff --git a/cipher/cipher-gcm-riscv-zvkg.c b/cipher/cipher-gcm-riscv-zvkg.c index c459a6fb..1f0b3797 100644 --- a/cipher/cipher-gcm-riscv-zvkg.c +++ b/cipher/cipher-gcm-riscv-zvkg.c @@ -44,9 +44,12 @@ static ASM_FUNC_ATTR_INLINE vuint32m1_t unaligned_load_u32m1(const void *ptr, size_t vl_u32) { +#ifdef RVV_UNALIGNED_NOT_ALLOWED size_t vl_bytes = vl_u32 * 4; - return cast_u8m1_u32m1(__riscv_vle8_v_u8m1(ptr, vl_bytes)); +#else + return __riscv_vle32_v_u32m1(ptr, vl_u32); +#endif } static ASM_FUNC_ATTR_INLINE vuint32m1_t diff --git a/cipher/rijndael-riscv-zvkned.c b/cipher/rijndael-riscv-zvkned.c index ed3887ae..ee9c2de4 100644 --- a/cipher/rijndael-riscv-zvkned.c +++ b/cipher/rijndael-riscv-zvkned.c @@ -83,54 +83,74 @@ bswap128_u32m1(vuint32m1_t vec, size_t vl_u32) __riscv_vrgather_vv_u8m1(cast_u32m1_u8m1(vec), bswap128, vl_bytes)); } +static ASM_FUNC_ATTR_INLINE vuint64m1_t +unaligned_load_u64m1(const void *ptr, size_t vl_u64) +{ +#ifdef RVV_UNALIGNED_NOT_ALLOWED + return cast_u8m1_u64m1(__riscv_vle8_v_u8m1(ptr, vl_u64 * 8)); +#else + return __riscv_vle64_v_u64m1(ptr, vl_u64); +#endif +} + static ASM_FUNC_ATTR_INLINE vuint32m1_t unaligned_load_u32m1(const void *ptr, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - return cast_u8m1_u32m1(__riscv_vle8_v_u8m1(ptr, vl_bytes)); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + return cast_u8m1_u32m1(__riscv_vle8_v_u8m1(ptr, vl_u32 * 4)); +#else + return __riscv_vle32_v_u32m1(ptr, vl_u32); +#endif } static ASM_FUNC_ATTR_INLINE void unaligned_store_u32m1(void *ptr, vuint32m1_t vec, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - __riscv_vse8_v_u8m1(ptr, cast_u32m1_u8m1(vec), vl_bytes); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + __riscv_vse8_v_u8m1(ptr, cast_u32m1_u8m1(vec), vl_u32 * 4); +#else + __riscv_vse32_v_u32m1(ptr, vec, vl_u32); +#endif } -static ASM_FUNC_ATTR_INLINE vuint32m4_t -unaligned_load_u32m4(const void *ptr, size_t vl_u32) +static ASM_FUNC_ATTR_INLINE vuint32m2_t +unaligned_load_u32m2(const void *ptr, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - return cast_u8m4_u32m4(__riscv_vle8_v_u8m4(ptr, vl_bytes)); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + return cast_u8m2_u32m2(__riscv_vle8_v_u8m2(ptr, vl_u32 * 4)); +#else + return __riscv_vle32_v_u32m2(ptr, vl_u32); +#endif } static ASM_FUNC_ATTR_INLINE void -unaligned_store_u32m4(void *ptr, vuint32m4_t vec, size_t vl_u32) +unaligned_store_u32m2(void *ptr, vuint32m2_t vec, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - __riscv_vse8_v_u8m4(ptr, cast_u32m4_u8m4(vec), vl_bytes); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + __riscv_vse8_v_u8m2(ptr, cast_u32m2_u8m2(vec), vl_u32 * 4); +#else + __riscv_vse32_v_u32m2(ptr, vec, vl_u32); +#endif } -static ASM_FUNC_ATTR_INLINE vuint32m1_t -vxor_u8_u32m1(vuint32m1_t a, vuint32m1_t b, size_t vl_u32) +static ASM_FUNC_ATTR_INLINE vuint32m4_t +unaligned_load_u32m4(const void *ptr, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - return cast_u8m1_u32m1(__riscv_vxor_vv_u8m1(cast_u32m1_u8m1(a), - cast_u32m1_u8m1(b), vl_bytes)); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + return cast_u8m4_u32m4(__riscv_vle8_v_u8m4(ptr, vl_u32 * 4)); +#else + return __riscv_vle32_v_u32m4(ptr, vl_u32); +#endif } -static ASM_FUNC_ATTR_INLINE vuint32m4_t -vxor_u8_u32m4(vuint32m4_t a, vuint32m4_t b, size_t vl_u32) +static ASM_FUNC_ATTR_INLINE void +unaligned_store_u32m4(void *ptr, vuint32m4_t vec, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - return cast_u8m4_u32m4(__riscv_vxor_vv_u8m4(cast_u32m4_u8m4(a), - cast_u32m4_u8m4(b), vl_bytes)); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + __riscv_vse8_v_u8m4(ptr, cast_u32m4_u8m4(vec), vl_u32 * 4); +#else + __riscv_vse32_v_u32m4(ptr, vec, vl_u32); +#endif } static ASM_FUNC_ATTR_INLINE vuint32m4_t @@ -660,29 +680,28 @@ _gcry_aes_riscv_zvkned_cfb_enc (void *context, unsigned char *iv_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t iv; ROUND_KEY_VARIABLES; PRELOAD_ROUND_KEYS (rk, rounds, vl); - iv = unaligned_load_u32m1(iv_arg, vl); + iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl); for (; nblocks; nblocks--) { - vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes); + vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); AES_CRYPT(e, m1, rounds, iv, vl); - data = __riscv_vxor_vv_u8m1(cast_u32m1_u8m1(iv), data, vl_bytes); - __riscv_vse8_v_u8m1(outbuf, data, vl_bytes); - iv = cast_u8m1_u32m1(data); + data = __riscv_vxor_vv_u32m1(iv, data, vl); + unaligned_store_u32m1(outbuf, data, vl); + iv = data; outbuf += BLOCKSIZE; inbuf += BLOCKSIZE; } - unaligned_store_u32m1(iv_arg, iv, vl); + __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl); clear_vec_regs(); } @@ -699,29 +718,27 @@ _gcry_aes_riscv_zvkned_cbc_enc (void *context, unsigned char *iv_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t iv; ROUND_KEY_VARIABLES; PRELOAD_ROUND_KEYS (rk, rounds, vl); - iv = unaligned_load_u32m1(iv_arg, vl); + iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl); for (; nblocks; nblocks--) { - vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes); - iv = cast_u8m1_u32m1( - __riscv_vxor_vv_u8m1(data, cast_u32m1_u8m1(iv), vl_bytes)); + vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); + iv = __riscv_vxor_vv_u32m1(data, iv, vl); AES_CRYPT(e, m1, rounds, iv, vl); - __riscv_vse8_v_u8m1(outbuf, cast_u32m1_u8m1(iv), vl_bytes); + unaligned_store_u32m1(outbuf, iv, vl); inbuf += BLOCKSIZE; outbuf += outbuf_add; } - unaligned_store_u32m1(iv_arg, iv, vl); + __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl); clear_vec_regs(); } @@ -731,7 +748,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, void *outbuf_arg, const void *inbuf_arg, size_t nblocks) { - static const byte add_u8_array[4][16] = + static const byte add_u8_array[4][16] __attribute__ ((aligned (16))) = { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 }, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 }, @@ -745,7 +762,6 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; u64 ctrlow; vuint32m1_t ctr; vuint32m1_t add1; @@ -753,20 +769,17 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); - add1 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[0], vl_bytes)); - ctr = unaligned_load_u32m1(ctr_arg, vl); + add1 = __riscv_vle32_v_u32m1((const void *)add_u8_array[0], vl); + ctr = __riscv_vle32_v_u32m1((void *)ctr_arg, vl); ctrlow = __riscv_vmv_x_s_u64m1_u64(cast_u32m1_u64m1(bswap128_u32m1(ctr, vl))); memory_barrier_with_vec(add1); if (nblocks >= 4) { - vuint32m1_t add2 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[1], - vl_bytes)); - vuint32m1_t add3 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[2], - vl_bytes)); - vuint32m1_t add4 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[3], - vl_bytes)); + vuint32m1_t add2 = __riscv_vle32_v_u32m1((const void *)add_u8_array[1], vl); + vuint32m1_t add3 = __riscv_vle32_v_u32m1((const void *)add_u8_array[2], vl); + vuint32m1_t add4 = __riscv_vle32_v_u32m1((const void *)add_u8_array[3], vl); memory_barrier_with_vec(add2); memory_barrier_with_vec(add3); @@ -774,7 +787,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, for (; nblocks >= 4; nblocks -= 4) { - vuint8m4_t data4blks; + vuint32m4_t data4blks; vuint32m4_t ctr4blks; /* detect if 8-bit carry handling is needed */ @@ -819,13 +832,12 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, ctr4blks = merge_4x_u32m1_to_u32m4(ctr0, ctr1, ctr2, ctr3); } - data4blks = __riscv_vle8_v_u8m4(inbuf, vl_bytes * 4); + data4blks = unaligned_load_u32m4(inbuf, vl * 4); AES_CRYPT(e, m4, rounds, ctr4blks, vl * 4); - data4blks = __riscv_vxor_vv_u8m4(cast_u32m4_u8m4(ctr4blks), data4blks, - vl_bytes * 4); - __riscv_vse8_v_u8m4(outbuf, data4blks, vl_bytes * 4); + data4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4); + unaligned_store_u32m4(outbuf, data4blks, vl * 4); inbuf += 4 * BLOCKSIZE; outbuf += 4 * BLOCKSIZE; @@ -835,7 +847,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, for (; nblocks; nblocks--) { vuint32m1_t block = ctr; - vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes); + vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); /* detect if 8-bit carry handling is needed */ if (UNLIKELY((++ctrlow & 0xff) == 0)) @@ -859,14 +871,14 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, AES_CRYPT(e, m1, rounds, block, vl); - data = __riscv_vxor_vv_u8m1(cast_u32m1_u8m1(block), data, vl_bytes); - __riscv_vse8_v_u8m1(outbuf, data, vl_bytes); + data = __riscv_vxor_vv_u32m1(block, data, vl); + unaligned_store_u32m1(outbuf, data, vl); inbuf += BLOCKSIZE; outbuf += BLOCKSIZE; } - unaligned_store_u32m1(ctr_arg, ctr, vl); + __riscv_vse32_v_u32m1((void *)ctr_arg, ctr, vl); clear_vec_regs(); } @@ -886,7 +898,6 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t ctr; vuint32m1_t add1; ROUND_KEY_VARIABLES; @@ -894,7 +905,7 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); add1 = __riscv_vle32_v_u32m1(add_u32_array[0], vl); - ctr = unaligned_load_u32m1(ctr_arg, vl); + ctr = __riscv_vle32_v_u32m1((void *)ctr_arg, vl); memory_barrier_with_vec(add1); @@ -914,18 +925,17 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg, vuint32m1_t ctr2 = __riscv_vadd_vv_u32m1(ctr, add2, vl); vuint32m1_t ctr3 = __riscv_vadd_vv_u32m1(ctr, add3, vl); vuint32m4_t ctr4blks; - vuint8m4_t data4blks; + vuint32m4_t data4blks; ctr4blks = merge_4x_u32m1_to_u32m4(ctr, ctr1, ctr2, ctr3); ctr = __riscv_vadd_vv_u32m1(ctr, add4, vl); - data4blks = __riscv_vle8_v_u8m4(inbuf, vl_bytes * 4); + data4blks = unaligned_load_u32m4(inbuf, vl * 4); AES_CRYPT(e, m4, rounds, ctr4blks, vl * 4); - data4blks = __riscv_vxor_vv_u8m4(cast_u32m4_u8m4(ctr4blks), data4blks, - vl_bytes * 4); - __riscv_vse8_v_u8m4(outbuf, data4blks, vl_bytes * 4); + data4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4); + unaligned_store_u32m4(outbuf, data4blks, vl * 4); inbuf += 4 * BLOCKSIZE; outbuf += 4 * BLOCKSIZE; @@ -935,20 +945,20 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg, for (; nblocks; nblocks--) { vuint32m1_t block = ctr; - vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes); + vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); ctr = __riscv_vadd_vv_u32m1(ctr, add1, vl); AES_CRYPT(e, m1, rounds, block, vl); - data = __riscv_vxor_vv_u8m1(cast_u32m1_u8m1(block), data, vl_bytes); - __riscv_vse8_v_u8m1(outbuf, data, vl_bytes); + data = __riscv_vxor_vv_u32m1(block, data, vl); + unaligned_store_u32m1(outbuf, data, vl); inbuf += BLOCKSIZE; outbuf += BLOCKSIZE; } - unaligned_store_u32m1(ctr_arg, ctr, vl); + __riscv_vse32_v_u32m1((void *)ctr_arg, ctr, vl); clear_vec_regs(); } @@ -969,7 +979,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); - iv = unaligned_load_u32m1(iv_arg, vl); + iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl); for (; nblocks >= 4; nblocks -= 4) { @@ -983,7 +993,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg, AES_CRYPT(e, m4, rounds, iv4blks, vl * 4); - data4blks = vxor_u8_u32m4(iv4blks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(iv4blks, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); inbuf += 4 * BLOCKSIZE; @@ -997,7 +1007,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg, AES_CRYPT(e, m1, rounds, iv, vl); - data = vxor_u8_u32m1(iv, data, vl); + data = __riscv_vxor_vv_u32m1(iv, data, vl); unaligned_store_u32m1(outbuf, data, vl); iv = new_iv; @@ -1005,7 +1015,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg, outbuf += BLOCKSIZE; } - unaligned_store_u32m1(iv_arg, iv, vl); + __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl); clear_vec_regs(); } @@ -1032,7 +1042,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); - iv = unaligned_load_u32m1(iv_arg, vl); + iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl); for (; nblocks >= 4; nblocks -= 4) { @@ -1045,7 +1055,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg, AES_CRYPT(d, m4, rounds, data4blks, vl * 4); - data4blks = vxor_u8_u32m4(iv4blks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(iv4blks, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); inbuf += 4 * BLOCKSIZE; @@ -1059,7 +1069,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg, AES_CRYPT(d, m1, rounds, data, vl); - data = vxor_u8_u32m1(iv, data, vl); + data = __riscv_vxor_vv_u32m1(iv, data, vl); unaligned_store_u32m1(outbuf, data, vl); iv = new_iv; @@ -1067,7 +1077,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg, outbuf += BLOCKSIZE; } - unaligned_store_u32m1(iv_arg, iv, vl); + __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl); clear_vec_regs(); } @@ -1083,7 +1093,6 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t iv; vuint32m1_t ctr; ROUND_KEY_VARIABLES; @@ -1091,8 +1100,8 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); /* Preload Offset and Checksum */ - iv = unaligned_load_u32m1(c->u_iv.iv, vl); - ctr = unaligned_load_u32m1(c->u_ctr.ctr, vl); + iv = __riscv_vle32_v_u32m1((void *)c->u_iv.iv, vl); + ctr = __riscv_vle32_v_u32m1((void *)c->u_ctr.ctr, vl); if (nblocks >= 4) { @@ -1101,44 +1110,44 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, for (; nblocks >= 4; nblocks -= 4) { - const unsigned char *l; - vuint8m1_t l_ntzi; + const void *l; + vuint32m1_t l_ntzi; vuint32m4_t data4blks = unaligned_load_u32m4(inbuf, vl * 4); vuint32m1_t offset0, offset1, offset2, offset3; vuint32m4_t offsets; /* Checksum_i = Checksum_{i-1} xor P_i */ - ctr4blks = vxor_u8_u32m4(ctr4blks, data4blks, vl * 4); + ctr4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4); /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */ l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset0 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset1 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset2 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset3 = iv; offsets = merge_4x_u32m1_to_u32m4(offset0, offset1, offset2, offset3); - data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4); AES_CRYPT(e, m4, rounds, data4blks, vl * 4); - data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); @@ -1149,35 +1158,35 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, /* Checksum_i = Checksum_{i-1} xor P_i */ { vuint32m1x4_t ctr0123 = split_u32m4_to_4x_u32m1(ctr4blks); - ctr = vxor_u8_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), + ctr = __riscv_vxor_vv_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), __riscv_vget_v_u32m1x4_u32m1(ctr0123, 1), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); } } for (; nblocks; nblocks--) { - const unsigned char *l; - vuint8m1_t l_ntzi; + const void *l; + vuint32m1_t l_ntzi; vuint32m1_t data; data = unaligned_load_u32m1(inbuf, vl); /* Checksum_i = Checksum_{i-1} xor P_i */ - ctr = vxor_u8_u32m1(ctr, data, vl); + ctr = __riscv_vxor_vv_u32m1(ctr, data, vl); /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */ l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); - data = vxor_u8_u32m1(data, iv, vl); + data = __riscv_vxor_vv_u32m1(data, iv, vl); AES_CRYPT(e, m1, rounds, data, vl); - data = vxor_u8_u32m1(iv, data, vl); + data = __riscv_vxor_vv_u32m1(iv, data, vl); unaligned_store_u32m1(outbuf, data, vl); inbuf += BLOCKSIZE; @@ -1186,8 +1195,8 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, c->u_mode.ocb.data_nblocks = n; - unaligned_store_u32m1(c->u_iv.iv, iv, vl); - unaligned_store_u32m1(c->u_ctr.ctr, ctr, vl); + __riscv_vse32_v_u32m1((void *)c->u_iv.iv, iv, vl); + __riscv_vse32_v_u32m1((void *)c->u_ctr.ctr, ctr, vl); clear_vec_regs(); @@ -1205,7 +1214,6 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, const u32 *rk = ctx->keyschdec32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t iv; vuint32m1_t ctr; ROUND_KEY_VARIABLES; @@ -1219,8 +1227,8 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); /* Preload Offset and Checksum */ - iv = unaligned_load_u32m1(c->u_iv.iv, vl); - ctr = unaligned_load_u32m1(c->u_ctr.ctr, vl); + iv = __riscv_vle32_v_u32m1((void *)c->u_iv.iv, vl); + ctr = __riscv_vle32_v_u32m1((void *)c->u_ctr.ctr, vl); if (nblocks >= 4) { @@ -1229,8 +1237,8 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, for (; nblocks >= 4; nblocks -= 4) { - const unsigned char *l; - vuint8m1_t l_ntzi; + const void *l; + vuint32m1_t l_ntzi; vuint32m4_t data4blks = unaligned_load_u32m4(inbuf, vl * 4); vuint32m1_t offset0, offset1, offset2, offset3; vuint32m4_t offsets; @@ -1238,37 +1246,37 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* P_i = Offset_i xor ENCIPHER(K, C_i xor Offset_i) */ l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset0 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset1 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset2 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset3 = iv; offsets = merge_4x_u32m1_to_u32m4(offset0, offset1, offset2, offset3); - data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4); AES_CRYPT(d, m4, rounds, data4blks, vl * 4); - data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); /* Checksum_i = Checksum_{i-1} xor P_i */ - ctr4blks = vxor_u8_u32m4(ctr4blks, data4blks, vl * 4); + ctr4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4); inbuf += 4 * BLOCKSIZE; outbuf += 4 * BLOCKSIZE; @@ -1277,37 +1285,34 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, /* Checksum_i = Checksum_{i-1} xor P_i */ { vuint32m1x4_t ctr0123 = split_u32m4_to_4x_u32m1(ctr4blks); - ctr = vxor_u8_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), + ctr = __riscv_vxor_vv_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), __riscv_vget_v_u32m1x4_u32m1(ctr0123, 1), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); } } for (; nblocks; nblocks--) { - const unsigned char *l; - vuint8m1_t l_ntzi; - vuint8m1_t data; - vuint32m1_t block; - - l = ocb_get_l(c, ++n); + const void *l; + vuint32m1_t l_ntzi; + vuint32m1_t data; /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */ - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - data = __riscv_vle8_v_u8m1(inbuf, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); - data = __riscv_vxor_vv_u8m1(data, cast_u32m1_u8m1(iv), vl_bytes); - block = cast_u8m1_u32m1(data); + l = ocb_get_l(c, ++n); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + data = unaligned_load_u32m1(inbuf, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); + data = __riscv_vxor_vv_u32m1(data, iv, vl); - AES_CRYPT(d, m1, rounds, block, vl); + AES_CRYPT(d, m1, rounds, data, vl); - block = vxor_u8_u32m1(iv, block, vl); - unaligned_store_u32m1(outbuf, block, vl); + data = __riscv_vxor_vv_u32m1(iv, data, vl); + unaligned_store_u32m1(outbuf, data, vl); /* Checksum_i = Checksum_{i-1} xor P_i */ - ctr = vxor_u8_u32m1(ctr, block, vl); + ctr = __riscv_vxor_vv_u32m1(ctr, data, vl); inbuf += BLOCKSIZE; outbuf += BLOCKSIZE; @@ -1315,8 +1320,8 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, c->u_mode.ocb.data_nblocks = n; - unaligned_store_u32m1(c->u_iv.iv, iv, vl); - unaligned_store_u32m1(c->u_ctr.ctr, ctr, vl); + __riscv_vse32_v_u32m1((void *)c->u_iv.iv, iv, vl); + __riscv_vse32_v_u32m1((void *)c->u_ctr.ctr, ctr, vl); clear_vec_regs(); @@ -1344,7 +1349,6 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t iv; vuint32m1_t ctr; ROUND_KEY_VARIABLES; @@ -1352,8 +1356,8 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); /* Preload Offset and Sum */ - iv = unaligned_load_u32m1(c->u_mode.ocb.aad_offset, vl); - ctr = unaligned_load_u32m1(c->u_mode.ocb.aad_sum, vl); + iv = __riscv_vle32_v_u32m1((void *)c->u_mode.ocb.aad_offset, vl); + ctr = __riscv_vle32_v_u32m1((void *)c->u_mode.ocb.aad_sum, vl); if (nblocks >= 4) { @@ -1362,8 +1366,8 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, for (; nblocks >= 4; nblocks -= 4) { - const unsigned char *l; - vuint8m1_t l_ntzi; + const void *l; + vuint32m1_t l_ntzi; vuint32m4_t data4blks = unaligned_load_u32m4(abuf, vl * 4); vuint32m1_t offset0, offset1, offset2, offset3; vuint32m4_t offsets; @@ -1371,32 +1375,32 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */ l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset0 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset1 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset2 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset3 = iv; offsets = merge_4x_u32m1_to_u32m4(offset0, offset1, offset2, offset3); - data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4); AES_CRYPT(e, m4, rounds, data4blks, vl * 4); - ctr4blks = vxor_u8_u32m4(ctr4blks, data4blks, vl * 4); + ctr4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4); abuf += 4 * BLOCKSIZE; } @@ -1404,17 +1408,17 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, /* Checksum_i = Checksum_{i-1} xor P_i */ { vuint32m1x4_t ctr0123 = split_u32m4_to_4x_u32m1(ctr4blks); - ctr = vxor_u8_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), + ctr = __riscv_vxor_vv_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), __riscv_vget_v_u32m1x4_u32m1(ctr0123, 1), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); } } for (; nblocks; nblocks--) { - const unsigned char *l; - vuint8m1_t l_ntzi; + const void *l; + vuint32m1_t l_ntzi; vuint32m1_t data; data = unaligned_load_u32m1(abuf, vl); @@ -1422,22 +1426,22 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */ l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); - data = vxor_u8_u32m1(data, iv, vl); + data = __riscv_vxor_vv_u32m1(data, iv, vl); AES_CRYPT(e, m1, rounds, data, vl); - ctr = vxor_u8_u32m1(ctr, data, vl); + ctr = __riscv_vxor_vv_u32m1(ctr, data, vl); abuf += BLOCKSIZE; } c->u_mode.ocb.aad_nblocks = n; - unaligned_store_u32m1(c->u_mode.ocb.aad_offset, iv, vl); - unaligned_store_u32m1(c->u_mode.ocb.aad_sum, ctr, vl); + __riscv_vse32_v_u32m1((void *)c->u_mode.ocb.aad_offset, iv, vl); + __riscv_vse32_v_u32m1((void *)c->u_mode.ocb.aad_sum, ctr, vl); clear_vec_regs(); @@ -1482,7 +1486,7 @@ aes_riscv_xts_enc (void *context, unsigned char *tweak_arg, void *outbuf_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); /* Preload tweak */ - tweak = unaligned_load_u32m1(tweak_arg, vl); + tweak = __riscv_vle32_v_u32m1((void *)tweak_arg, vl); memory_barrier_with_vec(xts_gfmul); memory_barrier_with_vec(xts_swap64); @@ -1504,11 +1508,11 @@ aes_riscv_xts_enc (void *context, unsigned char *tweak_arg, void *outbuf_arg, tweaks = merge_4x_u32m1_to_u32m4(tweak0, tweak1, tweak2, tweak3); - data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4); AES_CRYPT(e, m4, rounds, data4blks, vl * 4); - data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); @@ -1521,19 +1525,19 @@ aes_riscv_xts_enc (void *context, unsigned char *tweak_arg, void *outbuf_arg, vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); vuint32m1_t tweak0 = tweak; - data = vxor_u8_u32m1(data, tweak0, vl); + data = __riscv_vxor_vv_u32m1(data, tweak0, vl); tweak = xts_gfmul_byA(tweak, xts_gfmul, xts_swap64, vl); AES_CRYPT(e, m1, rounds, data, vl); - data = vxor_u8_u32m1(data, tweak0, vl); + data = __riscv_vxor_vv_u32m1(data, tweak0, vl); unaligned_store_u32m1(outbuf, data, vl); inbuf += BLOCKSIZE; outbuf += BLOCKSIZE; } - unaligned_store_u32m1(tweak_arg, tweak, vl); + __riscv_vse32_v_u32m1((void *)tweak_arg, tweak, vl); clear_vec_regs(); } @@ -1562,7 +1566,7 @@ aes_riscv_xts_dec (void *context, unsigned char *tweak_arg, void *outbuf_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); /* Preload tweak */ - tweak = unaligned_load_u32m1(tweak_arg, vl); + tweak = __riscv_vle32_v_u32m1((void *)tweak_arg, vl); memory_barrier_with_vec(xts_gfmul); memory_barrier_with_vec(xts_swap64); @@ -1584,11 +1588,11 @@ aes_riscv_xts_dec (void *context, unsigned char *tweak_arg, void *outbuf_arg, tweaks = merge_4x_u32m1_to_u32m4(tweak0, tweak1, tweak2, tweak3); - data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4); AES_CRYPT(d, m4, rounds, data4blks, vl * 4); - data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); @@ -1601,19 +1605,19 @@ aes_riscv_xts_dec (void *context, unsigned char *tweak_arg, void *outbuf_arg, vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); vuint32m1_t tweak0 = tweak; - data = vxor_u8_u32m1(data, tweak0, vl); + data = __riscv_vxor_vv_u32m1(data, tweak0, vl); tweak = xts_gfmul_byA(tweak, xts_gfmul, xts_swap64, vl); AES_CRYPT(d, m1, rounds, data, vl); - data = vxor_u8_u32m1(data, tweak0, vl); + data = __riscv_vxor_vv_u32m1(data, tweak0, vl); unaligned_store_u32m1(outbuf, data, vl); inbuf += BLOCKSIZE; outbuf += BLOCKSIZE; } - unaligned_store_u32m1(tweak_arg, tweak, vl); + __riscv_vse32_v_u32m1((void *)tweak_arg, tweak, vl); clear_vec_regs(); } diff --git a/cipher/sha256-riscv-zvknha-zvkb.c b/cipher/sha256-riscv-zvknha-zvkb.c index 6375f9aa..0ff80cf7 100644 --- a/cipher/sha256-riscv-zvknha-zvkb.c +++ b/cipher/sha256-riscv-zvknha-zvkb.c @@ -76,11 +76,14 @@ working_vsha2cl_vv_u32m1(vuint32m1_t hgcd, vuint32m1_t feba, static ASM_FUNC_ATTR_INLINE vuint32m1_t -load_and_swap (const byte * p, size_t vl, size_t vl_bytes) +load_and_swap (const byte * p, size_t vl) { - vuint8m1_t temp_bytes = __riscv_vle8_v_u8m1(p, vl_bytes); - return __riscv_vrev8_v_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(temp_bytes), - vl); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + vuint8m1_t temp = __riscv_vle8_v_u8m1(p, vl * 4); + return __riscv_vrev8_v_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(temp), vl); +#else + return __riscv_vrev8_v_u32m1(__riscv_vle32_v_u32m1((const void *)p, vl), vl); +#endif } @@ -111,7 +114,6 @@ sha256_transform_zvknha_zvkb (u32 state[8], const uint8_t * data, static const int feba_offset = 0; static const int hgcd_offset = 8 / sizeof(u32); size_t vl; - size_t vl_bytes; vuint32m1_t idx; vuint32m1_t v_feba_work, v_feba; vuint32m1_t v_hgcd_work, v_hgcd; @@ -121,7 +123,6 @@ sha256_transform_zvknha_zvkb (u32 state[8], const uint8_t * data, vuint32m1_t v_feba_hgcd_idx; vl = 4; - vl_bytes = vl * 4; idx = __riscv_vid_v_u32m1(vl); merge_mask = __riscv_vmseq_vx_u32m1_b32(idx, 0, vl); @@ -137,10 +138,10 @@ sha256_transform_zvknha_zvkb (u32 state[8], const uint8_t * data, v_feba_work = v_feba; v_hgcd_work = v_hgcd; - w0 = load_and_swap(data + 0, vl, vl_bytes); - w1 = load_and_swap(data + 16, vl, vl_bytes); - w2 = load_and_swap(data + 32, vl, vl_bytes); - w3 = load_and_swap(data + 48, vl, vl_bytes); + w0 = load_and_swap(data + 0, vl); + w1 = load_and_swap(data + 16, vl); + w2 = load_and_swap(data + 32, vl); + w3 = load_and_swap(data + 48, vl); QUAD_ROUND_W_SCHED(w0, w1, w2, w3); QUAD_ROUND_W_SCHED(w1, w2, w3, w0); diff --git a/cipher/sha512-riscv-zvknhb-zvkb.c b/cipher/sha512-riscv-zvknhb-zvkb.c index 5f5d483a..fea8f32a 100644 --- a/cipher/sha512-riscv-zvknhb-zvkb.c +++ b/cipher/sha512-riscv-zvknhb-zvkb.c @@ -81,11 +81,14 @@ working_vsha2cl_vv_u64m2(vuint64m2_t hgcd, vuint64m2_t feba, static ASM_FUNC_ATTR_INLINE vuint64m2_t -load_and_swap(const byte *p, size_t vl, size_t vl_bytes) +load_and_swap(const byte *p, size_t vl) { - vuint8m2_t temp_bytes = __riscv_vle8_v_u8m2(p, vl_bytes); - return __riscv_vrev8_v_u64m2(__riscv_vreinterpret_v_u8m2_u64m2(temp_bytes), - vl); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + vuint8m2_t temp = __riscv_vle8_v_u8m2(p, vl * 8); + return __riscv_vrev8_v_u64m2(__riscv_vreinterpret_v_u8m2_u64m2(temp), vl); +#else + return __riscv_vrev8_v_u64m2(__riscv_vle64_v_u64m2((const void *)p, vl), vl); +#endif } @@ -97,7 +100,6 @@ sha512_transform_zvknhb_zvkb (u64 state[8], const byte *data, static const int feba_offset = 0; static const int hgcd_offset = 16 / sizeof(u64); size_t vl; - size_t vl_bytes; vuint64m2_t idx; vuint64m2_t v_feba_work, v_feba; vuint64m2_t v_hgcd_work, v_hgcd; @@ -107,7 +109,6 @@ sha512_transform_zvknhb_zvkb (u64 state[8], const byte *data, vuint64m2_t v_feba_hgcd_idx; vl = 4; - vl_bytes = vl * 8; idx = __riscv_vid_v_u64m2(vl); merge_mask = __riscv_vmseq_vx_u64m2_b32(idx, 0, vl); @@ -124,10 +125,10 @@ sha512_transform_zvknhb_zvkb (u64 state[8], const byte *data, v_feba_work = v_feba; v_hgcd_work = v_hgcd; - w0 = load_and_swap(data + 0, vl, vl_bytes); - w1 = load_and_swap(data + 32, vl, vl_bytes); - w2 = load_and_swap(data + 64, vl, vl_bytes); - w3 = load_and_swap(data + 96, vl, vl_bytes); + w0 = load_and_swap(data + 0, vl); + w1 = load_and_swap(data + 32, vl); + w2 = load_and_swap(data + 64, vl); + w3 = load_and_swap(data + 96, vl); QUAD_ROUND_W_SCHED(w0, w1, w2, w3); QUAD_ROUND_W_SCHED(w1, w2, w3, w0); diff --git a/cipher/simd-common-riscv.h b/cipher/simd-common-riscv.h index 8381000f..8ddd5056 100644 --- a/cipher/simd-common-riscv.h +++ b/cipher/simd-common-riscv.h @@ -23,6 +23,19 @@ #include <config.h> +#undef RVV_UNALIGNED_NOT_ALLOWED +#if defined(HAVE_COMPATIBLE_CC_RISCV_VECTOR_INTRINSICS_WITH_CFLAGS) || \ + !(defined(__riscv_zicclsm) && (__riscv_zicclsm >= 1000000)) || \ + defined(__riscv_misaligned_avoid) +/* Zicclsm extension means that unaligned vector (and scalar) memory accesses + * are allowed. If build configuration is such that RVV intrinsics support had + * to be enabled with custom CFLAGS, assume that vector unaligned accesses are + * not allowed. This is to workaround broken build setups with RVA22 + * non-compliant Spacemit K1 (unaligned scalar allowed, unaligned vector + * not allowed). */ +#define RVV_UNALIGNED_NOT_ALLOWED 1 +#endif + #define memory_barrier_with_vec(a) __asm__("" : "+vr"(a) :: "memory") #define clear_vec_regs() __asm__ volatile("vsetvli zero, %0, e8, m1, ta, ma;\n" \ diff --git a/configure.ac b/configure.ac index 631ed81e..4ac3450a 100644 --- a/configure.ac +++ b/configure.ac @@ -2802,28 +2802,26 @@ fi # -# Check whether compiler supports RISC-V -mstrict-align flag +# Check whether compiler has RISC-V Zicclsm extension enable # -_gcc_cflags_save=$CFLAGS -# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads) -CFLAGS="$CFLAGS -mstrict-align" - -AC_CACHE_CHECK([whether compiler supports RISC-V -mstrict-align flag], - [gcry_cv_cc_riscv_mstrict_align], +AC_CACHE_CHECK([whether compiler has RISC-V Zicclsm extension enabled], + [gcry_cv_cc_riscv_zicclsm_enabled], [if test "$mpi_cpu_arch" != "riscv64" || test "$try_asm_modules" != "yes" ; then - gcry_cv_cc_riscv_mstrict_align="n/a" + gcry_cv_cc_riscv_zicclsm_enabled="n/a" else - gcry_cv_cc_riscv_mstrict_align=no - AC_COMPILE_IFELSE( - [AC_LANG_SOURCE([[void testfn(void) { }]])], - [gcry_cv_cc_riscv_mstrict_align=yes]) + gcry_cv_cc_riscv_zicclsm_enabled=no + AC_LINK_IFELSE([AC_LANG_PROGRAM( + [[#if defined(__riscv_zicclsm) && (__riscv_zicclsm >= 1000000) + int testfunc(void) { return 0; } + #endif + ]], [ testfunc(); ])], + [gcry_cv_cc_riscv_zicclsm_enabled=yes]) fi]) -AM_CONDITIONAL(SUPPORT_CC_RISCV_MSTRICT_ALIGN, - test "$gcry_cv_cc_riscv_mstrict_align" = "yes") - -# Restore flags. -CFLAGS=$_gcc_cflags_save; +if test "$gcry_cv_cc_riscv_zicclsm_enabled" = "yes" ; then + AC_DEFINE(HAVE_COMPATIBLE_CC_RISCV_ZICCLSM_ENABLED,1, + [Defined if underlying compiler has RISC-V Zicclsm extension enabled]) +fi # @@ -2896,7 +2894,10 @@ _gcc_cflags_save=$CFLAGS # better code generation for RISC-V vector implementations. MARCH_RVA22U64_WITH_VEC=rv64imafdcv_zba_zbb_zbs AC_SUBST([MARCH_RVA22U64_WITH_VEC]) -# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads) +# Note: -mstrict-align needed for SpacemiT K1 RVV bug (K1 does not support +# unaligned vector memory access). This is enabled only for plain RVV +# implementations, RVV-crypto extension implementations detect support for +# unaligned vector memory access based on Zicclsm extension. CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA22U64_WITH_VEC -mstrict-align" if test "$gcry_cv_cc_riscv_vector_intrinsics" = "no" && @@ -2992,6 +2993,13 @@ m4_define([GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST], vuint32m1_t d = __riscv_vghsh_vv_u32m1(a, b, c, vl); __riscv_vse32_v_u32m1(ptr + 0 * vl, d, vl); } + void test_rev8(unsigned int *ptr) + { + int vl = __riscv_vsetvl_e32m1 (4); + vuint32m1_t a = __riscv_vle32_v_u32m1(ptr + 0 * vl, vl); + vuint32m1_t b = __riscv_vrev8_v_u32m1(a, vl); + __riscv_vse32_v_u32m1(ptr + 0 * vl, b, vl); + } void test_inline_vec_asm(unsigned int *ptr) { int vl = __riscv_vsetvl_e32m1 (4); @@ -3013,6 +3021,7 @@ m4_define([GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST], test_aes_key(buf); test_aes_crypt(buf); test_ghash(buf); + test_rev8(buf); test_inline_vec_asm(buf); return 0; } @@ -3039,16 +3048,19 @@ fi _gcc_cflags_save=$CFLAGS # Enable B extension (Zba+Zbb+Zbs) to align with RVA23U64 profile and for # better code generation for RISC-V vector implementations. -MARCH_RVA23U64_BASE=${MARCH_RVA22U64_WITH_VEC} -MARCH_RVA23U64_WITH_VEC_CRYPTO=${MARCH_RVA23U64_BASE}_zvbc_zvkg_zvkn_zvks +if test "$gcry_cv_cc_riscv_zicclsm_enabled" = "yes" ; then + MARCH_RVA23U64_BASE=${MARCH_RVA22U64_WITH_VEC}_zicclsm +else + MARCH_RVA23U64_BASE=${MARCH_RVA22U64_WITH_VEC} +fi +MARCH_RVA23U64_WITH_VEC_CRYPTO=${MARCH_RVA23U64_BASE}_zvbc_zvkb_zvkg_zvkn_zvks AC_SUBST([MARCH_RVA23U64_BASE]) -# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads) -CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO -mstrict-align" +CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO" if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics" = "no" && test "$mpi_cpu_arch" = "riscv64" && test "$try_asm_modules" = "yes" ; then - AC_CACHE_CHECK([whether compiler supports RISC-V vector intrinsics with extra GCC flags], + AC_CACHE_CHECK([whether compiler supports RISC-V vector cryptography intrinsics with extra GCC flags], [gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags], [gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags=no AC_LINK_IFELSE( @@ -3082,7 +3094,7 @@ if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics" = "yes" || # Setup flags for test if needed. _gcc_cflags_save=$CFLAGS if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags" = "yes"; then - CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO -mstrict-align" + CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO" fi # Disable LTO optimization for following check to generate @@ -3131,7 +3143,7 @@ if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics" = "yes" || # Setup flags for test. _gcc_cflags_save=$CFLAGS if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags" = "yes"; then - CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO -mstrict-align" + CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO" else CFLAGS="$CFLAGS -O2" fi -- 2.53.0