[gcc r17-3018] aarch64: Implement vector DImode high-part multiply using SVE
Kyrylo Tkachov via Gcc-cvs <[email protected]> Thu, 6 Aug 2026 09:29:06 +0000 (GMT)
| Newsgroups | gmane.comp.gcc.cvs |
|---|---|
| Message-ID | <[email protected]> |
https://gcc.gnu.org/g:75ee89676c8c61263b21c1bbf90ad4fd42c417c1 commit r17-3018-g75ee89676c8c61263b21c1bbf90ad4fd42c417c1 Author: Kyrylo Tkachov <[email protected]> Date: Thu Aug 6 07:13:11 2026 +0000 aarch64: Implement vector DImode high-part multiply using SVE Advanced SIMD has no 64x64 high-part multiply, so the vectorizer currently cannot use its modes for loops such as: for (int i = 0; i < n; i++) d[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64); SVE has UMULH and SMULH for D elements and the Z registers overlap the V registers, so the Advanced SIMD optab can be implemented on top of the SVE instruction. Extend the high-part multiply patterns from SVE_I to SVE_I_SIMD_DI, exactly as mul<mode>3 already does for vector DImode multiplication, and print the operands with %Z so that the V2DI form uses the Z register names. For the loop above on a 128-bit SVE implementation we now generate: ldr q31, [x1, x4] ldr q30, [x2, x4] umulh z30.d, z31.d, z30.d str q30, [x0, x4] rather than: ldr x5, [x1, x4] ldr x6, [x2, x4] umulh x5, x5, x6 str x5, [x0, x4] I don't think people often run the testuite with -march=armv8.2-a+sve -mautovec-preference=asimd-only and I don't know how clean it is, so I've added the runtime test generically to torture and included it in gcc.target/aarch64/ with the aarch64 flags added on top. Bootstrapped and regression-tested on aarch64-unknown-linux-gnu. gcc/ChangeLog: * config/aarch64/aarch64-sve.md (<su>mul<mode>3_highpart): Extend from SVE_I to SVE_I_SIMD_DI. (@aarch64_pred_<optab><mode>): Likewise for the MUL_HIGHPART form, and print the data operands with %Z. * config/aarch64/aarch64-sve2.md (@aarch64_sve2_<optab><mode>): Likewise. gcc/testsuite/ChangeLog: * gcc.target/aarch64/sve/mul_highpart_v2di_1.c: New test. * gcc.target/aarch64/sve/mul_highpart_v2di_2.c: New test. * gcc.target/aarch64/sve/mul_highpart_v2di_3.c: New test. * gcc.target/aarch64/sve/mul_highpart_v2di_run.c: New test. * gcc.dg/torture/mul-highpart-1.c: New test. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/config/aarch64/aarch64-sve.md | 29 +++++----- gcc/config/aarch64/aarch64-sve2.md | 13 ++--- gcc/testsuite/gcc.dg/torture/mul-highpart-1.c | 62 ++++++++++++++++++++++ .../gcc.target/aarch64/sve/mul_highpart_v2di_1.c | 37 +++++++++++++ .../gcc.target/aarch64/sve/mul_highpart_v2di_2.c | 35 ++++++++++++ .../gcc.target/aarch64/sve/mul_highpart_v2di_3.c | 26 +++++++++ .../gcc.target/aarch64/sve/mul_highpart_v2di_run.c | 7 +++ 7 files changed, 190 insertions(+), 19 deletions(-) diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md index f665417dba41..5f9a19c42e96 100644 --- a/gcc/config/aarch64/aarch64-sve.md +++ b/gcc/config/aarch64/aarch64-sve.md @@ -4881,13 +4881,16 @@ ;; ------------------------------------------------------------------------- ;; Unpredicated highpart multiplication. +;; Advanced SIMD has no vector DImode high-part multiply, but SVE does. +;; Make use of the overlap between Z and V registers to implement the V2DI +;; optab for TARGET_SVE, in the same way as the mul<mode>3 expander above. (define_expand "<su>mul<mode>3_highpart" - [(set (match_operand:SVE_I 0 "register_operand") - (unspec:SVE_I + [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand") + (unspec:SVE_I_SIMD_DI [(match_dup 3) - (unspec:SVE_I - [(match_operand:SVE_I 1 "register_operand") - (match_operand:SVE_I 2 "register_operand")] + (unspec:SVE_I_SIMD_DI + [(match_operand:SVE_I_SIMD_DI 1 "register_operand") + (match_operand:SVE_I_SIMD_DI 2 "register_operand")] MUL_HIGHPART)] UNSPEC_PRED_X))] "TARGET_SVE" @@ -4898,22 +4901,22 @@ ;; Predicated highpart multiplication. (define_insn_and_split "@aarch64_pred_<optab><mode>" - [(set (match_operand:SVE_I 0 "register_operand") - (unspec:SVE_I + [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand") + (unspec:SVE_I_SIMD_DI [(match_operand:<VPRED> 1 "register_operand") - (unspec:SVE_I - [(match_operand:SVE_I 2 "register_operand") - (match_operand:SVE_I 3 "register_operand")] + (unspec:SVE_I_SIMD_DI + [(match_operand:SVE_I_SIMD_DI 2 "register_operand") + (match_operand:SVE_I_SIMD_DI 3 "register_operand")] MUL_HIGHPART)] UNSPEC_PRED_X))] "TARGET_SVE" {@ [ cons: =0 , 1 , %2 , 3 ; attrs: movprfx ] - [ w , Upl , 0 , w ; * ] <su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> - [ ?&w , Upl , w , w ; yes ] movprfx\t%0, %2\;<su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> + [ w , Upl , 0 , w ; * ] <su>mulh\t%Z0.<Vetype>, %1/m, %Z0.<Vetype>, %Z3.<Vetype> + [ ?&w , Upl , w , w ; yes ] movprfx\t%Z0, %Z2\;<su>mulh\t%Z0.<Vetype>, %1/m, %Z0.<Vetype>, %Z3.<Vetype> } "TARGET_SVE2" [(set (match_dup 0) - (unspec:SVE_I + (unspec:SVE_I_SIMD_DI [(match_dup 2) (match_dup 3)] MUL_HIGHPART))] diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md index 33b39412e366..fe6aa65823d6 100644 --- a/gcc/config/aarch64/aarch64-sve2.md +++ b/gcc/config/aarch64/aarch64-sve2.md @@ -983,15 +983,16 @@ ;; ---- [INT] Unpredicated high-part multiplication ;; ------------------------------------------------------------------------- -;; SVE2 unpredicated SMULH/UMULH. +;; SVE2 unpredicated SMULH/UMULH. V2DI is included so that the Advanced +;; SIMD high-part multiply optabs can be implemented on top of SVE. (define_insn "@aarch64_sve2_<optab><mode>" - [(set (match_operand:SVE_I 0 "register_operand" "=w") - (unspec:SVE_I - [(match_operand:SVE_I 1 "register_operand" "w") - (match_operand:SVE_I 2 "register_operand" "w")] + [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand" "=w") + (unspec:SVE_I_SIMD_DI + [(match_operand:SVE_I_SIMD_DI 1 "register_operand" "w") + (match_operand:SVE_I_SIMD_DI 2 "register_operand" "w")] MUL_HIGHPART))] "TARGET_SVE2" - "<su>mulh\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" + "<su>mulh\t%Z0.<Vetype>, %Z1.<Vetype>, %Z2.<Vetype>" [(set_attr "sve_type" "sve_int_mul")] ) diff --git a/gcc/testsuite/gcc.dg/torture/mul-highpart-1.c b/gcc/testsuite/gcc.dg/torture/mul-highpart-1.c new file mode 100644 index 000000000000..0c6b78128632 --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/mul-highpart-1.c @@ -0,0 +1,62 @@ +/* Check that a 64-bit high-part multiply gives the same answer whether or not + it is vectorized. */ +/* { dg-do run } */ +/* { dg-require-effective-target int128 } */ +/* { dg-additional-options "-ftree-vectorize" } */ + +#include <stdint.h> + +#define N 77 + +static int64_t sa[N], sb[N], sd[N]; +static uint64_t ua[N], ub[N], ud[N]; + +void __attribute__ ((noipa)) +mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b, + int count) +{ + for (int i = 0; i < count; ++i) + dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64); +} + +void __attribute__ ((noipa)) +mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b, + int count) +{ + for (int i = 0; i < count; ++i) + dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64); +} + +int +main (void) +{ + uint64_t s = 0x243f6a8885a308d3ULL; + for (int i = 0; i < N; ++i) + { + s ^= s << 13; s ^= s >> 7; s ^= s << 17; + sa[i] = (int64_t) s; + ua[i] = s; + s ^= s << 13; s ^= s >> 7; s ^= s << 17; + sb[i] = (int64_t) s; + ub[i] = s; + } + /* Boundary values. */ + sa[0] = INT64_MIN; sb[0] = INT64_MIN; + sa[1] = INT64_MIN; sb[1] = -1; + sa[2] = -1; sb[2] = -1; + ua[0] = 0; ub[0] = ~(uint64_t) 0; + ua[1] = ~(uint64_t) 0; ub[1] = ~(uint64_t) 0; + ua[2] = (uint64_t) 1 << 63; ub[2] = (uint64_t) 1 << 63; + + mulh_s64 (sd, sa, sb, N); + mulh_u64 (ud, ua, ub, N); + + for (int i = 0; i < N; ++i) + { + if (sd[i] != (int64_t) (((__int128) sa[i] * sb[i]) >> 64)) + __builtin_abort (); + if (ud[i] != (uint64_t) (((unsigned __int128) ua[i] * ub[i]) >> 64)) + __builtin_abort (); + } + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c new file mode 100644 index 000000000000..2544e4c6b79d --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c @@ -0,0 +1,37 @@ +/* Advanced SIMD has no 64-bit high-part multiply, but SVE does, and the two + register files overlap. Check that the SVE instruction is used for a + 128-bit vector high-part multiply when the vectoriser is restricted to + Advanced SIMD modes. */ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve -mautovec-preference=asimd-only" } */ +/* { dg-final { check-function-bodies "**" "" } } */ + +#include <stdint.h> + +/* +** mulh_s64: +** ... +** smulh z[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d +** ... +*/ +void __attribute__ ((noipa)) +mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b, + int count) +{ + for (int i = 0; i < count; ++i) + dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64); +} + +/* +** mulh_u64: +** ... +** umulh z[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d +** ... +*/ +void __attribute__ ((noipa)) +mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b, + int count) +{ + for (int i = 0; i < count; ++i) + dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64); +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c new file mode 100644 index 000000000000..3f3989fa901e --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c @@ -0,0 +1,35 @@ +/* Same as mul_highpart_v2di_1.c, but for SVE2, where the high-part multiply + has an unpredicated form. */ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-preference=asimd-only" } */ +/* { dg-final { check-function-bodies "**" "" } } */ + +#include <stdint.h> + +/* +** mulh_s64: +** ... +** smulh z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d +** ... +*/ +void __attribute__ ((noipa)) +mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b, + int count) +{ + for (int i = 0; i < count; ++i) + dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64); +} + +/* +** mulh_u64: +** ... +** umulh z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d +** ... +*/ +void __attribute__ ((noipa)) +mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b, + int count) +{ + for (int i = 0; i < count; ++i) + dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64); +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c new file mode 100644 index 000000000000..b459bbc27cc1 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c @@ -0,0 +1,26 @@ +/* The Shoup modular multiply used by lattice cryptography. The high-part + multiply is the only operation Advanced SIMD lacks, so without a 128-bit + vector form of it the whole loop stays scalar. */ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-preference=asimd-only" } */ +/* { dg-final { check-function-bodies "**" "" } } */ + +#include <stdint.h> + +/* +** mul_mod: +** ... +** umulh z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d +** ... +*/ +void __attribute__ ((noipa)) +mul_mod (uint64_t *restrict dst, const uint64_t *restrict src, + uint64_t operand, uint64_t quotient, uint64_t modulus, int count) +{ + for (int i = 0; i < count; ++i) + { + uint64_t hi = (uint64_t) (((unsigned __int128) src[i] * quotient) >> 64); + uint64_t t = operand * src[i] - hi * modulus; + dst[i] = t >= modulus ? t - modulus : t; + } +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c new file mode 100644 index 000000000000..a77f2ef22074 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c @@ -0,0 +1,7 @@ +/* Run the generic high-part multiply correctness test with the vectoriser + restricted to Advanced SIMD modes, so that the V2DI patterns are the ones + being exercised. */ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve -mautovec-preference=asimd-only" } */ + +#include "../../../gcc.dg/torture/mul-highpart-1.c"