[PATCH] aarch64: Implement vector DImode high-part multiply using SVE for Advanced SIMD modes
<[email protected]> Thu, 6 Aug 2026 08:45:15 +0200
| Newsgroups | gmane.comp.gcc.patches |
|---|---|
| Message-ID | <[email protected]> |
From: Kyrylo Tkachov <[email protected]> Advanced SIMD has no 64x64 high-part multiply, so the vectorizer currently cannot use its modes for loops such as: for (int i = 0; i < n; i++) d[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64); SVE has UMULH and SMULH for D elements and the Z registers overlap the V registers, so the Advanced SIMD optab can be implemented on top of the SVE instruction. Extend the high-part multiply patterns from SVE_I to SVE_I_SIMD_DI, exactly as mul<mode>3 already does for vector DImode multiplication, and print the operands with %Z so that the V2DI form uses the Z register names. For the loop above on a 128-bit SVE implementation, we now get ldr q31, [x1, x4] umulh x5, x5, x6 ldr q30, [x2, x4] instead of str x5, [x0, x4] umulh z30.d, z31.d, z30.d str q30, [x0, x4] Bootstrapped and regression-tested on aarch64-unknown-linux-gnu. Ok for trunk? Thanks, Kyrill gcc/ChangeLog: * config/aarch64/aarch64-sve.md (<su>mul<mode>3_highpart): Extend from SVE_I to SVE_I_SIMD_DI. (@aarch64_pred_<optab><mode>): Likewise for the MUL_HIGHPART form, and print the data operands with %Z. * config/aarch64/aarch64-sve2.md (@aarch64_sve2_<optab><mode>): Likewise. gcc/testsuite/ChangeLog: * gcc.target/aarch64/sve/mul_highpart_v2di_1.c: New test. * gcc.target/aarch64/sve/mul_highpart_v2di_2.c: New test. * gcc.target/aarch64/sve/mul_highpart_v2di_3.c: New test. * gcc.target/aarch64/sve/mul_highpart_v2di_run.c: New test. Signed-off-by: Kyrylo Tkachov <[email protected]> --- gcc/config/aarch64/aarch64-sve.md | 29 +++++++------ gcc/config/aarch64/aarch64-sve2.md | 13 +++--- .../aarch64/sve/mul_highpart_v2di_1.c | 25 +++++++++++ .../aarch64/sve/mul_highpart_v2di_2.c | 23 ++++++++++ .../aarch64/sve/mul_highpart_v2di_3.c | 22 ++++++++++ .../aarch64/sve/mul_highpart_v2di_run.c | 43 +++++++++++++++++++ 6 files changed, 136 insertions(+), 19 deletions(-) create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md index f665417dba4..5f9a19c42e9 100644 --- a/gcc/config/aarch64/aarch64-sve.md +++ b/gcc/config/aarch64/aarch64-sve.md @@ -4881,13 +4881,16 @@ ;; ------------------------------------------------------------------------- ;; Unpredicated highpart multiplication. +;; Advanced SIMD has no vector DImode high-part multiply, but SVE does. +;; Make use of the overlap between Z and V registers to implement the V2DI +;; optab for TARGET_SVE, in the same way as the mul<mode>3 expander above. (define_expand "<su>mul<mode>3_highpart" - [(set (match_operand:SVE_I 0 "register_operand") - (unspec:SVE_I + [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand") + (unspec:SVE_I_SIMD_DI [(match_dup 3) - (unspec:SVE_I - [(match_operand:SVE_I 1 "register_operand") - (match_operand:SVE_I 2 "register_operand")] + (unspec:SVE_I_SIMD_DI + [(match_operand:SVE_I_SIMD_DI 1 "register_operand") + (match_operand:SVE_I_SIMD_DI 2 "register_operand")] MUL_HIGHPART)] UNSPEC_PRED_X))] "TARGET_SVE" @@ -4898,22 +4901,22 @@ ;; Predicated highpart multiplication. (define_insn_and_split "@aarch64_pred_<optab><mode>" - [(set (match_operand:SVE_I 0 "register_operand") - (unspec:SVE_I + [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand") + (unspec:SVE_I_SIMD_DI [(match_operand:<VPRED> 1 "register_operand") - (unspec:SVE_I - [(match_operand:SVE_I 2 "register_operand") - (match_operand:SVE_I 3 "register_operand")] + (unspec:SVE_I_SIMD_DI + [(match_operand:SVE_I_SIMD_DI 2 "register_operand") + (match_operand:SVE_I_SIMD_DI 3 "register_operand")] MUL_HIGHPART)] UNSPEC_PRED_X))] "TARGET_SVE" {@ [ cons: =0 , 1 , %2 , 3 ; attrs: movprfx ] - [ w , Upl , 0 , w ; * ] <su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> - [ ?&w , Upl , w , w ; yes ] movprfx\t%0, %2\;<su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> + [ w , Upl , 0 , w ; * ] <su>mulh\t%Z0.<Vetype>, %1/m, %Z0.<Vetype>, %Z3.<Vetype> + [ ?&w , Upl , w , w ; yes ] movprfx\t%Z0, %Z2\;<su>mulh\t%Z0.<Vetype>, %1/m, %Z0.<Vetype>, %Z3.<Vetype> } "TARGET_SVE2" [(set (match_dup 0) - (unspec:SVE_I + (unspec:SVE_I_SIMD_DI [(match_dup 2) (match_dup 3)] MUL_HIGHPART))] diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md index a8bee6b6dc7..1cc0cefbee7 100644 --- a/gcc/config/aarch64/aarch64-sve2.md +++ b/gcc/config/aarch64/aarch64-sve2.md @@ -983,15 +983,16 @@ ;; ---- [INT] Unpredicated high-part multiplication ;; ------------------------------------------------------------------------- -;; SVE2 unpredicated SMULH/UMULH. +;; SVE2 unpredicated SMULH/UMULH. V2DI is included so that the Advanced +;; SIMD high-part multiply optabs can be implemented on top of SVE. (define_insn "@aarch64_sve2_<optab><mode>" - [(set (match_operand:SVE_I 0 "register_operand" "=w") - (unspec:SVE_I - [(match_operand:SVE_I 1 "register_operand" "w") - (match_operand:SVE_I 2 "register_operand" "w")] + [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand" "=w") + (unspec:SVE_I_SIMD_DI + [(match_operand:SVE_I_SIMD_DI 1 "register_operand" "w") + (match_operand:SVE_I_SIMD_DI 2 "register_operand" "w")] MUL_HIGHPART))] "TARGET_SVE2" - "<su>mulh\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" + "<su>mulh\t%Z0.<Vetype>, %Z1.<Vetype>, %Z2.<Vetype>" [(set_attr "sve_type" "sve_int_mul")] ) diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c new file mode 100644 index 00000000000..af1548571b7 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c @@ -0,0 +1,25 @@ +/* Advanced SIMD has no 64-bit high-part multiply, but SVE does, and the two + register files overlap. Check that a 128-bit vector high-part multiply is + used when the vectoriser is restricted to Advanced SIMD modes. */ +/* { dg-do assemble { target aarch64_asm_sve_ok } } */ +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve -mautovec-preference=asimd-only --save-temps" } */ + +#include <stdint.h> + +#define DEF_LOOP(TYPE, WIDE) \ + void __attribute__ ((noipa)) \ + mulh_##TYPE (TYPE *restrict dst, TYPE *restrict a, \ + TYPE *restrict b, int count) \ + { \ + for (int i = 0; i < count; ++i) \ + dst[i] = (TYPE) (((WIDE) a[i] * b[i]) >> 64); \ + } + +DEF_LOOP (int64_t, __int128) +DEF_LOOP (uint64_t, unsigned __int128) + +/* { dg-final { scan-assembler-times {\tsmulh\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tumulh\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */ +/* The loop must use Advanced SIMD 128-bit accesses, not a scalable loop. */ +/* { dg-final { scan-assembler {\tldr\tq[0-9]+} } } */ +/* { dg-final { scan-assembler-not {\twhilelo\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c new file mode 100644 index 00000000000..307d58258b5 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c @@ -0,0 +1,23 @@ +/* Same as mul_highpart_v2di_1.c, but for SVE2, where the high-part multiply + has an unpredicated form. */ +/* { dg-do assemble { target aarch64_asm_sve2_ok } } */ +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-preference=asimd-only --save-temps" } */ + +#include <stdint.h> + +#define DEF_LOOP(TYPE, WIDE) \ + void __attribute__ ((noipa)) \ + mulh_##TYPE (TYPE *restrict dst, TYPE *restrict a, \ + TYPE *restrict b, int count) \ + { \ + for (int i = 0; i < count; ++i) \ + dst[i] = (TYPE) (((WIDE) a[i] * b[i]) >> 64); \ + } + +DEF_LOOP (int64_t, __int128) +DEF_LOOP (uint64_t, unsigned __int128) + +/* { dg-final { scan-assembler-times {\tsmulh\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tumulh\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */ +/* { dg-final { scan-assembler {\tldr\tq[0-9]+} } } */ +/* { dg-final { scan-assembler-not {\twhilelo\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c new file mode 100644 index 00000000000..afb88be52ee --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c @@ -0,0 +1,22 @@ +/* The Shoup modular multiply used by lattice cryptography. The high-part + multiply is the only operation Advanced SIMD lacks, so without a 128-bit + vector form of it the whole loop stays scalar. */ +/* { dg-do assemble { target aarch64_asm_sve2_ok } } */ +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-preference=asimd-only --save-temps" } */ + +#include <stdint.h> + +void __attribute__ ((noipa)) +mul_mod (uint64_t *restrict dst, const uint64_t *restrict src, + uint64_t operand, uint64_t quotient, uint64_t modulus, int count) +{ + for (int i = 0; i < count; ++i) + { + uint64_t hi = (uint64_t) (((unsigned __int128) src[i] * quotient) >> 64); + uint64_t t = operand * src[i] - hi * modulus; + dst[i] = t >= modulus ? t - modulus : t; + } +} + +/* { dg-final { scan-assembler-times {\tumulh\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */ +/* { dg-final { scan-assembler {\tldr\tq[0-9]+} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c new file mode 100644 index 00000000000..9b9b2ab40dd --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c @@ -0,0 +1,43 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve -mautovec-preference=asimd-only" } */ + +#include "mul_highpart_v2di_1.c" + +#define N 77 + +static int64_t sa[N], sb[N], sd[N]; +static uint64_t ua[N], ub[N], ud[N]; + +int +main (void) +{ + uint64_t s = 0x243f6a8885a308d3ULL; + for (int i = 0; i < N; ++i) + { + s ^= s << 13; s ^= s >> 7; s ^= s << 17; + sa[i] = (int64_t) s; + ua[i] = s; + s ^= s << 13; s ^= s >> 7; s ^= s << 17; + sb[i] = (int64_t) s; + ub[i] = s; + } + /* Boundary values. */ + sa[0] = INT64_MIN; sb[0] = INT64_MIN; + sa[1] = INT64_MIN; sb[1] = -1; + sa[2] = -1; sb[2] = -1; + ua[0] = 0; ub[0] = ~0ULL; + ua[1] = ~0ULL; ub[1] = ~0ULL; + ua[2] = 1ULL << 63; ub[2] = 1ULL << 63; + + mulh_int64_t (sd, sa, sb, N); + mulh_uint64_t (ud, ua, ub, N); + + for (int i = 0; i < N; ++i) + { + if (sd[i] != (int64_t) (((__int128) sa[i] * sb[i]) >> 64)) + __builtin_abort (); + if (ud[i] != (uint64_t) (((unsigned __int128) ua[i] * ub[i]) >> 64)) + __builtin_abort (); + } + return 0; +} -- 2.50.1 (Apple Git-155)