Re: [PATCH] aarch64: Implement vector DImode high-part multiply using SVE for Advanced SIMD modes

On Wed, Aug 5, 2026 at 11:46 PM <[email protected]> wrote:
>
> From: Kyrylo Tkachov <[email protected]>
>
> Advanced SIMD has no 64x64 high-part multiply, so the vectorizer
> currently cannot use its modes for loops such as:
>
>   for (int i = 0; i < n; i++)
>     d[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
>
> SVE has UMULH and SMULH for D elements and the Z registers overlap the V
> registers, so the Advanced SIMD optab can be implemented on top of the SVE
> instruction.  Extend the high-part multiply patterns from SVE_I to
> SVE_I_SIMD_DI, exactly as mul<mode>3 already does for vector DImode
> multiplication, and print the operands with %Z so that the V2DI form uses
> the Z register names.
>
> For the loop above on a 128-bit SVE implementation, we now get
>
>         ldr     q31, [x1, x4]                   umulh   x5, x5, x6
>         ldr     q30, [x2, x4]        instead of  str     x5, [x0, x4]
>         umulh   z30.d, z31.d, z30.d
>         str     q30, [x0, x4]

The formatting for this code transformation looks messed up.

>
> Bootstrapped and regression-tested on aarch64-unknown-linux-gnu.
> Ok for trunk?
> Thanks,
> Kyrill
>
> gcc/ChangeLog:
>
>         * config/aarch64/aarch64-sve.md (<su>mul<mode>3_highpart): Extend
>         from SVE_I to SVE_I_SIMD_DI.
>         (@aarch64_pred_<optab><mode>): Likewise for the MUL_HIGHPART form,
>         and print the data operands with %Z.
>         * config/aarch64/aarch64-sve2.md (@aarch64_sve2_<optab><mode>):
>         Likewise.
>
> gcc/testsuite/ChangeLog:
>
>         * gcc.target/aarch64/sve/mul_highpart_v2di_1.c: New test.
>         * gcc.target/aarch64/sve/mul_highpart_v2di_2.c: New test.
>         * gcc.target/aarch64/sve/mul_highpart_v2di_3.c: New test.
>         * gcc.target/aarch64/sve/mul_highpart_v2di_run.c: New test.
>
> Signed-off-by: Kyrylo Tkachov <[email protected]>
> ---
>  gcc/config/aarch64/aarch64-sve.md             | 29 +++++++------
>  gcc/config/aarch64/aarch64-sve2.md            | 13 +++---
>  .../aarch64/sve/mul_highpart_v2di_1.c         | 25 +++++++++++
>  .../aarch64/sve/mul_highpart_v2di_2.c         | 23 ++++++++++
>  .../aarch64/sve/mul_highpart_v2di_3.c         | 22 ++++++++++
>  .../aarch64/sve/mul_highpart_v2di_run.c       | 43 +++++++++++++++++++
>  6 files changed, 136 insertions(+), 19 deletions(-)
>  create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
>  create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
>  create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
>  create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
>
> diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md
> index f665417dba4..5f9a19c42e9 100644
> --- a/gcc/config/aarch64/aarch64-sve.md
> +++ b/gcc/config/aarch64/aarch64-sve.md
> @@ -4881,13 +4881,16 @@
>  ;; -------------------------------------------------------------------------
>
>  ;; Unpredicated highpart multiplication.
> +;; Advanced SIMD has no vector DImode high-part multiply, but SVE does.
> +;; Make use of the overlap between Z and V registers to implement the V2DI
> +;; optab for TARGET_SVE, in the same way as the mul<mode>3 expander above.
>  (define_expand "<su>mul<mode>3_highpart"
> -  [(set (match_operand:SVE_I 0 "register_operand")
> -       (unspec:SVE_I
> +  [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand")
> +       (unspec:SVE_I_SIMD_DI
>           [(match_dup 3)
> -          (unspec:SVE_I
> -            [(match_operand:SVE_I 1 "register_operand")
> -             (match_operand:SVE_I 2 "register_operand")]
> +          (unspec:SVE_I_SIMD_DI
> +            [(match_operand:SVE_I_SIMD_DI 1 "register_operand")
> +             (match_operand:SVE_I_SIMD_DI 2 "register_operand")]
>              MUL_HIGHPART)]
>           UNSPEC_PRED_X))]
>    "TARGET_SVE"
> @@ -4898,22 +4901,22 @@
>
>  ;; Predicated highpart multiplication.
>  (define_insn_and_split "@aarch64_pred_<optab><mode>"
> -  [(set (match_operand:SVE_I 0 "register_operand")
> -       (unspec:SVE_I
> +  [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand")
> +       (unspec:SVE_I_SIMD_DI
>           [(match_operand:<VPRED> 1 "register_operand")
> -          (unspec:SVE_I
> -            [(match_operand:SVE_I 2 "register_operand")
> -             (match_operand:SVE_I 3 "register_operand")]
> +          (unspec:SVE_I_SIMD_DI
> +            [(match_operand:SVE_I_SIMD_DI 2 "register_operand")
> +             (match_operand:SVE_I_SIMD_DI 3 "register_operand")]
>              MUL_HIGHPART)]
>           UNSPEC_PRED_X))]
>    "TARGET_SVE"
>    {@ [ cons: =0 , 1   , %2 , 3 ; attrs: movprfx ]
> -     [ w        , Upl , 0  , w ; *              ] <su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
> -     [ ?&w      , Upl , w  , w ; yes            ] movprfx\t%0, %2\;<su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
> +     [ w        , Upl , 0  , w ; *              ] <su>mulh\t%Z0.<Vetype>, %1/m, %Z0.<Vetype>, %Z3.<Vetype>
> +     [ ?&w      , Upl , w  , w ; yes            ] movprfx\t%Z0, %Z2\;<su>mulh\t%Z0.<Vetype>, %1/m, %Z0.<Vetype>, %Z3.<Vetype>
>    }
>    "TARGET_SVE2"
>    [(set (match_dup 0)
> -       (unspec:SVE_I
> +       (unspec:SVE_I_SIMD_DI
>           [(match_dup 2)
>            (match_dup 3)]
>           MUL_HIGHPART))]
> diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md
> index a8bee6b6dc7..1cc0cefbee7 100644
> --- a/gcc/config/aarch64/aarch64-sve2.md
> +++ b/gcc/config/aarch64/aarch64-sve2.md
> @@ -983,15 +983,16 @@
>  ;; ---- [INT] Unpredicated high-part multiplication
>  ;; -------------------------------------------------------------------------
>
> -;; SVE2 unpredicated SMULH/UMULH.
> +;; SVE2 unpredicated SMULH/UMULH.  V2DI is included so that the Advanced
> +;; SIMD high-part multiply optabs can be implemented on top of SVE.
>  (define_insn "@aarch64_sve2_<optab><mode>"
> -  [(set (match_operand:SVE_I 0 "register_operand" "=w")
> -       (unspec:SVE_I
> -         [(match_operand:SVE_I 1 "register_operand" "w")
> -          (match_operand:SVE_I 2 "register_operand" "w")]
> +  [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand" "=w")
> +       (unspec:SVE_I_SIMD_DI
> +         [(match_operand:SVE_I_SIMD_DI 1 "register_operand" "w")
> +          (match_operand:SVE_I_SIMD_DI 2 "register_operand" "w")]
>           MUL_HIGHPART))]
>    "TARGET_SVE2"
> -  "<su>mulh\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
> +  "<su>mulh\t%Z0.<Vetype>, %Z1.<Vetype>, %Z2.<Vetype>"
>    [(set_attr "sve_type" "sve_int_mul")]
>  )
>
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
> new file mode 100644
> index 00000000000..af1548571b7
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
> @@ -0,0 +1,25 @@
> +/* Advanced SIMD has no 64-bit high-part multiply, but SVE does, and the two
> +   register files overlap.  Check that a 128-bit vector high-part multiply is
> +   used when the vectoriser is restricted to Advanced SIMD modes.  */
> +/* { dg-do assemble { target aarch64_asm_sve_ok } } */
> +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve -mautovec-preference=asimd-only --save-temps" } */
> +
> +#include <stdint.h>
> +
> +#define DEF_LOOP(TYPE, WIDE)                                   \
> +  void __attribute__ ((noipa))                                 \
> +  mulh_##TYPE (TYPE *restrict dst, TYPE *restrict a,           \
> +              TYPE *restrict b, int count)                     \
> +  {                                                            \
> +    for (int i = 0; i < count; ++i)                            \
> +      dst[i] = (TYPE) (((WIDE) a[i] * b[i]) >> 64);            \
> +  }
> +
> +DEF_LOOP (int64_t, __int128)
> +DEF_LOOP (uint64_t, unsigned __int128)
> +
> +/* { dg-final { scan-assembler-times {\tsmulh\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
> +/* { dg-final { scan-assembler-times {\tumulh\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
> +/* The loop must use Advanced SIMD 128-bit accesses, not a scalable loop.  */
> +/* { dg-final { scan-assembler {\tldr\tq[0-9]+} } } */
> +/* { dg-final { scan-assembler-not {\twhilelo\t} } } */

Please use check-function-bodies instead.

> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
> new file mode 100644
> index 00000000000..307d58258b5
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
> @@ -0,0 +1,23 @@
> +/* Same as mul_highpart_v2di_1.c, but for SVE2, where the high-part multiply
> +   has an unpredicated form.  */
> +/* { dg-do assemble { target aarch64_asm_sve2_ok } } */
> +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-preference=asimd-only --save-temps" } */
> +
> +#include <stdint.h>
> +
> +#define DEF_LOOP(TYPE, WIDE)                                   \
> +  void __attribute__ ((noipa))                                 \
> +  mulh_##TYPE (TYPE *restrict dst, TYPE *restrict a,           \
> +              TYPE *restrict b, int count)                     \
> +  {                                                            \
> +    for (int i = 0; i < count; ++i)                            \
> +      dst[i] = (TYPE) (((WIDE) a[i] * b[i]) >> 64);            \
> +  }
> +
> +DEF_LOOP (int64_t, __int128)
> +DEF_LOOP (uint64_t, unsigned __int128)
> +
> +/* { dg-final { scan-assembler-times {\tsmulh\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
> +/* { dg-final { scan-assembler-times {\tumulh\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
> +/* { dg-final { scan-assembler {\tldr\tq[0-9]+} } } */
> +/* { dg-final { scan-assembler-not {\twhilelo\t} } } */

Likewise.

> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
> new file mode 100644
> index 00000000000..afb88be52ee
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
> @@ -0,0 +1,22 @@
> +/* The Shoup modular multiply used by lattice cryptography.  The high-part
> +   multiply is the only operation Advanced SIMD lacks, so without a 128-bit
> +   vector form of it the whole loop stays scalar.  */
> +/* { dg-do assemble { target aarch64_asm_sve2_ok } } */
> +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-preference=asimd-only --save-temps" } */
> +
> +#include <stdint.h>
> +
> +void __attribute__ ((noipa))
> +mul_mod (uint64_t *restrict dst, const uint64_t *restrict src,
> +        uint64_t operand, uint64_t quotient, uint64_t modulus, int count)
> +{
> +  for (int i = 0; i < count; ++i)
> +    {
> +      uint64_t hi = (uint64_t) (((unsigned __int128) src[i] * quotient) >> 64);
> +      uint64_t t = operand * src[i] - hi * modulus;
> +      dst[i] = t >= modulus ? t - modulus : t;
> +    }
> +}
> +
> +/* { dg-final { scan-assembler-times {\tumulh\tz[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d\n} 1 } } */
> +/* { dg-final { scan-assembler {\tldr\tq[0-9]+} } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
> new file mode 100644
> index 00000000000..9b9b2ab40dd
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
> @@ -0,0 +1,43 @@
> +/* { dg-do run { target aarch64_sve_hw } } */
> +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve -mautovec-preference=asimd-only" } */

Seems like this could also be a generic torture testcase. And then you
run the testsuite with `-march=armv8.2-a+sve
-mautovec-preference=asimd-only` too.

> +
> +#include "mul_highpart_v2di_1.c"
> +
> +#define N 77
> +
> +static int64_t sa[N], sb[N], sd[N];
> +static uint64_t ua[N], ub[N], ud[N];
> +
> +int
> +main (void)
> +{
> +  uint64_t s = 0x243f6a8885a308d3ULL;
> +  for (int i = 0; i < N; ++i)
> +    {
> +      s ^= s << 13; s ^= s >> 7; s ^= s << 17;
> +      sa[i] = (int64_t) s;
> +      ua[i] = s;
> +      s ^= s << 13; s ^= s >> 7; s ^= s << 17;
> +      sb[i] = (int64_t) s;
> +      ub[i] = s;
> +    }
> +  /* Boundary values.  */
> +  sa[0] = INT64_MIN; sb[0] = INT64_MIN;
> +  sa[1] = INT64_MIN; sb[1] = -1;
> +  sa[2] = -1; sb[2] = -1;
> +  ua[0] = 0; ub[0] = ~0ULL;
> +  ua[1] = ~0ULL; ub[1] = ~0ULL;
> +  ua[2] = 1ULL << 63; ub[2] = 1ULL << 63;
> +
> +  mulh_int64_t (sd, sa, sb, N);
> +  mulh_uint64_t (ud, ua, ub, N);
> +
> +  for (int i = 0; i < N; ++i)
> +    {
> +      if (sd[i] != (int64_t) (((__int128) sa[i] * sb[i]) >> 64))
> +       __builtin_abort ();
> +      if (ud[i] != (uint64_t) (((unsigned __int128) ua[i] * ub[i]) >> 64))
> +       __builtin_abort ();
> +    }
> +  return 0;
> +}
> --
> 2.50.1 (Apple Git-155)
>