RE: [PATCH] [v2] aarch64: Implement vector DImode high-part multiply using SVE

> -----Original Message-----
> From: [email protected] <[email protected]>
> Sent: 06 August 2026 10:03
> To: [email protected]
> Cc: Tamar Christina <[email protected]>;
> [email protected]; Kyrylo Tkachov <[email protected]>
> Subject: [PATCH] [v2] aarch64: Implement vector DImode high-part multiply
> using SVE
> 
> From: Kyrylo Tkachov <[email protected]>
> 
> Advanced SIMD has no 64x64 high-part multiply, so the vectorizer
> currently cannot use its modes for loops such as:
> 
>   for (int i = 0; i < n; i++)
>     d[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
> 
> SVE has UMULH and SMULH for D elements and the Z registers overlap the V
> registers, so the Advanced SIMD optab can be implemented on top of the SVE
> instruction.  Extend the high-part multiply patterns from SVE_I to
> SVE_I_SIMD_DI, exactly as mul<mode>3 already does for vector DImode
> multiplication, and print the operands with %Z so that the V2DI form uses
> the Z register names.
> 
> For the loop above on a 128-bit SVE implementation we now generate:
> 
> 	ldr	q31, [x1, x4]
> 	ldr	q30, [x2, x4]
> 	umulh	z30.d, z31.d, z30.d
> 	str	q30, [x0, x4]
> 
> rather than:
> 
> 	ldr	x5, [x1, x4]
> 	ldr	x6, [x2, x4]
> 	umulh	x5, x5, x6
> 	str	x5, [x0, x4]
> 
> I don't think people often run the testuite with -march=armv8.2-a+sve
> -mautovec-preference=asimd-only and I don't know how clean it is, so

Not very wrt to codegen.

> I've added the runtime test generically to torture and included it in
> gcc.target/aarch64/ with the aarch64 flags added on top.
> 
> Bootstrapped and regression-tested on aarch64-unknown-linux-gnu.
> Ok for trunk?

Yeah, LGTM.

Thanks,
Tamar
> Thanks,
> Kyrill
> 
> gcc/ChangeLog:
> 
> 	* config/aarch64/aarch64-sve.md (<su>mul<mode>3_highpart):
> Extend
> 	from SVE_I to SVE_I_SIMD_DI.
> 	(@aarch64_pred_<optab><mode>): Likewise for the MUL_HIGHPART
> form,
> 	and print the data operands with %Z.
> 	* config/aarch64/aarch64-sve2.md
> (@aarch64_sve2_<optab><mode>):
> 	Likewise.
> 
> gcc/testsuite/ChangeLog:
> 
> 	* gcc.target/aarch64/sve/mul_highpart_v2di_1.c: New test.
> 	* gcc.target/aarch64/sve/mul_highpart_v2di_2.c: New test.
> 	* gcc.target/aarch64/sve/mul_highpart_v2di_3.c: New test.
> 	* gcc.target/aarch64/sve/mul_highpart_v2di_run.c: New test.
> 	* gcc.dg/torture/mul-highpart-1.c: New test.
> 
> Signed-off-by: Kyrylo Tkachov <[email protected]>
> ---
>  gcc/config/aarch64/aarch64-sve.md             | 29 +++++----
>  gcc/config/aarch64/aarch64-sve2.md            | 13 ++--
>  gcc/testsuite/gcc.dg/torture/mul-highpart-1.c | 62 +++++++++++++++++++
>  .../aarch64/sve/mul_highpart_v2di_1.c         | 37 +++++++++++
>  .../aarch64/sve/mul_highpart_v2di_2.c         | 35 +++++++++++
>  .../aarch64/sve/mul_highpart_v2di_3.c         | 26 ++++++++
>  .../aarch64/sve/mul_highpart_v2di_run.c       |  7 +++
>  7 files changed, 190 insertions(+), 19 deletions(-)
>  create mode 100644 gcc/testsuite/gcc.dg/torture/mul-highpart-1.c
>  create mode 100644
> gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
>  create mode 100644
> gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
>  create mode 100644
> gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
>  create mode 100644
> gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
> 
> diff --git a/gcc/config/aarch64/aarch64-sve.md
> b/gcc/config/aarch64/aarch64-sve.md
> index f665417dba4..5f9a19c42e9 100644
> --- a/gcc/config/aarch64/aarch64-sve.md
> +++ b/gcc/config/aarch64/aarch64-sve.md
> @@ -4881,13 +4881,16 @@
>  ;; -------------------------------------------------------------------------
> 
>  ;; Unpredicated highpart multiplication.
> +;; Advanced SIMD has no vector DImode high-part multiply, but SVE does.
> +;; Make use of the overlap between Z and V registers to implement the V2DI
> +;; optab for TARGET_SVE, in the same way as the mul<mode>3 expander
> above.
>  (define_expand "<su>mul<mode>3_highpart"
> -  [(set (match_operand:SVE_I 0 "register_operand")
> -	(unspec:SVE_I
> +  [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand")
> +	(unspec:SVE_I_SIMD_DI
>  	  [(match_dup 3)
> -	   (unspec:SVE_I
> -	     [(match_operand:SVE_I 1 "register_operand")
> -	      (match_operand:SVE_I 2 "register_operand")]
> +	   (unspec:SVE_I_SIMD_DI
> +	     [(match_operand:SVE_I_SIMD_DI 1 "register_operand")
> +	      (match_operand:SVE_I_SIMD_DI 2 "register_operand")]
>  	     MUL_HIGHPART)]
>  	  UNSPEC_PRED_X))]
>    "TARGET_SVE"
> @@ -4898,22 +4901,22 @@
> 
>  ;; Predicated highpart multiplication.
>  (define_insn_and_split "@aarch64_pred_<optab><mode>"
> -  [(set (match_operand:SVE_I 0 "register_operand")
> -	(unspec:SVE_I
> +  [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand")
> +	(unspec:SVE_I_SIMD_DI
>  	  [(match_operand:<VPRED> 1 "register_operand")
> -	   (unspec:SVE_I
> -	     [(match_operand:SVE_I 2 "register_operand")
> -	      (match_operand:SVE_I 3 "register_operand")]
> +	   (unspec:SVE_I_SIMD_DI
> +	     [(match_operand:SVE_I_SIMD_DI 2 "register_operand")
> +	      (match_operand:SVE_I_SIMD_DI 3 "register_operand")]
>  	     MUL_HIGHPART)]
>  	  UNSPEC_PRED_X))]
>    "TARGET_SVE"
>    {@ [ cons: =0 , 1   , %2 , 3 ; attrs: movprfx ]
> -     [ w        , Upl , 0  , w ; *              ] <su>mulh\t%0.<Vetype>, %1/m,
> %0.<Vetype>, %3.<Vetype>
> -     [ ?&w      , Upl , w  , w ; yes            ] movprfx\t%0,
> %2\;<su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>
> +     [ w        , Upl , 0  , w ; *              ] <su>mulh\t%Z0.<Vetype>, %1/m,
> %Z0.<Vetype>, %Z3.<Vetype>
> +     [ ?&w      , Upl , w  , w ; yes            ] movprfx\t%Z0,
> %Z2\;<su>mulh\t%Z0.<Vetype>, %1/m, %Z0.<Vetype>, %Z3.<Vetype>
>    }
>    "TARGET_SVE2"
>    [(set (match_dup 0)
> -	(unspec:SVE_I
> +	(unspec:SVE_I_SIMD_DI
>  	  [(match_dup 2)
>  	   (match_dup 3)]
>  	  MUL_HIGHPART))]
> diff --git a/gcc/config/aarch64/aarch64-sve2.md
> b/gcc/config/aarch64/aarch64-sve2.md
> index a8bee6b6dc7..1cc0cefbee7 100644
> --- a/gcc/config/aarch64/aarch64-sve2.md
> +++ b/gcc/config/aarch64/aarch64-sve2.md
> @@ -983,15 +983,16 @@
>  ;; ---- [INT] Unpredicated high-part multiplication
>  ;; -------------------------------------------------------------------------
> 
> -;; SVE2 unpredicated SMULH/UMULH.
> +;; SVE2 unpredicated SMULH/UMULH.  V2DI is included so that the
> Advanced
> +;; SIMD high-part multiply optabs can be implemented on top of SVE.
>  (define_insn "@aarch64_sve2_<optab><mode>"
> -  [(set (match_operand:SVE_I 0 "register_operand" "=w")
> -	(unspec:SVE_I
> -	  [(match_operand:SVE_I 1 "register_operand" "w")
> -	   (match_operand:SVE_I 2 "register_operand" "w")]
> +  [(set (match_operand:SVE_I_SIMD_DI 0 "register_operand" "=w")
> +	(unspec:SVE_I_SIMD_DI
> +	  [(match_operand:SVE_I_SIMD_DI 1 "register_operand" "w")
> +	   (match_operand:SVE_I_SIMD_DI 2 "register_operand" "w")]
>  	  MUL_HIGHPART))]
>    "TARGET_SVE2"
> -  "<su>mulh\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
> +  "<su>mulh\t%Z0.<Vetype>, %Z1.<Vetype>, %Z2.<Vetype>"
>    [(set_attr "sve_type" "sve_int_mul")]
>  )
> 
> diff --git a/gcc/testsuite/gcc.dg/torture/mul-highpart-1.c
> b/gcc/testsuite/gcc.dg/torture/mul-highpart-1.c
> new file mode 100644
> index 00000000000..0c6b7812863
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/torture/mul-highpart-1.c
> @@ -0,0 +1,62 @@
> +/* Check that a 64-bit high-part multiply gives the same answer whether or
> not
> +   it is vectorized.  */
> +/* { dg-do run } */
> +/* { dg-require-effective-target int128 } */
> +/* { dg-additional-options "-ftree-vectorize" } */
> +
> +#include <stdint.h>
> +
> +#define N 77
> +
> +static int64_t sa[N], sb[N], sd[N];
> +static uint64_t ua[N], ub[N], ud[N];
> +
> +void __attribute__ ((noipa))
> +mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b,
> +	  int count)
> +{
> +  for (int i = 0; i < count; ++i)
> +    dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64);
> +}
> +
> +void __attribute__ ((noipa))
> +mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b,
> +	  int count)
> +{
> +  for (int i = 0; i < count; ++i)
> +    dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
> +}
> +
> +int
> +main (void)
> +{
> +  uint64_t s = 0x243f6a8885a308d3ULL;
> +  for (int i = 0; i < N; ++i)
> +    {
> +      s ^= s << 13; s ^= s >> 7; s ^= s << 17;
> +      sa[i] = (int64_t) s;
> +      ua[i] = s;
> +      s ^= s << 13; s ^= s >> 7; s ^= s << 17;
> +      sb[i] = (int64_t) s;
> +      ub[i] = s;
> +    }
> +  /* Boundary values.  */
> +  sa[0] = INT64_MIN; sb[0] = INT64_MIN;
> +  sa[1] = INT64_MIN; sb[1] = -1;
> +  sa[2] = -1; sb[2] = -1;
> +  ua[0] = 0; ub[0] = ~(uint64_t) 0;
> +  ua[1] = ~(uint64_t) 0; ub[1] = ~(uint64_t) 0;
> +  ua[2] = (uint64_t) 1 << 63; ub[2] = (uint64_t) 1 << 63;
> +
> +  mulh_s64 (sd, sa, sb, N);
> +  mulh_u64 (ud, ua, ub, N);
> +
> +  for (int i = 0; i < N; ++i)
> +    {
> +      if (sd[i] != (int64_t) (((__int128) sa[i] * sb[i]) >> 64))
> +	__builtin_abort ();
> +      if (ud[i] != (uint64_t) (((unsigned __int128) ua[i] * ub[i]) >> 64))
> +	__builtin_abort ();
> +    }
> +  return 0;
> +}
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
> b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
> new file mode 100644
> index 00000000000..2544e4c6b79
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_1.c
> @@ -0,0 +1,37 @@
> +/* Advanced SIMD has no 64-bit high-part multiply, but SVE does, and the
> two
> +   register files overlap.  Check that the SVE instruction is used for a
> +   128-bit vector high-part multiply when the vectoriser is restricted to
> +   Advanced SIMD modes.  */
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve -mautovec-
> preference=asimd-only" } */
> +/* { dg-final { check-function-bodies "**" "" } } */
> +
> +#include <stdint.h>
> +
> +/*
> +** mulh_s64:
> +** ...
> +**	smulh	z[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d
> +** ...
> +*/
> +void __attribute__ ((noipa))
> +mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b,
> +	  int count)
> +{
> +  for (int i = 0; i < count; ++i)
> +    dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64);
> +}
> +
> +/*
> +** mulh_u64:
> +** ...
> +**	umulh	z[0-9]+\.d, p[0-7]/m, z[0-9]+\.d, z[0-9]+\.d
> +** ...
> +*/
> +void __attribute__ ((noipa))
> +mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b,
> +	  int count)
> +{
> +  for (int i = 0; i < count; ++i)
> +    dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
> +}
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
> b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
> new file mode 100644
> index 00000000000..3f3989fa901
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_2.c
> @@ -0,0 +1,35 @@
> +/* Same as mul_highpart_v2di_1.c, but for SVE2, where the high-part
> multiply
> +   has an unpredicated form.  */
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-
> preference=asimd-only" } */
> +/* { dg-final { check-function-bodies "**" "" } } */
> +
> +#include <stdint.h>
> +
> +/*
> +** mulh_s64:
> +** ...
> +**	smulh	z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
> +** ...
> +*/
> +void __attribute__ ((noipa))
> +mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b,
> +	  int count)
> +{
> +  for (int i = 0; i < count; ++i)
> +    dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64);
> +}
> +
> +/*
> +** mulh_u64:
> +** ...
> +**	umulh	z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
> +** ...
> +*/
> +void __attribute__ ((noipa))
> +mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b,
> +	  int count)
> +{
> +  for (int i = 0; i < count; ++i)
> +    dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
> +}
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
> b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
> new file mode 100644
> index 00000000000..b459bbc27cc
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_3.c
> @@ -0,0 +1,26 @@
> +/* The Shoup modular multiply used by lattice cryptography.  The high-part
> +   multiply is the only operation Advanced SIMD lacks, so without a 128-bit
> +   vector form of it the whole loop stays scalar.  */
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-
> preference=asimd-only" } */
> +/* { dg-final { check-function-bodies "**" "" } } */
> +
> +#include <stdint.h>
> +
> +/*
> +** mul_mod:
> +** ...
> +**	umulh	z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
> +** ...
> +*/
> +void __attribute__ ((noipa))
> +mul_mod (uint64_t *restrict dst, const uint64_t *restrict src,
> +	 uint64_t operand, uint64_t quotient, uint64_t modulus, int count)
> +{
> +  for (int i = 0; i < count; ++i)
> +    {
> +      uint64_t hi = (uint64_t) (((unsigned __int128) src[i] * quotient) >> 64);
> +      uint64_t t = operand * src[i] - hi * modulus;
> +      dst[i] = t >= modulus ? t - modulus : t;
> +    }
> +}
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
> b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
> new file mode 100644
> index 00000000000..a77f2ef2207
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_v2di_run.c
> @@ -0,0 +1,7 @@
> +/* Run the generic high-part multiply correctness test with the vectoriser
> +   restricted to Advanced SIMD modes, so that the V2DI patterns are the ones
> +   being exercised.  */
> +/* { dg-do run { target aarch64_sve_hw } } */
> +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve -mautovec-
> preference=asimd-only" } */
> +
> +#include "../../../gcc.dg/torture/mul-highpart-1.c"
> --
> 2.50.1 (Apple Git-155)