Re: [PATCH] [v2] vect: Recognise high-part multiply when the wide type has no vector form

On Thu, 6 Aug 2026, [email protected] wrote:

> From: Kyrylo Tkachov <[email protected]>
> 
> vect_recog_mulhs_pattern turns (a * b) >> N into IFN_MULH and casts the
> result back to the type of the shift, which the over-widening machinery
> then drops.  Building that cast needs a vector type for the wide type, and
> the pattern gives up when there is none.
> 
> For a 64-bit high-part multiply the wide type is 128 bits, and no target
> has a vector of 128-bit integers, so on AArch64 the pattern never fires
> for scalable SVE and the loop stays scalar.  It does fire under
> -msve-vector-bits=128, where the prevailing vector size is 128 bits and a
> one-element TImode vector exists, which is what makes the failure specific
> to variable-length vectors.
> 
> The pattern has already established that the users of the result need at
> most min_output_precision bits and that NEW_TYPE holds at least that many,
> so when the wide vector type is unavailable we can simply leave the result
> at NEW_TYPE precision and skip the cast.
> 
> 	for (int i = 0; i < n; i++)
> 	  d[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
> 
> on -march=armv8.2-a+sve2 -msve-vector-bits=scalable now gives:
> 
> 	ld1d	z31.d, p7/z, [x1, x4, lsl 3]
> 	ld1d	z30.d, p7/z, [x2, x4, lsl 3]
> 	umulh	z30.d, z30.d, z31.d
> 	st1d	z30.d, p7, [x0, x4, lsl 3]
> 
> rather than:
> 
> 	ldr	x5, [x1, x4]
> 	ldr	x6, [x2, x4]
> 	umulh	x5, x5, x6
> 	str	x5, [x0, x4]
> 
> Added a gcc.dg/vect/ test. I've added a new vect_mulh_di effective
> target and added what targets I could find that support the relevant
> optab at DImode but I've only tested aarch64 myself.
> 
> Bootstrapped and regression-tested on aarch64-unknown-linux-gnu.
> Ok for trunk?
> Thanks,
> Kyrill
> 
> gcc/ChangeLog:
> 
> 	* tree-vect-patterns.cc (vect_recog_mulhs_pattern): Keep the
> 	pattern result at NEW_TYPE precision when LHS_TYPE has no vector
> 	type.
> 
> gcc/testsuite/ChangeLog:
> 
> 	* lib/target-supports.exp (check_effective_target_vect_mulh_di):
> 	New procedure.
> 	* gcc.dg/vect/vect-mulh-1.c: New test.
> 	* gcc.dg/vect/vect-mulh-2.c: New test.
> 	* gcc.target/aarch64/sve/mul_highpart_scalable_1.c: New test.
> 
> Signed-off-by: Kyrylo Tkachov <[email protected]>
> ---
>  gcc/testsuite/gcc.dg/vect/vect-mulh-1.c       | 52 +++++++++++++++++++
>  gcc/testsuite/gcc.dg/vect/vect-mulh-2.c       |  9 ++++
>  .../aarch64/sve/mul_highpart_scalable_1.c     | 36 +++++++++++++
>  gcc/testsuite/lib/target-supports.exp         | 14 +++++
>  gcc/tree-vect-patterns.cc                     | 15 ++++--
>  5 files changed, 122 insertions(+), 4 deletions(-)
>  create mode 100644 gcc/testsuite/gcc.dg/vect/vect-mulh-1.c
>  create mode 100644 gcc/testsuite/gcc.dg/vect/vect-mulh-2.c
>  create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
> 
> diff --git a/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c b/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c
> new file mode 100644
> index 00000000000..06330cc0890
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c
> @@ -0,0 +1,52 @@
> +/* { dg-require-effective-target vect_int } */
> +/* { dg-require-effective-target int128 } */
> +
> +#include "tree-vect.h"
> +
> +#ifndef SIGNEDNESS
> +#define SIGNEDNESS signed
> +#endif
> +
> +void __attribute__ ((noipa))
> +f (SIGNEDNESS long long *restrict a, SIGNEDNESS long long *restrict b,
> +   SIGNEDNESS long long *restrict c, __INTPTR_TYPE__ n)
> +{
> +  for (__INTPTR_TYPE__ i = 0; i < n; ++i)
> +    a[i] = ((SIGNEDNESS __int128) b[i] * c[i]) >> 64;
> +}
> +
> +#define N 50
> +#define BASE1 0x1234567890abcdefULL
> +#define BASE2 0x0fedcba098765432ULL
> +#define CONST1 0x0123456789abcdefULL
> +#define CONST2 0x0f0e0d0c0b0a0908ULL
> +
> +int
> +main (void)
> +{
> +  check_vect ();
> +
> +  SIGNEDNESS long long a[N], b[N], c[N];
> +  /* Compute the inputs with wrapping unsigned arithmetic so that they cover
> +     the whole 64-bit range without overflowing a signed type.  */
> +  for (int i = 0; i < N; ++i)
> +    {
> +      b[i] = (SIGNEDNESS long long) (BASE1 + (unsigned long long) i * CONST1);
> +      c[i] = (SIGNEDNESS long long) (BASE2 + (unsigned long long) i * CONST2);
> +      asm volatile ("" ::: "memory");
> +    }
> +  b[0] = 0;
> +  c[0] = -1;
> +  b[1] = -1;
> +  c[1] = -1;
> +  f (a, b, c, N);
> +#pragma GCC novector
> +  for (int i = 0; i < N; ++i)
> +    if (a[i] != (SIGNEDNESS long long) (((SIGNEDNESS __int128) b[i] * c[i])
> +					>> 64))
> +      __builtin_abort ();
> +  return 0;
> +}
> +
> +/* { dg-final { scan-tree-dump {\.MULH} "vect" { target vect_mulh_di } } } */
> +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" { target vect_mulh_di } } } */
> diff --git a/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c b/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c
> new file mode 100644
> index 00000000000..0249caea3de
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c
> @@ -0,0 +1,9 @@
> +/* { dg-require-effective-target vect_int } */
> +/* { dg-require-effective-target int128 } */
> +
> +#define SIGNEDNESS unsigned
> +
> +#include "vect-mulh-1.c"
> +
> +/* { dg-final { scan-tree-dump {\.MULH} "vect" { target vect_mulh_di } } } */
> +/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" { target vect_mulh_di } } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
> new file mode 100644
> index 00000000000..2d19cde259e
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
> @@ -0,0 +1,36 @@
> +/* A 64-bit high-part multiply has a 128-bit product type, and there is no
> +   scalable vector of 128-bit integers.  Check that the vectoriser recognises
> +   the high-part multiply anyway.  */
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-preference=sve-only -msve-vector-bits=scalable" } */
> +/* { dg-final { check-function-bodies "**" "" } } */
> +
> +#include <stdint.h>
> +
> +/*
> +** mulh_s64:
> +** ...
> +**	smulh	z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
> +** ...
> +*/
> +void __attribute__ ((noipa))
> +mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b,
> +	  int count)
> +{
> +  for (int i = 0; i < count; ++i)
> +    dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64);
> +}
> +
> +/*
> +** mulh_u64:
> +** ...
> +**	umulh	z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
> +** ...
> +*/
> +void __attribute__ ((noipa))
> +mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b,
> +	  int count)
> +{
> +  for (int i = 0; i < count; ++i)
> +    dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
> +}
> diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
> index 42cbb5ce3df..89784ca68cd 100644
> --- a/gcc/testsuite/lib/target-supports.exp
> +++ b/gcc/testsuite/lib/target-supports.exp
> @@ -9302,6 +9302,20 @@ proc check_effective_target_vect_mulhrs_hi {} {
>  		   && [check_effective_target_aarch64_sve2] }]
>  }
>  
> +# Return 1 if the target plus current options supports both signed
> +# and unsigned high-part multiplication on vectors of 8-byte integers.
> +
> +proc check_effective_target_vect_mulh_di {} {
> +    return [expr { ([istarget aarch64*-*-*]
> +		    && [check_effective_target_aarch64_sve])
> +		   || ([istarget riscv*-*-*]
> +		       && [check_effective_target_riscv_v])
> +		   || ([istarget powerpc*-*-*]
> +		       && [check_effective_target_has_arch_pwr10])
> +		   || ([istarget loongarch*-*-*]
> +		       && [check_effective_target_loongarch_sx]) }]
> +}
> +
>  # Return 1 if the target plus current options supports signed division
>  # by power-of-2 operations on vectors of 4-byte integers.
>  
> diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
> index c57e215be57..185b28302d4 100644
> --- a/gcc/tree-vect-patterns.cc
> +++ b/gcc/tree-vect-patterns.cc
> @@ -3412,11 +3412,18 @@ vect_recog_mulhs_pattern (vec_info *vinfo,
>  	    (ifn, new_vectype, OPTIMIZE_FOR_SPEED))
>      return NULL;
>  
> -  /* The IR requires a valid vector type for the cast result, even though
> -     it's likely to be discarded.  */
> +  /* *TYPE_OUT is the vector type of the trailing cast back to LHS_TYPE, which
> +     the over-widening machinery removes anyway.  Drop the cast when LHS_TYPE
> +     has no vector type, as for the 128-bit product of a 64-bit high-part
> +     multiply.  The uses need at most TARGET_PRECISION bits, which NEW_TYPE
> +     has.  */
> +  tree out_type = lhs_type;
>    *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
>    if (!*type_out)
> -    return NULL;
> +    {
> +      out_type = new_type;
> +      *type_out = new_vectype;
> +    }
>  
>    /* Generate the IFN_MULHRS call.  */
>    tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
> @@ -3432,7 +3439,7 @@ vect_recog_mulhs_pattern (vec_info *vinfo,
>      dump_printf_loc (MSG_NOTE, vect_location,
>  		     "created pattern stmt: %G", (gimple *) mulhrs_stmt);
>  
> -  return vect_convert_output (vinfo, last_stmt_info, lhs_type,
> +  return vect_convert_output (vinfo, last_stmt_info, out_type,
>  			      mulhrs_stmt, new_vectype);

So as I understand this replaces a stmt (aka record a pattern stmt)
with a pattern stmt with a different LHS type?

That's unexpected and will cause interesting effects downstream.

Richard.

>  }
>  
> 

-- 
Richard Biener <[email protected]>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Jochen Jaser, Andrew McDonald, Abhinav Puri; (HRB 36809, AG Nuernberg)