[PATCH] [v2] vect: Recognise high-part multiply when the wide type has no vector form

From: Kyrylo Tkachov <[email protected]>

vect_recog_mulhs_pattern turns (a * b) >> N into IFN_MULH and casts the
result back to the type of the shift, which the over-widening machinery
then drops.  Building that cast needs a vector type for the wide type, and
the pattern gives up when there is none.

For a 64-bit high-part multiply the wide type is 128 bits, and no target
has a vector of 128-bit integers, so on AArch64 the pattern never fires
for scalable SVE and the loop stays scalar.  It does fire under
-msve-vector-bits=128, where the prevailing vector size is 128 bits and a
one-element TImode vector exists, which is what makes the failure specific
to variable-length vectors.

The pattern has already established that the users of the result need at
most min_output_precision bits and that NEW_TYPE holds at least that many,
so when the wide vector type is unavailable we can simply leave the result
at NEW_TYPE precision and skip the cast.

	for (int i = 0; i < n; i++)
	  d[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);

on -march=armv8.2-a+sve2 -msve-vector-bits=scalable now gives:

	ld1d	z31.d, p7/z, [x1, x4, lsl 3]
	ld1d	z30.d, p7/z, [x2, x4, lsl 3]
	umulh	z30.d, z30.d, z31.d
	st1d	z30.d, p7, [x0, x4, lsl 3]

rather than:

	ldr	x5, [x1, x4]
	ldr	x6, [x2, x4]
	umulh	x5, x5, x6
	str	x5, [x0, x4]

Added a gcc.dg/vect/ test. I've added a new vect_mulh_di effective
target and added what targets I could find that support the relevant
optab at DImode but I've only tested aarch64 myself.

Bootstrapped and regression-tested on aarch64-unknown-linux-gnu.
Ok for trunk?
Thanks,
Kyrill

gcc/ChangeLog:

	* tree-vect-patterns.cc (vect_recog_mulhs_pattern): Keep the
	pattern result at NEW_TYPE precision when LHS_TYPE has no vector
	type.

gcc/testsuite/ChangeLog:

	* lib/target-supports.exp (check_effective_target_vect_mulh_di):
	New procedure.
	* gcc.dg/vect/vect-mulh-1.c: New test.
	* gcc.dg/vect/vect-mulh-2.c: New test.
	* gcc.target/aarch64/sve/mul_highpart_scalable_1.c: New test.

Signed-off-by: Kyrylo Tkachov <[email protected]>
---
 gcc/testsuite/gcc.dg/vect/vect-mulh-1.c       | 52 +++++++++++++++++++
 gcc/testsuite/gcc.dg/vect/vect-mulh-2.c       |  9 ++++
 .../aarch64/sve/mul_highpart_scalable_1.c     | 36 +++++++++++++
 gcc/testsuite/lib/target-supports.exp         | 14 +++++
 gcc/tree-vect-patterns.cc                     | 15 ++++--
 5 files changed, 122 insertions(+), 4 deletions(-)
 create mode 100644 gcc/testsuite/gcc.dg/vect/vect-mulh-1.c
 create mode 100644 gcc/testsuite/gcc.dg/vect/vect-mulh-2.c
 create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c

diff --git a/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c b/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c
new file mode 100644
index 00000000000..06330cc0890
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-mulh-1.c
@@ -0,0 +1,52 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target int128 } */
+
+#include "tree-vect.h"
+
+#ifndef SIGNEDNESS
+#define SIGNEDNESS signed
+#endif
+
+void __attribute__ ((noipa))
+f (SIGNEDNESS long long *restrict a, SIGNEDNESS long long *restrict b,
+   SIGNEDNESS long long *restrict c, __INTPTR_TYPE__ n)
+{
+  for (__INTPTR_TYPE__ i = 0; i < n; ++i)
+    a[i] = ((SIGNEDNESS __int128) b[i] * c[i]) >> 64;
+}
+
+#define N 50
+#define BASE1 0x1234567890abcdefULL
+#define BASE2 0x0fedcba098765432ULL
+#define CONST1 0x0123456789abcdefULL
+#define CONST2 0x0f0e0d0c0b0a0908ULL
+
+int
+main (void)
+{
+  check_vect ();
+
+  SIGNEDNESS long long a[N], b[N], c[N];
+  /* Compute the inputs with wrapping unsigned arithmetic so that they cover
+     the whole 64-bit range without overflowing a signed type.  */
+  for (int i = 0; i < N; ++i)
+    {
+      b[i] = (SIGNEDNESS long long) (BASE1 + (unsigned long long) i * CONST1);
+      c[i] = (SIGNEDNESS long long) (BASE2 + (unsigned long long) i * CONST2);
+      asm volatile ("" ::: "memory");
+    }
+  b[0] = 0;
+  c[0] = -1;
+  b[1] = -1;
+  c[1] = -1;
+  f (a, b, c, N);
+#pragma GCC novector
+  for (int i = 0; i < N; ++i)
+    if (a[i] != (SIGNEDNESS long long) (((SIGNEDNESS __int128) b[i] * c[i])
+					>> 64))
+      __builtin_abort ();
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump {\.MULH} "vect" { target vect_mulh_di } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" { target vect_mulh_di } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c b/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c
new file mode 100644
index 00000000000..0249caea3de
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-mulh-2.c
@@ -0,0 +1,9 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target int128 } */
+
+#define SIGNEDNESS unsigned
+
+#include "vect-mulh-1.c"
+
+/* { dg-final { scan-tree-dump {\.MULH} "vect" { target vect_mulh_di } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" { target vect_mulh_di } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
new file mode 100644
index 00000000000..2d19cde259e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mul_highpart_scalable_1.c
@@ -0,0 +1,36 @@
+/* A 64-bit high-part multiply has a 128-bit product type, and there is no
+   scalable vector of 128-bit integers.  Check that the vectoriser recognises
+   the high-part multiply anyway.  */
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8.2-a+sve2 -mautovec-preference=sve-only -msve-vector-bits=scalable" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <stdint.h>
+
+/*
+** mulh_s64:
+** ...
+**	smulh	z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
+** ...
+*/
+void __attribute__ ((noipa))
+mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b,
+	  int count)
+{
+  for (int i = 0; i < count; ++i)
+    dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64);
+}
+
+/*
+** mulh_u64:
+** ...
+**	umulh	z[0-9]+\.d, z[0-9]+\.d, z[0-9]+\.d
+** ...
+*/
+void __attribute__ ((noipa))
+mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b,
+	  int count)
+{
+  for (int i = 0; i < count; ++i)
+    dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
+}
diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
index 42cbb5ce3df..89784ca68cd 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -9302,6 +9302,20 @@ proc check_effective_target_vect_mulhrs_hi {} {
 		   && [check_effective_target_aarch64_sve2] }]
 }
 
+# Return 1 if the target plus current options supports both signed
+# and unsigned high-part multiplication on vectors of 8-byte integers.
+
+proc check_effective_target_vect_mulh_di {} {
+    return [expr { ([istarget aarch64*-*-*]
+		    && [check_effective_target_aarch64_sve])
+		   || ([istarget riscv*-*-*]
+		       && [check_effective_target_riscv_v])
+		   || ([istarget powerpc*-*-*]
+		       && [check_effective_target_has_arch_pwr10])
+		   || ([istarget loongarch*-*-*]
+		       && [check_effective_target_loongarch_sx]) }]
+}
+
 # Return 1 if the target plus current options supports signed division
 # by power-of-2 operations on vectors of 4-byte integers.
 
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index c57e215be57..185b28302d4 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -3412,11 +3412,18 @@ vect_recog_mulhs_pattern (vec_info *vinfo,
 	    (ifn, new_vectype, OPTIMIZE_FOR_SPEED))
     return NULL;
 
-  /* The IR requires a valid vector type for the cast result, even though
-     it's likely to be discarded.  */
+  /* *TYPE_OUT is the vector type of the trailing cast back to LHS_TYPE, which
+     the over-widening machinery removes anyway.  Drop the cast when LHS_TYPE
+     has no vector type, as for the 128-bit product of a 64-bit high-part
+     multiply.  The uses need at most TARGET_PRECISION bits, which NEW_TYPE
+     has.  */
+  tree out_type = lhs_type;
   *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
   if (!*type_out)
-    return NULL;
+    {
+      out_type = new_type;
+      *type_out = new_vectype;
+    }
 
   /* Generate the IFN_MULHRS call.  */
   tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
@@ -3432,7 +3439,7 @@ vect_recog_mulhs_pattern (vec_info *vinfo,
     dump_printf_loc (MSG_NOTE, vect_location,
 		     "created pattern stmt: %G", (gimple *) mulhrs_stmt);
 
-  return vect_convert_output (vinfo, last_stmt_info, lhs_type,
+  return vect_convert_output (vinfo, last_stmt_info, out_type,
 			      mulhrs_stmt, new_vectype);
 }
 
-- 
2.50.1 (Apple Git-155)