[gcc r17-2819] AArch64: Improve costing of truncated stores

https://gcc.gnu.org/g:10c43486134d6a64e941fc758f2aa20008b43190

commit r17-2819-g10c43486134d6a64e941fc758f2aa20008b43190
Author: Christopher Bazley <[email protected]>
Date:   Wed Jul 29 14:25:13 2026 +0000

    AArch64: Improve costing of truncated stores
    
    PR target/126480
    
    The following pattern of truncating assignments whose results are
    consumed only by store operations is relatively common:
    
    dst[0] = (unsigned char) src[0];
    dst[1] = (unsigned char) src[1];
    ....
    dst[N] = (unsigned char) src[N];
    
    Prior to this change, the vectorizer estimated unrealistically high
    costs for some scalar code:  a cost was charged for each narrowing
    conversion, even though those conversions are effectively free as
    part of the associated stores.  Consequently, the vectorizer could
    decide to vectorize code that should not have been vectorized.
    
    Scalar costs are inevitably somewhat overestimated in the case of
    byte order reversals that should cause GCC to generate a 'rev'
    instruction, because the vectorizer estimates costs independently of
    the store-merging pass that discovers such reversals in scalar code.
    When predicated tails are enabled for basic block SLP, the scalar cost
    of reversals can be overestimated by so much that they are vectorized.
    That will not happen after this change is applied.
    
    The AArch64 backend now uses a new vectorizer function,
    vect_is_truncating_store, to tell whether a given stmt truncates the
    input of a store.  This function is analogous to an existing
    function, vect_is_extending_load, which tells whether a given stmt
    extends the result of a load.  The two functions are called in
    roughly the same places, to help with the accuracy of costing scalar
    and vector stmts.
    
    A truncating assignment that has multiple uses should not be in an
    SLP tree being costed, but it seems convenient to use single_imm_use
    anyway (and it fits the expected/desired case we need to identify).
    
    gcc/ChangeLog:
    
            * config/aarch64/aarch64.cc (aarch64_detect_scalar_stmt_subtype):
            Call the new vect_is_truncating_store function and return 0 if
            vect_is_truncating_store returns true.
            (aarch64_sve_adjust_stmt_cost): Call vect_is_truncating_store
            and assign 0 to stmt_cost if vect_is_truncating_store returns
            true.
            * tree-vectorizer.h (vect_is_truncating_store): New function
            analogous to vect_is_extending_load.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.target/aarch64/pr126480.c: New test.

Diff:
---
 gcc/config/aarch64/aarch64.cc               | 16 +++++++++++++---
 gcc/testsuite/gcc.target/aarch64/pr126480.c | 20 ++++++++++++++++++++
 gcc/tree-vectorizer.h                       | 29 +++++++++++++++++++++++++++++
 3 files changed, 62 insertions(+), 3 deletions(-)

diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 93c00c23a50d..d19ca305d829 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -18247,9 +18247,12 @@ aarch64_detect_scalar_stmt_subtype (vec_info *vinfo, vect_cost_for_stmt kind,
 				    stmt_vec_info stmt_info,
 				    fractional_cost stmt_cost)
 {
-  /* Detect an extension of a loaded value.  In general, we'll be able to fuse
-     the extension with the load.  */
-  if (kind == scalar_stmt && vect_is_extending_load (vinfo, stmt_info))
+  /* Detect an extension of a loaded value or truncation of a value being
+     stored.  In general, we'll be able to fuse the extension/truncation with
+     the load/store.  */
+  if (kind == scalar_stmt
+      && (vect_is_extending_load (vinfo, stmt_info)
+	  || vect_is_truncating_store (vinfo, stmt_info)))
     return 0;
 
   return stmt_cost;
@@ -18377,6 +18380,13 @@ aarch64_sve_adjust_stmt_cost (class vec_info *vinfo, vect_cost_for_stmt kind,
   if (kind == vector_stmt && vect_is_extending_load (vinfo, stmt_info))
     stmt_cost = 0;
 
+  /* Most stores have truncating forms that can do the truncation on the fly.
+     Optimistically assume that a truncation followed by a store will fold to
+     this form during combine, and that the truncation therefore comes for free.
+   */
+  if (kind == vector_stmt && vect_is_truncating_store (vinfo, stmt_info))
+    stmt_cost = 0;
+
   /* For similar reasons, vector_stmt integer truncations are a no-op,
      because we can just ignore the unused upper bits of the source.  */
   if (kind == vector_stmt && vect_is_integer_truncation (stmt_info))
diff --git a/gcc/testsuite/gcc.target/aarch64/pr126480.c b/gcc/testsuite/gcc.target/aarch64/pr126480.c
new file mode 100644
index 000000000000..e0b55130250e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr126480.c
@@ -0,0 +1,20 @@
+/* PR target/126480 */
+/* { dg-do compile } */
+/* { dg-additional-options "-O2 -march=armv8.2-a -fdump-tree-slp2-details" } */
+
+void
+foo (unsigned int *foo, unsigned long *a, unsigned long *b)
+{
+  foo[0] = a[0] >> 33;
+  foo[1] = a[0] >> 44;
+}
+
+/* If costs are pessimistically charged for scalar truncations, then the
+   function is vectorized.  Truncation is expected to be free as part of the
+   store operations, therefore the scalar alternative is expected to be more
+   efficient than the vectorized version.  */
+/* { dg-final { scan-tree-dump-not "basic block part vectorized" "slp2" } } */
+
+/* { dg-final { scan-assembler-not {\tushl\tv[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d\n} } } */
+/* { dg-final { scan-assembler-times {\tlsr\tx[0-9]+, x[0-9]+, 33\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tlsr\tx[0-9]+, x[0-9]+, 44\n} 1 } } */
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index d4eb20f1a107..94ef81383f42 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -31,6 +31,7 @@ typedef struct _slp_tree *slp_tree;
 #include "tree-ssa-operands.h"
 #include "gimple-match.h"
 #include "dominance.h"
+#include "ssa.h"
 
 /* Used for naming of new temporaries.  */
 enum vect_var_kind {
@@ -3000,6 +3001,34 @@ vect_is_extending_load (class vec_info *vinfo, stmt_vec_info stmt_info)
 	  && DR_IS_READ (STMT_VINFO_DATA_REF (def_stmt_info)));
 }
 
+/* Return true if STMT_INFO truncates the input of a store.  */
+inline bool
+vect_is_truncating_store (class vec_info *vinfo, stmt_vec_info stmt_info)
+{
+  /* Although this is quite large for an inline function, this part
+     at least should be inline.  */
+  gassign *assign = dyn_cast<gassign *> (stmt_info->stmt);
+  if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign)))
+    return false;
+
+  tree rhs = gimple_assign_rhs1 (stmt_info->stmt);
+  tree lhs = gimple_assign_lhs (assign);
+  tree lhs_type = TREE_TYPE (lhs);
+  tree rhs_type = TREE_TYPE (rhs);
+  if (!INTEGRAL_TYPE_P (lhs_type) || !INTEGRAL_TYPE_P (rhs_type)
+      || TYPE_PRECISION (lhs_type) >= TYPE_PRECISION (rhs_type))
+    return false;
+
+  gimple *use_stmt;
+  use_operand_p use_p;
+  if (!single_imm_use (lhs, &use_p, &use_stmt))
+    return false;
+
+  stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
+  return (use_stmt_info && STMT_VINFO_DATA_REF (use_stmt_info)
+	  && DR_IS_WRITE (STMT_VINFO_DATA_REF (use_stmt_info)));
+}
+
 /* Return true if STMT_INFO is an integer truncation.  */
 inline bool
 vect_is_integer_truncation (stmt_vec_info stmt_info)