[gcc r17-2723] i386: Prefer shorter fold-left reduction chains

https://gcc.gnu.org/g:3f2b3eced0f8278fa335b1337a98b96df3ea062e

commit r17-2723-g3f2b3eced0f8278fa335b1337a98b96df3ea062e
Author: liuhongt <[email protected]>
Date:   Mon Jul 6 20:14:01 2026 -0700

    i386: Prefer shorter fold-left reduction chains
    
    Fold-left FP reductions are lowered to scalar operations on x86.  In
    mixed-width loops the per-statement costs can therefore make a wider
    vector mode look better even though it lengthens the serial reduction
    chain.
    
    Count the fold-left reduction lanes from reduction groups and chains during
    finish_cost and use that as an extra loop-candidate preference.  Do not let
    emulated sub-SSE modes win on this preference alone.  Dump the lane count when
    present.
    
    gcc/ChangeLog:
    
            * config/i386/i386.cc (ix86_vector_costs): Add
            better_fold_left_reduc_than_p and m_num_fold_left_reduc_lanes.
            (ix86_vector_costs::ix86_vector_costs): Initialize it.
            (ix86_vector_costs::finish_cost): Count and dump fold-left
            reduction lanes.
            (ix86_vector_costs::better_fold_left_reduc_than_p): New function.
            (ix86_vector_costs::better_main_loop_than_p): Use it.
            (ix86_vector_costs::better_epilogue_loop_than_p): Likewise.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.target/i386/fold-left-reduc-cost.c: New test.
            * gcc.target/i386/fold-left-reduc-chain-cost.c: New test.
            * gcc.target/i386/vect-epilogues-6.c: Update expected vector size.
            * gcc.target/i386/vect-epilogues-7.c: Likewise.
            * gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c: Likewise.

Diff:
---
 gcc/config/i386/i386.cc                            | 42 +++++++++++++++++++++-
 .../vect/costmodel/x86_64/costmodel-vect-epil-1.c  |  7 ++--
 .../gcc.target/i386/fold-left-reduc-chain-cost.c   | 19 ++++++++++
 .../gcc.target/i386/fold-left-reduc-cost.c         | 18 ++++++++++
 gcc/testsuite/gcc.target/i386/vect-epilogues-6.c   |  3 +-
 gcc/testsuite/gcc.target/i386/vect-epilogues-7.c   |  4 +--
 6 files changed, 84 insertions(+), 9 deletions(-)

diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index 804f35b29739..33e5e063b991 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -26452,6 +26452,7 @@ public:
 
 private:
 
+  bool better_fold_left_reduc_than_p (const vector_costs *) const;
   /* Estimate register pressure of the vectorized code.  */
   void ix86_vect_estimate_reg_pressure ();
   /* Number of GENERAL_REGS/SSE_REGS used in the vectorizer, it's used for
@@ -26468,6 +26469,8 @@ private:
   unsigned m_num_reduc[X86_REDUC_LAST];
   /* Don't do unroll if m_prefer_unroll is false, default is true.  */
   bool m_prefer_unroll;
+  /* Scalar lanes in fold-left reductions.  */
+  unsigned int m_num_fold_left_reduc_lanes;
 };
 
 ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool costing_for_scalar)
@@ -26477,7 +26480,8 @@ ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool costing_for_scalar)
     m_num_avx256_vec_perm (),
     m_num_avx512_vec_perm (),
     m_num_reduc (),
-    m_prefer_unroll (true)
+    m_prefer_unroll (true),
+    m_num_fold_left_reduc_lanes (0)
 {}
 
 /* Implement targetm.vectorize.create_costs.  */
@@ -27183,6 +27187,20 @@ ix86_vector_costs::finish_cost (const vector_costs *scalar_costs)
   loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
   if (loop_vinfo && !m_costing_for_scalar)
     {
+      unsigned int vf = vect_vf_for_cost (loop_vinfo);
+      for (auto inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
+	if ((SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group
+	     || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain)
+	    && (vect_reduc_type (loop_vinfo, SLP_INSTANCE_TREE (inst))
+		== FOLD_LEFT_REDUCTION))
+	  m_num_fold_left_reduc_lanes
+	    += vf * SLP_TREE_LANES (SLP_INSTANCE_TREE (inst));
+
+      if (m_num_fold_left_reduc_lanes && dump_enabled_p ())
+	dump_printf_loc (MSG_NOTE, vect_location,
+			 "in-order FP reduction lanes: %u\n",
+			 m_num_fold_left_reduc_lanes);
+
       /* We are currently not asking the vectorizer to compare costs
 	 between different vector mode sizes.  When using predication
 	 that will end up always choosing the preferred mode size even
@@ -27347,6 +27365,21 @@ ix86_vector_costs::finish_cost (const vector_costs *scalar_costs)
   vector_costs::finish_cost (scalar_costs);
 }
 
+/* Return true if THIS has a shorter fold-left reduction chain than OTHER.  */
+
+bool
+ix86_vector_costs::better_fold_left_reduc_than_p
+  (const vector_costs *other) const
+{
+  auto other_costs = static_cast<const ix86_vector_costs *> (other);
+  if (m_num_fold_left_reduc_lanes >= other_costs->m_num_fold_left_reduc_lanes)
+    return false;
+
+  /* Do not let emulated sub-SSE modes win on this alone.  */
+  loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
+  return known_ge (GET_MODE_SIZE (loop_vinfo->vector_mode), 16);
+}
+
 /* Return true if THIS should be preferred over OTHER as main vector loop.  */
 
 bool
@@ -27355,6 +27388,9 @@ ix86_vector_costs::better_main_loop_than_p (const vector_costs *other) const
   loop_vec_info this_loop_vinfo = as_a<loop_vec_info> (this->vinfo ());
   loop_vec_info other_loop_vinfo = as_a<loop_vec_info> (other->vinfo ());
 
+  if (better_fold_left_reduc_than_p (other))
+    return true;
+
   /* If the other loop is masked it does not need an epilog.  Prefer that
      if the current loop cannot be vectorized fully with a vector
      epilogs with at most one scalar iteration left.  */
@@ -27377,6 +27413,10 @@ ix86_vector_costs::better_epilogue_loop_than_p (const vector_costs *other,
 						loop_vec_info main_loop) const
 {
   loop_vec_info this_loop_info = as_a <loop_vec_info> (this->vinfo ());
+
+  if (better_fold_left_reduc_than_p (other))
+    return true;
+
   /* The x86 target allows for multiple vector epilogues, if THIS is
      the suggested epilog mode of OTHER then keep the latter unless
      THIS has a VF of one which means no further epilog needed.  */
diff --git a/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c b/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
index abb9f6681c2e..370e7ed4d11e 100644
--- a/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
+++ b/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
@@ -52,7 +52,6 @@ void test (const unsigned char * __restrict__ pi,
     pp_avg_rgb[3] = pp_avg_rgb_3;
 }
 
-/* Even though there's an SLP opportunity in-order reductions should never use
-   masked epilogs.  */
-/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte vectors" "vect" } } */
-/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using 32 byte vectors" "vect" } } */
+/* In-order reductions should not use masked epilogs here.  */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized using 16 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump-not "optimized: epilogue loop vectorized using masked" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c b/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
new file mode 100644
index 000000000000..634647287d8b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fdump-tree-vect-details" } */
+
+/* The two dependent updates form a fold-left reduction chain.  */
+
+float
+foo (float *__restrict__ a, int n)
+{
+  float sum = 0;
+  for (int i = 0; i != n; i++)
+    {
+      sum += a[2 * i];
+      sum += a[2 * i + 1];
+    }
+  return sum;
+}
+
+/* { dg-final { scan-tree-dump "Starting SLP discovery of reduction chain" "vect" } } */
+/* { dg-final { scan-tree-dump "in-order FP reduction lanes" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c b/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
new file mode 100644
index 000000000000..6babd761d6f2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fdump-tree-vect-details" } */
+
+/* The byte loads make V32QI available, but the fold-left reduction should
+   make the smaller SSE loop win.  */
+
+float
+foo (char *a, char *b, int n)
+{
+  float sum = 0;
+  for (int i = 0; i != n; i++)
+    sum += a[i] * b[i];
+  return sum;
+}
+
+/* { dg-final { scan-tree-dump "in-order FP reduction lanes" "vect" } } */
+/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump-not "loop vectorized using 32 byte vectors" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c b/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
index 8cd8740c6ecc..9f0445390f4f 100644
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
@@ -17,5 +17,4 @@ foo (double *a, char *mask, int n)
   return sum;
 }
 
-/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte vectors" "vect" } } */
-/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using 32 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized using 32 byte vectors" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c b/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
index 63c29895f9bb..be56819fc547 100644
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
@@ -17,5 +17,5 @@ foo (double *a, char *mask, int n)
   return sum;
 }
 
-/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte vectors" "vect" } } */
-/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using masked 64 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized using 32 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using masked 32 byte vectors" "vect" } } */