[gcc r17-2282] tree-ssa-math-opts: separate FMA-deferring trigger from reassoc reorder

https://gcc.gnu.org/g:ea6244c6f8db7663aca4e684c3105325d8282ede

commit r17-2282-gea6244c6f8db7663aca4e684c3105325d8282ede
Author: Philipp Tomsich <[email protected]>
Date:   Thu Jul 9 18:08:08 2026 +0200

    tree-ssa-math-opts: separate FMA-deferring trigger from reassoc reorder
    
    param_avoid_fma_max_bits gates two independent transforms: the
    widening_mul FMA-deferring in tree-ssa-math-opts.cc, which leaves a
    loop-carried multiply-add as fmul + fadd, and the reassoc
    loop-carried-FMA reorder added in r14-5779-g746344dd538
    (PR tree-optimization/110279), which parallelises 3+ operand chains.
    Both fire when TYPE_SIZE (elt) <= avoid-fma-max-bits, so for a given
    type they switch on at the same threshold and a target cannot keep one
    while dropping the other.
    
    This hurts the AArch64 AVOID_CROSS_LOOP_FMA cores (the Ampere-1
    family): a 2-operand reduction such as an sgemm inner K-loop is left as
    fmul + fadd, slower than fmadd on their dispatch-bound pipeline, yet
    setting avoid-fma-max-bits to 0 to avoid it also disables the reorder.
    
    Add a new --param=widening-mul-defer-fma (default 1) that gates only the
    widening_mul deferring; avoid-fma-max-bits keeps gating the reorder
    alone.  The AVOID_CROSS_LOOP_FMA callback sets the param to 0.  No other
    target and no default behaviour changes.
    
    gcc/ChangeLog:
    
            * doc/params.texi (widening-mul-defer-fma): Document.
            * params.opt (-param=widening-mul-defer-fma=): New param.
            * tree-ssa-math-opts.cc (math_opts_dom_walker::after_dom_children):
            Gate fma_deferring_state's enable predicate on
            param_widening_mul_defer_fma on top of param_avoid_fma_max_bits.
            * config/aarch64/aarch64-tuning-flags.def (AVOID_CROSS_LOOP_FMA):
            Update the comment for the widening-mul-defer-fma effect.
            * config/aarch64/aarch64.cc (aarch64_override_options_internal):
            Inside the AARCH64_EXTRA_TUNE_AVOID_CROSS_LOOP_FMA block, set
            param_widening_mul_defer_fma to 0.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.target/aarch64/widening-mul-defer-fma-1.c: New test.
            * gcc.target/aarch64/widening-mul-defer-fma-2.c: New test.
            * gcc.target/aarch64/widening-mul-defer-fma-3.c: New test.

Diff:
---
 gcc/config/aarch64/aarch64-tuning-flags.def         |  4 ++++
 gcc/config/aarch64/aarch64.cc                       | 12 +++++++++---
 gcc/doc/params.texi                                 |  6 ++++++
 gcc/params.opt                                      |  6 ++++++
 .../gcc.target/aarch64/widening-mul-defer-fma-1.c   | 19 +++++++++++++++++++
 .../gcc.target/aarch64/widening-mul-defer-fma-2.c   | 21 +++++++++++++++++++++
 .../gcc.target/aarch64/widening-mul-defer-fma-3.c   | 21 +++++++++++++++++++++
 gcc/tree-ssa-math-opts.cc                           |  3 ++-
 8 files changed, 88 insertions(+), 4 deletions(-)

diff --git a/gcc/config/aarch64/aarch64-tuning-flags.def b/gcc/config/aarch64/aarch64-tuning-flags.def
index 058dadecccaa..32bc1c4f3eec 100644
--- a/gcc/config/aarch64/aarch64-tuning-flags.def
+++ b/gcc/config/aarch64/aarch64-tuning-flags.def
@@ -40,6 +40,10 @@ AARCH64_EXTRA_TUNING_OPTION ("cse_sve_vl_constants", CSE_SVE_VL_CONSTANTS)
 
 AARCH64_EXTRA_TUNING_OPTION ("matched_vector_throughput", MATCHED_VECTOR_THROUGHPUT)
 
+/* For cores whose pipeline disfavours loop-carried serial FMAs: set
+   avoid-fma-max-bits to its maximum (512, i.e. all FMA widths) to enable
+   the reassoc reorder for 3+ operand chains, and set widening-mul-defer-fma
+   to 0 to suppress the widening-mul pass's FMA deferring.  */
 AARCH64_EXTRA_TUNING_OPTION ("avoid_cross_loop_fma", AVOID_CROSS_LOOP_FMA)
 
 AARCH64_EXTRA_TUNING_OPTION ("fully_pipelined_fma", FULLY_PIPELINED_FMA)
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 5c018d929861..f76db91d4238 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -20095,11 +20095,17 @@ aarch64_override_options_internal (struct gcc_options *opts,
       && opts->x_optimize >= aarch64_tune_params.prefetch->default_opt_level)
     opts->x_flag_prefetch_loop_arrays = 1;
 
-  /* Avoid loop-dependant FMA chains.  */
+  /* Avoid loop-dependant FMA chains.  The reassoc-side reorder helper
+     keeps using --param=avoid-fma-max-bits; the widening-mul-side
+     deferring is gated separately by --param=widening-mul-defer-fma, so we
+     suppress only the deferring on these cores while leaving the reassoc
+     reorder active.  */
   if (aarch64_tune_params.extra_tuning_flags
       & AARCH64_EXTRA_TUNE_AVOID_CROSS_LOOP_FMA)
-    SET_OPTION_IF_UNSET (opts, opts_set, param_avoid_fma_max_bits,
-			 512);
+    {
+      SET_OPTION_IF_UNSET (opts, opts_set, param_avoid_fma_max_bits, 512);
+      SET_OPTION_IF_UNSET (opts, opts_set, param_widening_mul_defer_fma, 0);
+    }
 
   /* Consider fully pipelined FMA in reassociation.  */
   if (aarch64_tune_params.extra_tuning_flags
diff --git a/gcc/doc/params.texi b/gcc/doc/params.texi
index 28cfe73d9d73..26b34a59aa98 100644
--- a/gcc/doc/params.texi
+++ b/gcc/doc/params.texi
@@ -1704,6 +1704,12 @@ reassociation considers the benefit of parallelizing FMA's multiplication
 part and addition part, assuming FMUL and FMA use the same units that can
 also do FADD.
 
+@paindex widening-mul-defer-fma
+@item widening-mul-defer-fma
+When nonzero, the widening-multiply pass defers forming an FMA whose result
+feeds a loop-header PHI, leaving a separate multiply and add.  Set to zero to
+contract such loop-carried reductions into an FMA instead.
+
 @paindex sms-loop-average-count-threshold
 @item sms-loop-average-count-threshold
 A threshold on the average loop count considered by the swing modulo scheduler.
diff --git a/gcc/params.opt b/gcc/params.opt
index 90f9943c8cb4..044c4a10bc44 100644
--- a/gcc/params.opt
+++ b/gcc/params.opt
@@ -1323,4 +1323,10 @@ Maximum number of outgoing edges in a switch before VRP does not process it.
 Common Joined UInteger Var(param_vrp_vector_threshold) Init(250) Optimization Param
 Maximum number of basic blocks for VRP to use a basic cache vector.
 
+-param=widening-mul-defer-fma=
+Common Joined UInteger Var(param_widening_mul_defer_fma) Init(1) IntegerRange(0, 1) Param Optimization
+When nonzero, the widening-multiply pass defers forming an FMA whose result
+feeds a loop-header PHI, leaving a separate multiply and add.  Set to zero to
+contract such loop-carried reductions into an FMA instead.
+
 ; This comment is to ensure we retain the blank line above.
diff --git a/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-1.c b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-1.c
new file mode 100644
index 000000000000..dbee0ecb62b6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-1.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ffast-math -fno-tree-vectorize -mcpu=ampere1" } */
+
+/* The ampere1 family sets AARCH64_EXTRA_TUNE_AVOID_CROSS_LOOP_FMA, which
+   sets widening-mul-defer-fma to 0 so that a 2-operand loop-carried
+   reduction is contracted to fmadd rather than deferred to fmul + fadd.
+   (avoid-fma-max-bits stays 512 to keep the reassoc reorder enabled.)  */
+
+double
+dot (const double *a, const double *b, int n)
+{
+  double s = 0.0;
+  for (int i = 0; i < n; i++)
+    s += a[i] * b[i];
+  return s;
+}
+
+/* { dg-final { scan-assembler {\tfmadd\t} } } */
+/* { dg-final { scan-assembler-not {\tfmul\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-2.c b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-2.c
new file mode 100644
index 000000000000..a73d1ac1e9d3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-2.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ffast-math -fno-tree-vectorize --param=avoid-fma-max-bits=512" } */
+
+/* With FMA deferring active (avoid-fma-max-bits > 0) and the default
+   widening-mul-defer-fma=1, the widening_mul pass refuses to form an FMA
+   whose result feeds the loop-header phi: the reduction stays as a
+   separate fmul + fadd.  This is the behaviour the new param decouples
+   from the reassoc reorder (also gated by avoid-fma-max-bits).  */
+
+double
+dot (const double *a, const double *b, int n)
+{
+  double s = 0.0;
+  for (int i = 0; i < n; i++)
+    s += a[i] * b[i];
+  return s;
+}
+
+/* { dg-final { scan-assembler {\tfmul\t} } } */
+/* { dg-final { scan-assembler {\tfadd\t} } } */
+/* { dg-final { scan-assembler-not {\tfmadd\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-3.c b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-3.c
new file mode 100644
index 000000000000..6463ca617734
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-3.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ffast-math -fno-tree-vectorize --param=avoid-fma-max-bits=512 --param=widening-mul-defer-fma=0" } */
+
+/* Companion to widening-mul-defer-fma-2.c: the same reduction with FMA
+   avoidance still requested for the reassoc reorder (avoid-fma-max-bits=512)
+   but --param=widening-mul-defer-fma=0 now suppresses the widening_mul
+   deferring, so the loop-carried multiply-add is contracted to a single
+   fmadd.  This proves the new param decouples the deferring from
+   avoid-fma-max-bits.  */
+
+double
+dot (const double *a, const double *b, int n)
+{
+  double s = 0.0;
+  for (int i = 0; i < n; i++)
+    s += a[i] * b[i];
+  return s;
+}
+
+/* { dg-final { scan-assembler {\tfmadd\t} } } */
+/* { dg-final { scan-assembler-not {\tfmul\t} } } */
diff --git a/gcc/tree-ssa-math-opts.cc b/gcc/tree-ssa-math-opts.cc
index f0ede668d95e..c4a1d7bcf0bd 100644
--- a/gcc/tree-ssa-math-opts.cc
+++ b/gcc/tree-ssa-math-opts.cc
@@ -6607,7 +6607,8 @@ math_opts_dom_walker::after_dom_children (basic_block bb)
 {
   gimple_stmt_iterator gsi;
 
-  fma_deferring_state fma_state (param_avoid_fma_max_bits > 0);
+  fma_deferring_state fma_state (param_avoid_fma_max_bits > 0
+				 && param_widening_mul_defer_fma);
 
   for (gphi_iterator psi_next, psi = gsi_start_phis (bb); !gsi_end_p (psi);
        psi = psi_next)