[gcc r17-3212] vect: fix virtual PHI after header rewrite [PR126778]

https://gcc.gnu.org/g:67c9a2c1c7ed4414151acef4a8e336c751959225

commit r17-3212-g67c9a2c1c7ed4414151acef4a8e336c751959225
Author: Tamar Christina <[email protected]>
Date:   Wed Aug 12 07:57:39 2026 +0100

    vect: fix virtual PHI after header rewrite [PR126778]
    
    The loop header rewrite in vect_set_loop_condition_partial_vectors runs after
    early break store movement and so it has to update any virtual operands on the
    exit edge.
    
    gcc/ChangeLog:
    
            PR tree-optimization/126778
            * tree-vect-loop-manip.cc (get_live_virtual_operand_on_edge): Move it.
            (vect_set_loop_condition_partial_vectors): Use it to update edge.
    
    gcc/testsuite/ChangeLog:
    
            PR tree-optimization/126778
            * gcc.dg/vect/vect-early-break_145-pr126778_1.c: New test.
            * gcc.dg/vect/vect-early-break_145-pr126778_2.c: New test.

Diff:
---
 .../gcc.dg/vect/vect-early-break_145-pr126778_1.c  | 16 +++++++
 .../gcc.dg/vect/vect-early-break_145-pr126778_2.c  | 45 +++++++++++++++++
 gcc/tree-vect-loop-manip.cc                        | 56 ++++++++++++----------
 3 files changed, 91 insertions(+), 26 deletions(-)

diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_1.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_1.c
new file mode 100644
index 000000000000..e49710367cf1
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_1.c
@@ -0,0 +1,16 @@
+/* PR tree-optimization/126778 */
+/* { dg-do compile } */
+/* { dg-add-options vect_early_break } */
+/* { dg-require-effective-target vect_early_break } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+/* { dg-additional-options "-march=armv8-a+sve" { target aarch64*-*-* } } */
+
+void
+glob3 (char *sc, short *dc)
+{
+  while (dc && (*dc++ = *sc))
+    ;
+}
+
+/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" { target aarch64*-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_2.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_2.c
new file mode 100644
index 000000000000..67092a1d37f4
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_2.c
@@ -0,0 +1,45 @@
+/* PR tree-optimization/126778 */
+/* { dg-do run } */
+/* { dg-add-options vect_early_break } */
+/* { dg-require-effective-target vect_early_break_hw } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target aarch64_sve_hw { target aarch64*-*-* } } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+/* { dg-additional-options "-march=armv8-a+sve" { target aarch64*-*-* } } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa))
+void
+glob3 (char *sc, short *dc)
+{
+  while (dc && (*dc++ = *sc))
+    ;
+}
+
+int
+main (void)
+{
+  check_vect ();
+
+  char c = 0;
+  short dst[16];
+
+#pragma GCC novector
+  for (int i = 0; i < 16; ++i)
+    dst[i] = 42;
+
+  glob3 (&c, dst);
+
+  if (dst[0] != 0)
+    __builtin_abort ();
+
+#pragma GCC novector
+  for (int i = 1; i < 16; ++i)
+    if (dst[i] != 42)
+      __builtin_abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" { target aarch64*-*-* } } } */
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 96c10340efd9..b54353399d9e 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -464,6 +464,32 @@ vect_iv_increment_position (edge loop_exit, gimple_stmt_iterator *bsi,
   *insert_after = false;
 }
 
+/* Get the virtual operand live on E.  The precondition on this is valid
+   immediate dominators and an actual virtual definition dominating E.  */
+/* ???  Costly band-aid.  For the use in question we can populate a
+   live-on-exit/end-of-BB virtual operand when copying stmts.  */
+
+static tree
+get_live_virtual_operand_on_edge (edge e)
+{
+  basic_block bb = e->src;
+  do
+    {
+      for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
+	{
+	  gimple *stmt = gsi_stmt (gsi);
+	  if (gimple_vdef (stmt))
+	    return gimple_vdef (stmt);
+	  if (gimple_vuse (stmt))
+	    return gimple_vuse (stmt);
+	}
+      if (gphi *vphi = get_virtual_phi (bb))
+	return gimple_phi_result (vphi);
+      bb = get_immediate_dominator (CDI_DOMINATORS, bb);
+    }
+  while (1);
+}
+
 /* If this is a loop where the latch condition should be rewritten to reflect
    a control flow change from a while-do to a do-while loop.  */
 
@@ -1031,6 +1057,10 @@ vect_set_loop_condition_partial_vectors (class loop *loop, edge exit_edge,
       latch_exit_edge->probability = exit_edge->probability;
       latch_exit_edge->count () = exit_edge->count ();
       copy_phi_arg_into_existing_phi (exit_edge, latch_exit_edge);
+      if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest))
+	SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge,
+				 get_live_virtual_operand_on_edge
+				   (latch_exit_edge));
       gimple_stmt_iterator latch_gsi = gsi_last_bb (latch);
       gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT);
       LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge;
@@ -1559,32 +1589,6 @@ vect_set_loop_condition (class loop *loop, edge loop_e, loop_vec_info loop_vinfo
 		     (gimple *) cond_stmt);
 }
 
-/* Get the virtual operand live on E.  The precondition on this is valid
-   immediate dominators and an actual virtual definition dominating E.  */
-/* ???  Costly band-aid.  For the use in question we can populate a
-   live-on-exit/end-of-BB virtual operand when copying stmts.  */
-
-static tree
-get_live_virtual_operand_on_edge (edge e)
-{
-  basic_block bb = e->src;
-  do
-    {
-      for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
-	{
-	  gimple *stmt = gsi_stmt (gsi);
-	  if (gimple_vdef (stmt))
-	    return gimple_vdef (stmt);
-	  if (gimple_vuse (stmt))
-	    return gimple_vuse (stmt);
-	}
-      if (gphi *vphi = get_virtual_phi (bb))
-	return gimple_phi_result (vphi);
-      bb = get_immediate_dominator (CDI_DOMINATORS, bb);
-    }
-  while (1);
-}
-
 /* Given LOOP this function generates a new copy of it and puts it
    on E which is either the entry or exit of LOOP.  If SCALAR_LOOP is
    non-NULL, assume LOOP and SCALAR_LOOP are equivalent and copy the