[gcc r17-3216] tree-optimization/126789 - MMX and SSE mask interaction with AVX512

https://gcc.gnu.org/g:5d123455af27c526d9aa1e12a47d0177262b0e51

commit r17-3216-g5d123455af27c526d9aa1e12a47d0177262b0e51
Author: Richard Biener <[email protected]>
Date:   Wed Aug 12 08:59:18 2026 +0200

    tree-optimization/126789 - MMX and SSE mask interaction with AVX512
    
    The following makes the vectorizer deal with the fact that the x86
    backend can have a set of integer mode masks and a set of integer
    vector mode masks depending on active ISAs.  To convert between
    both representations we have to use vcond_mask for integer mode
    to integer vector mode.  For the case of V2SImode the backend
    currently misses patterns which the following adds.  Now, when
    doing BB vectorization the vectorizer pattern recognition cannot
    know that we'll use V2SImode with a vector integer mask mode but
    it guesses V16SImode which would use DImode.  So we cannot arrange
    for the mask representation conversion during pattern recognition,
    but it's easy to handle VECTOR_BOOLEAN_TYPE_P conversions with
    vcond_mask as option in vectorizable_conversion, so that's what the
    patch does.
    
    I don't have a testcase for the reverse which would need to use
    vcmp[_eq] to convert from integer vector mode to integer mode mask,
    so I'm not implementing that with this patch.
    
            PR tree-optimization/126789
            * config/i386/mmx.md (mmxxmmmode): Add V2SF and V2SI.
            (mmxxmmmodelower): Likewise.
            (vcond_mask_<mode>qi): New expander for V2SF and V2SI.
            * tree-vect-stmts.cc (vectorizable_conversion): Handle
            conversion via VEC_COND_EXPR.
            (supportable_indirect_convert_operation): For mask vector
            conversions to vector integer mode try using a VEC_COND_EXPR.
    
            * gcc.target/i386/vect-pr126789-2.c: New testcase.

Diff:
---
 gcc/config/i386/mmx.md                          | 24 +++++++++++--
 gcc/testsuite/gcc.target/i386/vect-pr126789-2.c | 19 ++++++++++
 gcc/tree-vect-stmts.cc                          | 48 ++++++++++++++++++++++---
 3 files changed, 84 insertions(+), 7 deletions(-)

diff --git a/gcc/config/i386/mmx.md b/gcc/config/i386/mmx.md
index f61335af0b9c..640b2c0e63f5 100644
--- a/gcc/config/i386/mmx.md
+++ b/gcc/config/i386/mmx.md
@@ -2085,11 +2085,13 @@
 
 (define_mode_attr mmxxmmmode
   [(V2HF "V8HF") (V2HI "V8HI") (V2BF "V8BF")
-   (V4HF "V8HF") (V4HI "V8HI") (V4BF "V8BF")])
+   (V4HF "V8HF") (V4HI "V8HI") (V4BF "V8BF")
+   (V2SF "V4SF") (V2SI "V4SI")])
 
 (define_mode_attr mmxxmmmodelower
   [(V2HF "v8hf") (V2HI "v8hi") (V2BF "v8bf")
-   (V4HF "v8hf") (V4HI "v8hi") (V4BF "v8bf")])
+   (V4HF "v8hf") (V4HI "v8hi") (V4BF "v8bf")
+   (V2SF "v4sf") (V2SI "v4si")])
 
 (define_expand "movd_<mode>_to_sse"
   [(set (match_operand:<mmxxmmmode> 0 "register_operand")
@@ -2341,6 +2343,24 @@
   DONE;
 })
 
+(define_expand "vcond_mask_<mode>qi"
+  [(set (match_operand:V2FI 0 "register_operand")
+	(vec_merge:V2FI
+	  (match_operand:V2FI 1 "register_operand")
+	  (match_operand:V2FI 2 "register_operand")
+	  (match_operand:QI 3 "register_operand")))]
+  "TARGET_MMX_WITH_SSE && TARGET_AVX512VL"
+{
+  rtx op0 = gen_reg_rtx (<mmxxmmmode>mode);
+  operands[1] = lowpart_subreg (<mmxxmmmode>mode, operands[1], <MODE>mode);
+  operands[2] = lowpart_subreg (<mmxxmmmode>mode, operands[2], <MODE>mode);
+  emit_insn (gen_vcond_mask_<mmxxmmmodelower>qi (op0, operands[1],
+						 operands[2], operands[3]));
+  emit_move_insn (operands[0],
+		  lowpart_subreg (<MODE>mode, op0, <mmxxmmmode>mode));
+  DONE;
+})
+
 (define_expand "vec_cmpv2hfqi"
   [(set (match_operand:QI 0 "register_operand")
 	(match_operator:QI 1 ""
diff --git a/gcc/testsuite/gcc.target/i386/vect-pr126789-2.c b/gcc/testsuite/gcc.target/i386/vect-pr126789-2.c
new file mode 100644
index 000000000000..85d177bc3a40
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-pr126789-2.c
@@ -0,0 +1,19 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2 -mavx512vl -fno-vect-cost-model" } */
+
+int foo (double g, int f, double *r, int *s)
+{
+  int hu = 0;
+  bool test0 = r[0] < g;
+  bool test1 = r[1] < g;
+  bool test2 = s[0] < f;
+  bool test3 = s[1] < f;
+  hu += (test0 & test2) + (test1 & test3);
+  return hu;
+}
+
+/* { dg-final { scan-assembler "vcmppd" } } */
+/* That we use vpcmpgtd and not vpcmpd is because ix86_get_mask_mode
+   does not get us QImode for MMX modes.  But we should be able to
+   inter-operate with mixed SSE/AVX512 masks and vectorize the reduction.  */
+/* { dg-final { scan-assembler "vpcmpgtd" } } */
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 518ce00d585a..cfbd2040b8b0 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -5778,12 +5778,33 @@ vectorizable_conversion (vec_info *vinfo,
 	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 	      vop0 = new_temp;
 	    }
-	  new_stmt = vect_gimple_build (vec_dest, code1, vop0);
-	  new_temp = make_ssa_name (vec_dest, new_stmt);
-	  gimple_set_lhs (new_stmt, new_temp);
-	  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+	  if (code1 == COND_EXPR)
+	    {
+	      gcc_assert (!multi_step_cvt);
+	      new_stmt
+		= gimple_build_assign (vec_dest, VEC_COND_EXPR, vop0,
+				       build_minus_one_cst
+					 (TREE_TYPE (vec_dest)),
+				       build_zero_cst (TREE_TYPE (vec_dest)));
+	      new_temp = make_ssa_name (vec_dest, new_stmt);
+	      gimple_set_lhs (new_stmt, new_temp);
+	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+	      tree new_temp2 = make_ssa_name (vectype_out);
+	      new_stmt = gimple_build_assign (new_temp2,
+					      build1 (VIEW_CONVERT_EXPR,
+						      vectype_out, new_temp));
+	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+	      slp_node->push_vec_def (new_temp2);
+	    }
+	  else
+	    {
+	      new_stmt = vect_gimple_build (vec_dest, code1, vop0);
+	      new_temp = make_ssa_name (vec_dest, new_stmt);
+	      gimple_set_lhs (new_stmt, new_temp);
+	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 
-	  slp_node->push_vec_def (new_stmt);
+	      slp_node->push_vec_def (new_stmt);
+	    }
 	}
       break;
 
@@ -14551,6 +14572,23 @@ supportable_indirect_convert_operation (code_helper code,
       return true;
     }
 
+  /* For conversions between mask types where the destination has
+     a data mode attempt a vcond_mask conversion.  */
+  if (VECTOR_BOOLEAN_TYPE_P (vectype_in)
+      && VECTOR_BOOLEAN_TYPE_P (vectype_out)
+      && GET_MODE_CLASS (TYPE_MODE (vectype_out)) == MODE_VECTOR_INT)
+    {
+      tree scalar_datatype
+	= build_nonstandard_integer_type (element_precision (vectype_out), 0);
+      tree datatype_out = build_vector_type_for_mode (scalar_datatype,
+						      TYPE_MODE (vectype_out));
+      if (expand_vec_cond_expr_p (datatype_out, vectype_in))
+	{
+	  converts.safe_push (std::make_pair (datatype_out, COND_EXPR));
+	  return true;
+	}
+    }
+
   /* For conversions between float and integer types try whether
      we can use intermediate signed integer types to support the
      conversion.  */