[PATCH v14 02/19] vect: Update constant creation for BB SLP with predicated tails

gcc/ChangeLog:

	* gimple-fold.cc (gimple_build_vector): Check that the
	passed-in vector builder produces element values that
	are constant zero if they lie outside the minimum length
	of a variable-length vector type, and also a repeating tail
	of constant zeros.
	* tree-vect-slp.cc (vect_create_constant_vectors):
	Initialize the vector builder with two elements per pattern
	instead of one,	if doing BB SLP vectorization and the vector
	type is	variable-length or the SLP group has fewer lanes than
	needed to fill a fixed-length vector type.  Assign a zero
	constant to the second (repeating) element of each pattern.
---
 gcc/gimple-fold.cc   | 33 +++++++++++++++++++++--
 gcc/tree-vect-slp.cc | 62 +++++++++++++++++++++++++++++++++++---------
 2 files changed, 81 insertions(+), 14 deletions(-)

diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc
index fdb9b419766..132205c6ba9 100644
--- a/gcc/gimple-fold.cc
+++ b/gcc/gimple-fold.cc
@@ -11487,7 +11487,13 @@ gimple_build_vector_from_val (gimple_stmt_iterator *gsi,
    BUILDER must not have a stepped encoding on entry.  This is because
    the function is not geared up to handle the arithmetic that would
    be needed in the variable case, and any code building a vector that
-   is known to be constant should use BUILDER->build () directly.  */
+   is known to be constant should use BUILDER->build () directly.
+
+   If at least one element is non-constant and the type associated with BUILDER
+   is a variable-length vector type then the encoding used by BUILDER must have
+   two elements per pattern; the second (repeating) element of each pattern
+   must be constant zero, as must the first if it falls outside the minimum
+   vector length.  */
 
 tree
 gimple_build_vector (gimple_stmt_iterator *gsi,
@@ -11501,7 +11507,30 @@ gimple_build_vector (gimple_stmt_iterator *gsi,
       {
 	gimple_seq seq = NULL;
 	tree type = builder->type ();
-	unsigned int nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
+	unsigned int min_nelts
+	  = constant_lower_bound (TYPE_VECTOR_SUBPARTS (type));
+
+	/* All vector builders produce an infinitely repeating sequence; we only
+	   care about truncation of that sequence if the vector type has a
+	   variable length.  */
+	unsigned HOST_WIDE_INT nelts;
+	if (!TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
+	  {
+	    /* Initial elements outside the minimum vector length must be
+	       constant zero because that is the only implicit value that a
+	       CONSTRUCTOR node can have.  */
+	    for (i = min_nelts; i < builder->npatterns (); ++i)
+	      gcc_assert (zerop ((*builder)[i]));
+
+	    /* The builder must also produce an infinitely repeating tail of
+	       constant zeros.  */
+	    gcc_assert (builder->nelts_per_pattern () == 2);
+	    for (i = builder->npatterns (); i < encoded_nelts; ++i)
+	      gcc_assert (zerop ((*builder)[i]));
+
+	    nelts = min_nelts;
+	  }
+
 	vec<constructor_elt, va_gc> *v;
 	vec_alloc (v, nelts);
 	for (i = 0; i < nelts; ++i)
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 510e8daeec6..71b7288c94f 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -10768,7 +10768,7 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
   unsigned j, number_of_places_left_in_vector;
   tree vector_type;
   tree vop;
-  int group_size = op_node->ops.length ();
+  unsigned int group_size = op_node->ops.length ();
   unsigned int vec_num, i;
   unsigned number_of_copies = 1;
   bool constant_p;
@@ -10798,18 +10798,49 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
      (s1, s2, ..., s8).  We will create two vectors {s1, s2, s3, s4} and
      {s5, s6, s7, s8}.  */
 
-  /* When using duplicate_and_interleave, we just need one element for
-     each scalar statement.  */
-  if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
-    nunits = group_size;
+  unsigned int npatterns, nelts_per_pattern = 1;
+  poly_uint64 type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
+  if (is_a <bb_vec_info> (vinfo))
+    {
+      /* We don't use duplicate_and_interleave for basic block vectorization.
+	 We know that either the group size is exactly divisible by the vector
+	 length or it fits within a single vector.  */
+      gcc_checking_assert (multiple_p (group_size, type_nunits)
+			   || known_le (group_size, type_nunits));
+
+      /* If the vector length is unknown then we need two elements per pattern
+	 to encode an infinite-length tail of constant zeros.  */
+      if (!type_nunits.is_constant (&nunits))
+	{
+	  nunits = constant_lower_bound (type_nunits);
+	  nelts_per_pattern = 2;
+	}
+
+      npatterns = nunits;
+
+      /* If the group does not occupy a full vector then only the first
+	 GROUP_SIZE elements come from scalar operands.  The remainder are
+	 assigned constant zero below.  */
+      if (group_size < nunits)
+	  nunits = group_size;
+    }
+  else
+    {
+      /* When using duplicate_and_interleave, we just need one element for
+	 each scalar statement.  */
+      if (!type_nunits.is_constant (&nunits))
+	nunits = group_size;
+
+      npatterns = nunits;
+    }
 
   number_of_copies = nunits * number_of_vectors / group_size;
 
   number_of_places_left_in_vector = nunits;
   constant_p = true;
   tree uniform_elt = NULL_TREE;
-  tree_vector_builder elts (vector_type, nunits, 1);
-  elts.quick_grow (nunits);
+  tree_vector_builder elts (vector_type, npatterns, nelts_per_pattern);
+  elts.quick_grow (elts.encoded_nelts());
   stmt_vec_info insert_after = NULL;
   for (j = 0; j < number_of_copies; j++)
     {
@@ -10893,10 +10924,17 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
 
           if (number_of_places_left_in_vector == 0)
             {
-	      auto type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
 	      if (uniform_elt)
 		vec_cst = gimple_build_vector_from_val (&ctor_seq, vector_type,
 							elts[0]);
+	      else if (is_a <bb_vec_info> (vinfo))
+	      {
+		unsigned int encoded_nelts = elts.encoded_nelts ();
+		tree zero_val = build_zero_cst (TREE_TYPE (vector_type));
+		for (unsigned int k = nunits; k < encoded_nelts; k++)
+		  elts[k] = zero_val;
+		vec_cst = gimple_build_vector (&ctor_seq, &elts);
+	      }
 	      else if (constant_p
 		       ? multiple_p (type_nunits, nunits)
 		       : known_eq (type_nunits, nunits))
@@ -10946,10 +10984,10 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
 	      insert_after = NULL;
               number_of_places_left_in_vector = nunits;
 	      constant_p = true;
-	      elts.new_vector (vector_type, nunits, 1);
-	      elts.quick_grow (nunits);
-            }
-        }
+	      elts.new_vector (vector_type, npatterns, nelts_per_pattern);
+	      elts.quick_grow (elts.encoded_nelts ());
+	    }
+	}
     }
 
   /* Since the vectors are created in the reverse order, we should invert
-- 
2.43.0