[PATCH v2 03/13] aarch64: Add fold_contiguous_load helper function.

gcc/ChangeLog:

	* config/aarch64/aarch64-acle-builtins.cc
	(gimple_folder::fold_contiguous_load): New function.
	* config/aarch64/aarch64-acle-builtins.h
	(gimple_folder::fold_contiguous_load): New function.
	* config/aarch64/aarch64-sve-builtins-base.cc:
	(svld1_impl::fold): Change to use fold_contiguous_load.
	(svldxf1_extend_impl::fold): Change to use fold_contiguous_load.
	* config/aarch64/aarch64-sve.md
	(aarch64_ld<fn>f1_<ANY_EXTEND:optab><SVE_HSDI:mode><SVE_PARTIAL_I:mode>):
	Add else operand.

Co-Authored-By: Alex Coplan <[email protected]>
---
 gcc/config/aarch64/aarch64-acle-builtins.cc   | 24 +++++++++++++++++++
 gcc/config/aarch64/aarch64-acle-builtins.h    |  1 +
 .../aarch64/aarch64-sve-builtins-base.cc      | 20 ++--------------
 gcc/config/aarch64/aarch64-sve.md             |  1 +
 4 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/gcc/config/aarch64/aarch64-acle-builtins.cc b/gcc/config/aarch64/aarch64-acle-builtins.cc
index 98ba6c8d347..cf746171b4c 100644
--- a/gcc/config/aarch64/aarch64-acle-builtins.cc
+++ b/gcc/config/aarch64/aarch64-acle-builtins.cc
@@ -2850,6 +2850,30 @@ gimple_folder::fold_contiguous_base (gimple_seq &stmts, tree vectype)
   return base;
 }
 
+/* Common code to fold a contiguous load intrinsic call into a gimple
+   call to IFN_CODE.  */
+gimple *
+gimple_folder::fold_contiguous_load (internal_fn ifn_code)
+{
+  if (vectors_per_tuple () != 1)
+    return nullptr;
+
+  tree vectype = vector_type (0);
+
+  /* Get the predicate and base pointer.  */
+  gimple_seq stmts = NULL;
+  tree pred = convert_pred (stmts, vectype, 0);
+  tree base = fold_contiguous_base (stmts, vectype);
+  tree els = build_zero_cst (vectype);
+  gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
+
+  tree cookie = load_store_cookie (TREE_TYPE (vectype));
+  gcall *new_call = gimple_build_call_internal (ifn_code, 4,
+						base, cookie, pred, els);
+  gimple_call_set_lhs (new_call, lhs);
+  return new_call;
+}
+
 /* Return the alignment and TBAA argument to an internal load or store
    function like IFN_MASK_LOAD or IFN_MASK_STORE, given that it accesses
    memory elements of type TYPE.  */
diff --git a/gcc/config/aarch64/aarch64-acle-builtins.h b/gcc/config/aarch64/aarch64-acle-builtins.h
index 42150a2632b..4ce860450bb 100644
--- a/gcc/config/aarch64/aarch64-acle-builtins.h
+++ b/gcc/config/aarch64/aarch64-acle-builtins.h
@@ -650,6 +650,7 @@ public:
   tree force_vector (gimple_seq &, tree, tree);
   tree convert_pred (gimple_seq &, tree, unsigned int);
   tree fold_contiguous_base (gimple_seq &, tree);
+  gimple *fold_contiguous_load (internal_fn);
   tree load_store_cookie (tree);
 
   gcall *redirect_call (const function_instance &);
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
index 5d01d875a9d..800541a9262 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
@@ -1704,23 +1704,7 @@ public:
   gimple *
   fold (gimple_folder &f) const override
   {
-    if (f.vectors_per_tuple () != 1)
-      return nullptr;
-
-    tree vectype = f.vector_type (0);
-
-    /* Get the predicate and base pointer.  */
-    gimple_seq stmts = NULL;
-    tree pred = f.convert_pred (stmts, vectype, 0);
-    tree base = f.fold_contiguous_base (stmts, vectype);
-    tree els = build_zero_cst (vectype);
-    gsi_insert_seq_before (f.gsi, stmts, GSI_SAME_STMT);
-
-    tree cookie = f.load_store_cookie (TREE_TYPE (vectype));
-    gcall *new_call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
-						  base, cookie, pred, els);
-    gimple_call_set_lhs (new_call, f.lhs);
-    return new_call;
+    return f.fold_contiguous_load (IFN_MASK_LOAD);
   }
 
   rtx
@@ -2108,7 +2092,7 @@ public:
     insn_code icode = code_for_aarch64_ldf1 (m_unspec, extend_rtx_code (),
 					     e.vector_mode (0),
 					     e.memory_vector_mode ());
-    return e.use_contiguous_load_insn (icode);
+    return e.use_contiguous_load_insn (icode, true);
   }
 
   /* The unspec associated with the load.  */
diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md
index 3f59aa77437..878dadf6f61 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -1487,6 +1487,7 @@ (define_insn_and_rewrite "@aarch64_ld<fn>f1_<ANY_EXTEND:optab><SVE_HSDI:mode><SV
 	     (unspec:SVE_PARTIAL_I
 	       [(match_operand:<SVE_PARTIAL_I:VPRED> 2 "register_operand" "Upl")
 		(match_operand:SVE_PARTIAL_I 1 "aarch64_sve_ld<fn>f1_operand" "Ut<fn>")
+		(match_operand:SVE_PARTIAL_I 4 "aarch64_maskload_else_operand")
 		(reg:VNx16BI FFRT_REGNUM)]
 	       SVE_LDFF1_LDNF1))]
 	  UNSPEC_PRED_X))]
-- 
2.43.0