[PATCH v2 10/13] vect: Add HSSR analysis
Alfie Richards <[email protected]>
| Newsgroups | gmane.comp.gcc.patches |
|---|---|
| Message-ID | <[email protected]> |
Checks if a loop can use hardware safe speculative reads for
vectorization.
As part of this establishes HSSR regions, a region is attached to an
early break SLP instance where safe speculative reads would be required.
The vector loads that are required at the start of the instance define
the region, and then the nodes that use the values from those loads
are part of the region that need to use the mask of successfully loaded
elements.
gcc/ChangeLog:
* config/aarch64/aarch64.cc (better_main_loop_than_p):
Add code to optionally prefer HSSR loops.
* optabs-tree.cc (target_supports_hssr_mask_load_p):
New function.
(can_vec_hssr_mask_load_p): New function.
* optabs-tree.h (can_vec_hssr_mask_load_p): New function.
* tree-vect-data-refs.cc (vect_supportable_dr_alignment):
Add logic that unaligned safe speculative loads are supported
when using HSSR.
* tree-vect-loop-manip.cc (vect_can_add_hssr_controls):
New function.
* tree-vect-loop.cc (_loop_vec_info::_loop_vec_info):
Add new memebers must_use_hssr_p, can_use_hssr_p, using_hssr_p.
(vect_verify_full_masking): Add call to check
vect_can_add_hssr_controls.
(set_hssr_regions): New helper function.
(sort_instances_by_dominators): New helper function.
(vect_determine_hssr_and_versioning): New function.
(vectorizable_induction): Add logic to disable HSSR for multilane
inductions.
* tree-vect-slp.cc (_slp_tree::_slp_tree): Add initialization for
hssr_region and hssr_mask_needed.
(vect_free_slp_instance): Add handling for hssr_region.
(vect_build_slp_instance): Likewise.
(vect_analyze_slp_reduc_chain): Likewise.
(vect_analyze_slp_reduction): Likewise.
(vect_analyze_slp_reduction_group): Likewise.
(vect_analyze_slp_instance): Likewise.
* tree-vect-stmts.cc (check_load_for_hssr): New function.
(get_load_store_type): Add handling for HSSR.
(vectorizable_conversion): Add setting of
SLP_TREE_HSSR_MASK_NEEDED to false when no fault could trigger.
(vectorizable_load): Add call to check_load_for_hssr.
* tree-vectorizer.h (struct hssr_region): New struct.
(struct vect_load_store_data): Add hssr_ifn and defines_region
members.
(struct _slp_tree): Add hssr_region and hssr_mask_needed members.
(SLP_INSTANCE_HSSR_REGION): New macro.
(SLP_TREE_HSSR_REGION): Likewise.
(SLP_TREE_HSSR_MASK_NEEDED): Likewise.
(LOOP_VINFO_CAN_USE_HSSR_P): Likewise.
(LOOP_VINFO_MUST_USE_HSSR_P): Likewise.
(LOOP_VINFO_USING_HSSR_P): Likewise.
(add_stmt_cost): Add costing for HSSR regions.
(vect_can_add_hssr_controls): New function.
---
gcc/config/aarch64/aarch64.cc | 10 ++
gcc/optabs-tree.cc | 57 +++++++
gcc/optabs-tree.h | 2 +
gcc/tree-vect-data-refs.cc | 3 +-
gcc/tree-vect-loop-manip.cc | 36 ++++
gcc/tree-vect-loop.cc | 310 ++++++++++++++++++++++++++++++++++
gcc/tree-vect-slp.cc | 10 ++
gcc/tree-vect-stmts.cc | 66 ++++++++
gcc/tree-vectorizer.h | 54 +++++-
9 files changed, 546 insertions(+), 2 deletions(-)
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index dd5e6cdb838..7ea3bc0f061 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -19617,6 +19617,16 @@ better_main_loop_than_p (const vector_costs *uncast_other) const
auto other_estimated_vf = (vect_vf_for_cost (other_loop_vinfo)
* other->m_ops[i].vf_factor ());
+ if ((LOOP_VINFO_USING_HSSR_P (this_loop_vinfo)
+ != LOOP_VINFO_USING_HSSR_P (other_loop_vinfo))
+ && param_vect_hssr_usage == 2)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "HSSR: Preferring loop with HSSR\n");
+ return LOOP_VINFO_USING_HSSR_P (this_loop_vinfo);
+ }
+
/* If it appears that one loop could process the same amount of data
in fewer cycles, prefer that loop over the other one. */
fractional_cost this_cost
diff --git a/gcc/optabs-tree.cc b/gcc/optabs-tree.cc
index 1b80cac85c7..cd930e4ef37 100644
--- a/gcc/optabs-tree.cc
+++ b/gcc/optabs-tree.cc
@@ -600,6 +600,63 @@ can_vec_mask_load_store_p (machine_mode mode,
return false;
}
+/* Return true if the target has support for hssr masked load. */
+
+bool
+target_supports_hssr_mask_load_p (machine_mode mode, machine_mode mask_mode,
+ internal_fn *ifn)
+{
+ optab op = mask_ff_hssr_load_optab;
+ enum insn_code icode;
+ if ((icode = convert_optab_handler (op, mode, mask_mode)) != CODE_FOR_nothing)
+ {
+ if (ifn)
+ *ifn = IFN_MASK_FF_HSSR_LOAD;
+ return true;
+ }
+ return false;
+}
+
+/* Return true if target supports vector masked hssr load for mode.
+ An additional output in the last argument which is the IFN pointer.
+ We set IFN as MASK_FIRSTFAULT_LOAD. */
+
+bool
+can_vec_hssr_mask_load_p (machine_mode mode, machine_mode mask_mode,
+ internal_fn *ifn)
+{
+ machine_mode vmode;
+
+ /* If mode is vector mode, check it directly. */
+ if (VECTOR_MODE_P (mode))
+ return target_supports_hssr_mask_load_p (mode, mask_mode, ifn);
+
+ /* Otherwise, return true if there is some vector mode with
+ the mask load/store supported. */
+
+ /* See if there is any chance the mask load or store might be
+ vectorized. If not, punt. */
+ scalar_mode smode;
+ if (!is_a<scalar_mode> (mode, &smode))
+ return false;
+
+ vmode = targetm.vectorize.preferred_simd_mode (smode);
+ if (VECTOR_MODE_P (vmode)
+ && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
+ && target_supports_hssr_mask_load_p (vmode, mask_mode, ifn))
+ return true;
+
+ auto_vector_modes vector_modes;
+ targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
+ for (machine_mode base_mode : vector_modes)
+ if (related_vector_mode (base_mode, smode).exists (&vmode)
+ && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
+ && target_supports_hssr_mask_load_p (vmode, mask_mode, ifn))
+ return true;
+
+ return false;
+}
+
/* Return true if the target has support for len load/store.
We can support len load/store by either len_{load,store}
or mask_len_{load,store}.
diff --git a/gcc/optabs-tree.h b/gcc/optabs-tree.h
index dad9ed9b0ba..bc0ec93958e 100644
--- a/gcc/optabs-tree.h
+++ b/gcc/optabs-tree.h
@@ -52,6 +52,8 @@ bool target_supports_mask_load_store_p (machine_mode, machine_mode,
bool can_vec_mask_load_store_p (machine_mode, machine_mode, bool,
internal_fn * = nullptr,
vec<int> * = nullptr);
+bool can_vec_hssr_mask_load_p (machine_mode, machine_mode,
+ internal_fn * = nullptr);
opt_machine_mode get_len_load_store_mode (machine_mode, bool,
internal_fn * = nullptr,
vec<int> * = nullptr);
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index 92aecc656e1..e1b4381bc9c 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -6829,7 +6829,8 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
if (misalignment == 0)
return dr_aligned;
- else if (dr_safe_speculative_read_required (stmt_info))
+ else if (dr_safe_speculative_read_required (stmt_info)
+ && !(loop_vinfo && LOOP_VINFO_MUST_USE_HSSR_P (loop_vinfo)))
return dr_unaligned_unsupported;
if (loop_vinfo)
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 10fc0353078..d7a815ebaec 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -4922,3 +4922,39 @@ vect_loop_versioning (loop_vec_info loop_vinfo,
return nloop;
}
+
+/* Checks if it's possible to create the necessary controls for an HSSR loop.
+
+ For loops with HSSR we need to generate all the required masks from the
+ successfully loaded element mask. */
+
+
+bool
+vect_can_add_hssr_controls (loop_vec_info loop_vinfo)
+{
+ if (!LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo))
+ return false;
+
+ int i;
+ rgroup_controls *rgc;
+ FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo).rgc_vec, i, rgc)
+ {
+ /* The base case is handled by reading the HSSR value. */
+ if (i == 0)
+ continue;
+
+ unsigned int nmasks = i + 1;
+
+ rgroup_controls *half_controls_rgc
+ = &LOOP_VINFO_MASKS (loop_vinfo).rgc_vec[nmasks / 2 - 1];
+
+ if (!rgc->type)
+ return false;
+
+ /* Permute the masks from the half rgoup to this one. */
+ if (!vect_maybe_permute_loop_masks (NULL, rgc, half_controls_rgc))
+ return false;
+ }
+
+ return true;
+}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index b8eb58246a7..b54cdf35aed 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -755,6 +755,9 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in, vec_info_shared *shared)
vectorizable (false),
can_use_partial_vectors_p (true),
must_use_partial_vectors_p (false),
+ must_use_hssr_p (false),
+ can_use_hssr_p (true),
+ using_hssr_p (false),
using_partial_vectors_p (false),
using_decrementing_iv_p (false),
using_select_vl_p (false),
@@ -1102,6 +1105,18 @@ vect_verify_full_masking (loop_vec_info loop_vinfo)
LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo) = cmp_type;
LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo) = iv_type;
LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) = vect_partial_vectors_while_ult;
+
+ /* Check we can generate all the masks we need for HSSR. */
+ if (LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo)
+ && !vect_can_add_hssr_controls (loop_vinfo))
+ {
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "not using HSSR: Cannot generate the masks from the "
+ "HSSR mask.\n");
+ }
+
return true;
}
@@ -2140,6 +2155,284 @@ vect_determine_partial_vectors_and_peeling (loop_vec_info loop_vinfo,
return opt_result::success ();
}
+/* Sets the hssr region for NODE and its children to be REGION. */
+
+static void
+set_hssr_regions (loop_vec_info loop_vinfo, slp_tree node, hssr_region *region,
+ hash_set<slp_tree> &visited, hssr_region *load_region)
+{
+ /* If the node is an HSSR load, then return the region it defines. */
+ if (!node || visited.add (node) || SLP_TREE_TYPE (node) == load_vec_info_type
+ || SLP_TREE_TYPE (node) == phi_info_type || SLP_TREE_HSSR_REGION (node))
+ return;
+
+ SLP_TREE_HSSR_REGION (node)
+ = SLP_TREE_HSSR_MASK_NEEDED (node) ? region : load_region;
+
+ /* Propagate the region to all unset children. */
+ for (slp_tree child : SLP_TREE_CHILDREN (node))
+ set_hssr_regions (loop_vinfo, child, region, visited, load_region);
+}
+
+static int
+sort_instances_by_dominators (const void *instance_1_, const void *instance_2_,
+ void *)
+{
+ const slp_instance *instance_1 = (const slp_instance*) instance_1_;
+ const slp_instance *instance_2 = (const slp_instance*) instance_2_;
+
+ bool gcond_1_p = SLP_INSTANCE_KIND (*instance_1) == slp_inst_kind_gcond;
+ bool gcond_2_p = SLP_INSTANCE_KIND (*instance_2) == slp_inst_kind_gcond;
+
+ /* If neither of the instances aree GCOND, then we don't care, and these
+ should all come at the end and use the last mask. */
+ if (!gcond_1_p && !gcond_2_p)
+ return 0;
+ /* If only instance_2 is a GCOND, then that should be processed first. */
+ else if (!gcond_1_p)
+ return 1;
+ /* If only instance_1 is a GCOND, then that should be processed first. */
+ else if (!gcond_2_p)
+ return -1;
+
+ stmt_vec_info stmt_info_1
+ = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (*instance_1));
+ stmt_vec_info stmt_info_2
+ = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (*instance_2));
+
+ /* If both instances are GCONDS, then they should be ordered why which
+ statements BB dominates the other. */
+ if (vect_stmt_dominates_stmt_p (stmt_info_1->stmt, stmt_info_2->stmt))
+ return -1;
+ else if (vect_stmt_dominates_stmt_p (stmt_info_2->stmt, stmt_info_1->stmt))
+ return 1;
+ return 0;
+}
+
+/* Checks if hssr loads can be used for this loop and makes a
+ decision enabling them.
+
+ Establishes hssr-regions. */
+
+static opt_result
+vect_determine_hssr_and_versioning (loop_vec_info loop_vinfo)
+{
+ vec<slp_instance> instances;
+ hssr_region *current_region = NULL;
+ hssr_region *previous_region = NULL;
+ slp_instance instance;
+ int i;
+ hash_set<slp_tree> visited;
+
+ if (!LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo))
+ goto exit;
+
+ if (param_vect_hssr_usage == 0)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "HSSR: Disabled.\n");
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ goto exit;
+ }
+
+ if (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "HSSR: Loop not fully masked.\n");
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ goto exit;
+ }
+
+ if (param_vect_hssr_usage != 3 && !LOOP_VINFO_MUST_USE_HSSR_P (loop_vinfo)
+ && !LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "HSSR: There is no reason to.\n");
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ goto exit;
+ }
+
+ /* If any of the SLP instances cover more than a single lane
+ we cannot use HSSR at the moment, even if the number
+ of lanes is uniform throughout the SLP graph. */
+ for (slp_instance inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
+ if (SLP_TREE_LANES (SLP_INSTANCE_TREE (inst)) != 1
+ && !(SLP_INSTANCE_KIND (inst) == slp_inst_kind_store
+ && SLP_INSTANCE_TREE (inst)->ldst_lanes))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "HSSR: Can't handle multiple lanes.\n");
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ goto exit;
+ }
+
+ /* Sort the instances by their gcond domination. */
+
+ instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo).copy ();
+ instances.stablesort (sort_instances_by_dominators, NULL);
+
+ /* Work out the HSSR regions. */
+ FOR_EACH_VEC_ELT (instances, i, instance)
+ {
+ /* Check if the instance needs an HSSR region.
+
+ NOTE: This assumes all HSSR loads are leaf nodes.
+
+ This is a safe assumption as HSSR loads can't sensibly be made to
+ do conditional loads so can never take a mask, and we don't yet
+ support gather HSSR loads. */
+ if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_gcond)
+ {
+ SLP_INSTANCE_HSSR_REGION (instance) = new hssr_region ();
+ for (slp_tree load : SLP_INSTANCE_LOADS (instance))
+ {
+ gcc_assert (load->children.length () == 0);
+
+ /* Set the load "defines region" to this one. */
+ vect_load_store_data _ls_data{};
+ vect_load_store_data &ls = load->get_data (_ls_data);
+
+ stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (load);
+ if (!dr_safe_speculative_read_required (stmt_info))
+ continue;
+ const vect_memory_access_type memory_access_type
+ = ls.memory_access_type;
+ if (memory_access_type == VMAT_INVARIANT)
+ continue;
+
+ /* The number of elements of the load must equal the
+ vectorization factor, as we must only fault if the next
+ scalar iteration is a fault, and optionally load later
+ iterations.
+
+ This is checked by vectoriable_load.
+
+ TODO: We could support other cases and grain sizes here.
+ However, this would likely require separate HSSR regions
+ which would have high runtime cost. */
+ gcc_assert (known_eq (GET_MODE_NUNITS (TYPE_MODE (load->vectype)),
+ LOOP_VINFO_VECT_FACTOR (loop_vinfo)));
+
+ /* If this load is already handled by another region, we don't
+ need to handle it in this one. */
+ if (ls.defines_region)
+ continue;
+
+ ls.defines_region = SLP_INSTANCE_HSSR_REGION (instance);
+
+ /* This load will be part of the region for this break. */
+ SLP_INSTANCE_HSSR_REGION (instance)->loads.safe_push (load);
+ SLP_INSTANCE_HSSR_REGION (instance)->prev_region = current_region;
+
+ /* Set the region it uses for its mask to be the previous one.
+ */
+ SLP_TREE_HSSR_REGION (load) = current_region;
+ /* Set the read point to be the last load. */
+ if (!SLP_INSTANCE_HSSR_REGION (instance)->hssr_read_point
+ || vect_stmt_dominates_stmt_p
+ (SLP_INSTANCE_HSSR_REGION (instance)->hssr_read_point,
+ SLP_TREE_REPRESENTATIVE (load)->stmt))
+ SLP_INSTANCE_HSSR_REGION (instance)->hssr_read_point
+ = SLP_TREE_REPRESENTATIVE (load)->stmt;
+ }
+
+ /* If this region doesnt handle any loads, it isn't needed. */
+ if (SLP_INSTANCE_HSSR_REGION (instance)->loads.is_empty ())
+ {
+ delete SLP_INSTANCE_HSSR_REGION (instance);
+ SLP_INSTANCE_HSSR_REGION (instance) = NULL;
+ }
+ else
+ current_region = SLP_INSTANCE_HSSR_REGION (instance);
+ }
+
+ /* If none of the instances require loads, then HSSR is pointless. */
+ if (!current_region)
+ {
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "HSSR: No early break instances require HSSR.\n");
+ goto exit;
+ }
+
+ /* We can and will use HSSR. */
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "HSSR: Will use hssr\n");
+
+ /* We no longer need a bunch of versioning. */
+ LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
+ LOOP_VINFO_USING_HSSR_P (loop_vinfo) = true;
+ LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo) = vNULL;
+ LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (loop_vinfo) = false;
+ LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo) = 0;
+
+ /* Populate the hssr_regions for the nodes in the tree. */
+ set_hssr_regions (loop_vinfo, SLP_INSTANCE_TREE (instance),
+ current_region, visited, previous_region);
+ previous_region = current_region;
+ }
+
+ /* If none of the instances require loads, then HSSR is pointless. */
+ if (!current_region)
+ {
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "HSSR: No early break instances require HSSR.\n");
+ goto exit;
+ }
+
+ /* If none of the instances require loads, then HSSR is pointless. */
+ if (!current_region)
+ {
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "HSSR: No early break instances require HSSR.\n");
+ goto exit;
+ }
+
+ /* We can use HSSR. */
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "HSSR: Can use hssr\n");
+
+ /* If we are going to use versioning, then record that we can use HSSR, but
+ don't use it for this loop, as we can consider it for the versioned
+ alternative. */
+ if (param_vect_hssr_usage == 2
+ && LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "HSSR: Not using HSSR because versioning, will "
+ "consider it for versioned unaligned loop.\n");
+ goto exit;
+ }
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "HSSR: Will use hssr\n");
+
+ /* Can and will use HSSR. */
+ LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
+ LOOP_VINFO_USING_HSSR_P (loop_vinfo) = true;
+ LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo) = vNULL;
+ LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (loop_vinfo) = false;
+ LOOP_VINFO_MAX_SPEC_READ_AMOUNT (loop_vinfo) = 0;
+
+exit:
+ if (LOOP_VINFO_MUST_USE_HSSR_P (loop_vinfo)
+ && !LOOP_VINFO_USING_HSSR_P (loop_vinfo))
+ return opt_result::failure_at (vect_location,
+ "not vectorized: loop needs but cannot "
+ "use hssr\n");
+
+ return opt_result::success ();
+}
+
/* Function vect_analyze_loop_2.
Apply a set of analyses on LOOP specified by LOOP_VINFO, the different
@@ -2409,6 +2702,11 @@ start_over:
if (!ok)
return ok;
+ /* Check if we can use HSSR instead of peeling and/or versioning. */
+ ok = vect_determine_hssr_and_versioning (loop_vinfo);
+ if (!ok)
+ return ok;
+
/* If we're vectorizing a loop that uses length "controls" and
can iterate more than once, we apply decrementing IV approach
in loop control. */
@@ -2731,6 +3029,9 @@ again:
LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
= saved_can_use_partial_vectors_p;
LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ LOOP_VINFO_USING_HSSR_P (loop_vinfo) = false;
+ LOOP_VINFO_MUST_USE_HSSR_P (loop_vinfo) = false;
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = true;
LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = false;
@@ -9647,6 +9948,15 @@ vectorizable_induction (loop_vec_info loop_vinfo,
return false;
}
+ if (SLP_TREE_LANES (slp_node) != 1)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "HSSR: Cannot use HSSR as multiple-lane SLP inductions"
+ "aren't supported.\n");
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ }
+
if (FLOAT_TYPE_P (vectype) && !param_vect_induction_float)
{
if (dump_enabled_p ())
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index e4a29f85922..47d8bef38da 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -135,6 +135,8 @@ _slp_tree::_slp_tree ()
this->lanes = 0;
SLP_TREE_TYPE (this) = undef_vec_info_type;
this->data = NULL;
+ SLP_TREE_HSSR_REGION (this) = NULL;
+ SLP_TREE_HSSR_MASK_NEEDED (this) = true;
}
/* Tear down a SLP node. */
@@ -217,6 +219,7 @@ vect_free_slp_instance (slp_instance instance)
SLP_INSTANCE_REMAIN_DEFS (instance).release ();
instance->subgraph_entries.release ();
instance->cost_vec.release ();
+ delete SLP_INSTANCE_HSSR_REGION (instance);
free (instance);
}
@@ -4279,6 +4282,7 @@ vect_build_slp_instance (vec_info *vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_HSSR_REGION (new_instance) = NULL;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -4655,6 +4659,7 @@ vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_HSSR_REGION (new_instance) = NULL;
vinfo->slp_instances.safe_push (new_instance);
@@ -4721,6 +4726,7 @@ vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_HSSR_REGION (new_instance) = NULL;
vect_reduc_info reduc_info = info_for_reduction (vinfo, node);
reduc_info->is_reduc_chain = true;
@@ -4861,6 +4867,7 @@ vect_analyze_slp_reduction (loop_vec_info vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_HSSR_REGION (new_instance) = NULL;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -4933,6 +4940,7 @@ vect_analyze_slp_reduction_group (loop_vec_info loop_vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_HSSR_REGION (new_instance) = NULL;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -5189,6 +5197,7 @@ vect_analyze_slp_instance (vec_info *vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_HSSR_REGION (new_instance) = NULL;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -5436,6 +5445,7 @@ vect_analyze_slp_instance (vec_info *vinfo,
new_instance->reduc_phis = NULL;
new_instance->cost_vec = vNULL;
new_instance->subgraph_entries = vNULL;
+ SLP_INSTANCE_HSSR_REGION (new_instance) = NULL;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 7579c5c8cc6..b77688b6f97 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -1583,6 +1583,45 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
}
}
+/* Checks if a load can support HSSR. If supported sets ls->hssr_ifn to the
+ hssr load internal function. */
+
+static void
+check_load_for_hssr (loop_vec_info loop_vinfo, tree vectype, slp_tree slp_node,
+ vect_load_store_data *ls, int num_vec)
+{
+ vect_memory_access_type memory_access_type = ls->memory_access_type;
+
+ /* Never need HSSR for invariant addresses. */
+ if (memory_access_type == VMAT_INVARIANT)
+ return;
+
+ if (!LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo))
+ return;
+
+ machine_mode vecmode = TYPE_MODE (vectype);
+ machine_mode mask_mode;
+ /* For now, dont handle group_size != 1 or vec_num != 1. */
+ if (num_vec != 1 || slp_node->lanes != 1)
+ {
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "HSSR: multiple lane loads aren't supported\n");
+ }
+
+ /* If we can't do a hssr load, remember that. */
+ if (LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo)
+ && (!targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
+ || !can_vec_hssr_mask_load_p (vecmode, mask_mode, &ls->hssr_ifn)))
+ {
+ LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "HSSR: load not supported\n");
+ }
+}
+
/* Return the mask input to a masked load or store. VEC_MASK is the vectorized
form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
that needs to be applied to all loads and stores in a vectorized loop.
@@ -2520,6 +2559,19 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
= vect_supportable_dr_alignment
(vinfo, first_dr_info, vectype, *misalignment,
mat_gather_scatter_p (*memory_access_type));
+
+ /* If we would currently fail, but could be fine with HSSR, use HSSR. */
+ if (loop_vinfo
+ && *alignment_support_scheme == dr_unaligned_unsupported
+ && LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo)
+ && !LOOP_VINFO_MUST_USE_HSSR_P (loop_vinfo))
+ {
+ LOOP_VINFO_MUST_USE_HSSR_P (loop_vinfo) = true;
+ *alignment_support_scheme = vect_supportable_dr_alignment
+ (vinfo, first_dr_info, vectype, *misalignment,
+ mat_gather_scatter_p (*memory_access_type));
+ }
+
if (grouped_gather_fallback != VMAT_UNINITIALIZED
&& *alignment_support_scheme != dr_aligned
&& *alignment_support_scheme != dr_unaligned_supported)
@@ -5685,6 +5737,14 @@ vectorizable_conversion (vec_info *vinfo,
if (cost_vec) /* transformation not required. */
{
+ /* If this is not a conversion that could trap, then we do not require
+ a totally accurate HSSR mask.
+
+ This allows conversions to be emitted next to loads and be combined
+ into widening loads. */
+ if (!gimple_could_trap_p (stmt))
+ SLP_TREE_HSSR_MASK_NEEDED (slp_node) = false;
+
if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in)
|| !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))
{
@@ -9948,6 +10008,12 @@ vectorizable_load (vec_info *vinfo,
return false;
}
+ if (loop_vinfo
+ && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
+ && LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo)
+ && dr_safe_speculative_read_required (stmt_info))
+ check_load_for_hssr (loop_vinfo, vectype, slp_node, &ls, vec_num);
+
if (dump_enabled_p ()
&& memory_access_type != VMAT_ELEMENTWISE
&& !mat_gather_scatter_p (memory_access_type)
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 01e72b0955b..a52e3dae738 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -276,6 +276,31 @@ struct vect_simd_clone_data : vect_data {
auto_vec<tree> simd_clone_info;
};
+struct hssr_region {
+ hssr_region () = default;
+
+ /* The HSSR loads that create the mask that defines this region. */
+ vec<slp_tree> loads;
+
+ /* Where the HSSR and the fixup code for this region read should be
+ inserted. */
+ gimple* hssr_read_point;
+
+ /* The SSA node to store how much IV's should be incremented by. */
+ tree num_iter;
+
+ /* The SSA node for the mask of elements not processed by this vectorized
+ iteration that will need to be re-processed. */
+ tree next_iter_mask;
+
+ /* The SSA nodes for the masks that operations in this region will use. */
+ vec<vec<tree>> controls;
+
+ /* The previous region before this one, or NULL if this is the first
+ region. */
+ hssr_region *prev_region;
+};
+
/* Analysis data from vectorizable_load and vectorizable_store for
load_vec_info_type and store_vec_info_type. */
struct vect_load_store_data : vect_data {
@@ -287,6 +312,7 @@ struct vect_load_store_data : vect_data {
dr_alignment_support alignment_support_scheme;
int misalignment;
internal_fn lanes_ifn; // VMAT_LOAD_STORE_LANES
+ internal_fn hssr_ifn;
poly_int64 poffset;
union {
internal_fn ifn; // VMAT_GATHER_SCATTER_IFN
@@ -314,6 +340,10 @@ struct vect_load_store_data : vect_data {
unsigned n_loads; // SLP_TREE_LOAD_PERMUTATION
/* Whether the load permutation is consecutive and simple. */
bool subchain_p; // VMAT_STRIDED_SLP and VMAT_GATHER_SCATTER
+
+ /* If this is a hssr load, then the region that the resultant
+ mask will define. */
+ struct hssr_region *defines_region;
};
/* A computation tree of an SLP instance. Each node corresponds to a group of
@@ -406,6 +436,14 @@ struct _slp_tree {
/* Linked list of nodes to release when we free the slp_tree_pool. */
slp_tree next_node;
slp_tree prev_node;
+
+ /* The hssr region this node is goverened by and gets it's mask from.
+ Will be NULL if there is no hssr region for this node and it should use the
+ normal loop masks. */
+ struct hssr_region *hssr_region;
+ /* If this HSSR node needs to have an accurate HSSR mask, or can make do with
+ a possibly inaccurate superset mask. */
+ bool hssr_mask_needed;
};
/* The enum describes the type of operations that an SLP instance
@@ -451,6 +489,9 @@ public:
/* The type of operation the SLP instance is performing. */
slp_instance_kind kind;
+ /* The hssr region defining the mask for this node. */
+ hssr_region *region;
+
dump_user_location_t location () const;
} *slp_instance;
@@ -461,6 +502,7 @@ public:
#define SLP_INSTANCE_ROOT_STMTS(S) (S)->root_stmts
#define SLP_INSTANCE_REMAIN_DEFS(S) (S)->remain_defs
#define SLP_INSTANCE_KIND(S) (S)->kind
+#define SLP_INSTANCE_HSSR_REGION(S) (S)->region
#define SLP_TREE_CHILDREN(S) (S)->children
#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
@@ -480,6 +522,8 @@ public:
#define SLP_TREE_GS_BASE(S) (S)->gs_base
#define SLP_TREE_REDUC_IDX(S) (S)->cycle_info.reduc_idx
#define SLP_TREE_PERMUTE_P(S) ((S)->code == VEC_PERM_EXPR)
+#define SLP_TREE_HSSR_REGION(S) (S)->hssr_region
+#define SLP_TREE_HSSR_MASK_NEEDED(S) (S)->hssr_mask_needed
inline vect_memory_access_type
SLP_TREE_MEMORY_ACCESS_TYPE (slp_tree node)
@@ -1141,6 +1185,10 @@ public:
/* Records whether we must use niter masking for correctness reasons. */
bool must_use_partial_vectors_p;
+ bool must_use_hssr_p;
+ bool can_use_hssr_p;
+ bool using_hssr_p;
+
/* True if we've decided to use partially-populated vectors, so that
the vector loop can handle fewer than VF scalars. */
bool using_partial_vectors_p;
@@ -1302,6 +1350,9 @@ public:
#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
#define LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P(L) (L)->must_use_partial_vectors_p
#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
+#define LOOP_VINFO_CAN_USE_HSSR_P(L) (L)->can_use_hssr_p
+#define LOOP_VINFO_MUST_USE_HSSR_P(L) (L)->must_use_hssr_p
+#define LOOP_VINFO_USING_HSSR_P(L) (L)->using_hssr_p
#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
#define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)->allow_mutual_alignment
@@ -2133,7 +2184,7 @@ add_stmt_cost (vector_costs *costs, int count, enum vect_cost_for_stmt kind,
enum vect_cost_model_location where)
{
gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken
- || kind == scalar_stmt);
+ || kind == scalar_stmt || kind == hssr_read);
return add_stmt_cost (costs, count, kind, NULL, NULL, NULL_TREE, 0, where);
}
@@ -2505,6 +2556,7 @@ class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class loop *, edge,
bool = false, bool = false,
bool = true);
class loop *vect_loop_versioning (loop_vec_info, gimple *);
+extern bool vect_can_add_hssr_controls (loop_vec_info);
extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
tree *, tree *, tree *, int, bool, bool,
tree *);
--
2.43.0