[PATCH v5 19/22] RISC-V: Add load/store macro-fusion scheduling priorities

The sched_fusion pass can bring independent memory accesses together
before sched2.  Add a priority hook that groups scalar load/store-pair
candidates by register file, access kind, mode and base register, then
orders each group by offset.  Prefer decreasing offsets when both
directions are available, matching frame save/restore memory order.

Enable scheduling fusion by default for load/store-pair tunes while
preserving an explicit -f[no-]schedule-fusion option.  Keep the positive
test as XFAIL until the tuning patch enables the pairs.

Static sched2 macro-fusion counts for SPEC CPU 2017 Integer built with
-O3 -flto, measured independently of ready-list reordering against a
baseline with xt-c9501fdvt fusion pairs enabled:

Benchmark             Baseline  With patch  Change
500.perlbench_r           15302       17077  +11.6%
502.gcc_r                 72806       78135   +7.3%
505.mcf_r                    63          85  +34.9%
520.omnetpp_r             20479       22084   +7.8%
523.xalancbmk_r           42442       44651   +5.2%
525.x264_r                 6232        7033  +12.9%
531.deepsjeng_r             299         328   +9.7%
541.leela_r                 794         851   +7.2%
548.exchange2_r             353         426  +20.7%
557.xz_r                   1217        1321   +8.5%
SUM                      159987      171991   +7.5%

gcc/ChangeLog:

	* config/riscv/riscv-fusion.cc (riscv_sched_fusion_priority): New
	function.
	* config/riscv/riscv-protos.h (riscv_sched_fusion_priority):
	Declare.
	* config/riscv/riscv.cc (riscv_override_options_internal): Set the
	scheduling-fusion default for the current tune.
	(TARGET_SCHED_FUSION_PRIORITY): Define.

gcc/testsuite/ChangeLog:

	* gcc.target/riscv/sched-fusion-priority.c: New test.

Signed-off-by: Jin Ma <[email protected]>
---
 gcc/config/riscv/riscv-fusion.cc              | 68 +++++++++++++++++++
 gcc/config/riscv/riscv-protos.h               |  1 +
 gcc/config/riscv/riscv.cc                     | 10 +++
 .../gcc.target/riscv/sched-fusion-priority.c  | 49 +++++++++++++
 4 files changed, 128 insertions(+)
 create mode 100644 gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c

diff --git a/gcc/config/riscv/riscv-fusion.cc b/gcc/config/riscv/riscv-fusion.cc
index 332f64bd7cc..06cdab5694d 100644
--- a/gcc/config/riscv/riscv-fusion.cc
+++ b/gcc/config/riscv/riscv-fusion.cc
@@ -907,6 +907,74 @@ riscv_fuse_ldst_pair_p (rtx_insn *prev, rtx_insn *curr,
   return diff == access_size;
 }
 
+/* Implement TARGET_SCHED_FUSION_PRIORITY.  Group load/store pair candidates
+   by register file, access kind, mode and base register, then by offset.  */
+
+void
+riscv_sched_fusion_priority (rtx_insn *insn, int max_pri,
+			     int *fusion_pri, int *pri)
+{
+  struct riscv_fusion_mem_info mem;
+  unsigned HOST_WIDE_INT fusible_ops;
+  enum riscv_fusion_pairs inc_op, dec_op;
+  HOST_WIDE_INT access_size;
+  unsigned int base_regno;
+  bool isload_p, fp_p, inc_p;
+  int fusion_type, tmp;
+
+  gcc_assert (INSN_P (insn));
+
+  tmp = max_pri - 1;
+  *fusion_pri = tmp;
+  *pri = tmp;
+
+  if (!riscv_fuse_mem_p (insn, &mem)
+      || mem.type == SCHED_FUSION_LD_ZERO_EXTEND
+      || mem.addr.type != ADDRESS_REG
+      || !CONST_INT_P (mem.addr.offset))
+    return;
+
+  base_regno = riscv_regno (mem.addr.reg);
+  if (base_regno >= FIRST_PSEUDO_REGISTER)
+    return;
+
+  isload_p = mem.type != SCHED_FUSION_ST;
+  fp_p = mem.fp_p;
+  access_size = GET_MODE_SIZE (mem.mode).to_constant ();
+  if (access_size != 4 && access_size != 8)
+    return;
+
+  inc_op = fp_p ? RISCV_FUSE_FLDFST_PAIR_INC : RISCV_FUSE_LDST_PAIR_INC;
+  dec_op = fp_p ? RISCV_FUSE_FLDFST_PAIR_DEC : RISCV_FUSE_LDST_PAIR_DEC;
+  fusible_ops = riscv_get_fusible_ops ();
+  if (!(fusible_ops & (inc_op | dec_op)))
+    return;
+
+  /* Prefer decreasing offsets to match frame save/restore order.  */
+  inc_p = !(fusible_ops & dec_op);
+
+  /* Give each load/store class and base register a distinct priority below
+     that of unrelated instructions.  */
+  fusion_type = (fp_p ? 4 : 0) + (isload_p ? 0 : 2);
+  fusion_type += access_size == 8;
+  fusion_type++;
+  *fusion_pri -= (fusion_type * FIRST_PSEUDO_REGISTER
+		  + (int) base_regno);
+
+  tmp /= 2;
+  HOST_WIDE_INT off_val = INTVAL (mem.addr.offset);
+  unsigned HOST_WIDE_INT magnitude = off_val < 0
+				       ? -(unsigned HOST_WIDE_INT) off_val
+				       : off_val;
+  int offset_pri = magnitude & 0xfffff;
+
+  /* Order offsets in the preferred pair direction.  */
+  if (inc_p == (off_val >= 0))
+    *pri = tmp - offset_pri;
+  else
+    *pri = tmp + offset_pri;
+}
+
 /* Check the common RTL for ZEXTW, ZEXTWS and ZEXTH fusion.  */
 
 static bool
diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
index cf36efccf75..0921b3a7618 100644
--- a/gcc/config/riscv/riscv-protos.h
+++ b/gcc/config/riscv/riscv-protos.h
@@ -884,6 +884,7 @@ enum riscv_fusion_pairs
 
 extern bool riscv_macro_fusion_p (void);
 extern bool riscv_macro_fusion_pair_p (rtx_insn *, rtx_insn *);
+extern void riscv_sched_fusion_priority (rtx_insn *, int, int *, int *);
 extern unsigned HOST_WIDE_INT riscv_get_fusible_ops (void);
 
 /* Routines implemented in thead.cc.  */
diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc
index a2a51c019ec..9f0c1d704c2 100644
--- a/gcc/config/riscv/riscv.cc
+++ b/gcc/config/riscv/riscv.cc
@@ -12121,6 +12121,14 @@ riscv_override_options_internal (struct gcc_options *opts)
 		 ? &optimize_size_tune_info
 		 : cpu->tune_param;
 
+  /* Enable scheduling fusion for load/store-pair tunes.  */
+  if (!OPTION_SET_P (flag_schedule_fusion))
+    opts->x_flag_schedule_fusion
+      = (riscv_get_fusible_ops ()
+	 & (RISCV_FUSE_LDST_PAIR_INC | RISCV_FUSE_LDST_PAIR_DEC
+	    | RISCV_FUSE_FLDFST_PAIR_INC | RISCV_FUSE_FLDFST_PAIR_DEC))
+	!= RISCV_FUSE_NOTHING;
+
   /* If not optimizing for size, set the default
       alignment to what the target wants.  */
   if (!opts->x_optimize_size)
@@ -16617,6 +16625,8 @@ riscv_memtag_tag_bitsize ()
 #define TARGET_SCHED_MACRO_FUSION_P riscv_macro_fusion_p
 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
 #define TARGET_SCHED_MACRO_FUSION_PAIR_P riscv_macro_fusion_pair_p
+#undef TARGET_SCHED_FUSION_PRIORITY
+#define TARGET_SCHED_FUSION_PRIORITY riscv_sched_fusion_priority
 
 #undef TARGET_SCHED_INIT
 #define TARGET_SCHED_INIT riscv_sched_init
diff --git a/gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c b/gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c
new file mode 100644
index 00000000000..3d2c100a408
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sched-fusion-priority.c
@@ -0,0 +1,49 @@
+/* { dg-do compile { target { rv64 } } } */
+/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-O3" "-O[sgz]" "-flto" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -mtune=xt-c9501fdvt -O2" } */
+/* { dg-additional-options "-fno-schedule-insns2" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+/* Start before sched_fusion and disable sched2 to isolate the priority
+   hook.  */
+
+/*
+**sched_fusion_load_pair: { xfail *-*-* }
+**	...
+**	ld	[a-z][0-9]+,8\(a0\)
+**	ld	[a-z][0-9]+,0\(a0\)
+**	...
+**	ret
+*/
+long __RTL (startwith ("compgotos"))
+sched_fusion_load_pair (void)
+{
+(function "sched_fusion_load_pair"
+  (insn-chain
+    (block 2
+      (edge-from entry (flags "FALLTHRU"))
+      (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+      (cnote 2 NOTE_INSN_FUNCTION_BEG)
+      (cinsn 3 (set (reg:DI a1)
+                    (mem:DI (reg:DI a0) [0 S8 A64])))
+      (cinsn 4 (set (reg:DI a3)
+                    (plus:DI (reg:DI a4) (reg:DI a5))))
+      (cinsn 5 (set (reg:DI a2)
+                    (mem:DI
+                      (plus:DI (reg:DI a0)
+                               (const_int 8)) [0 S8 A64])))
+      (cinsn 6 (use (reg:DI a1)))
+      (cinsn 7 (use (reg:DI a2)))
+      (cinsn 8 (use (reg:DI a3)))
+      (cjump_insn 9 (simple_return))
+      (edge-to exit)
+    ) ;; block 2
+    (cbarrier 10)
+  ) ;; insn-chain
+  (crtl
+    (return_rtx
+      (reg/i:DI a0)
+    ) ;; return_rtx
+  ) ;; crtl
+) ;; function "sched_fusion_load_pair"
+}
-- 
2.52.0