[PATCH v5 15/18] RISC-V: Add ascending integer load/store pair fusion

Recognize adjacent 4-byte or 8-byte integer loads or stores whose
addresses share a base and increase by one access size.  Factor common
pair parsing into a helper reused by ALIGNED_STD, and parameterize the
direction and register-class differences for the remaining pair rules.

Leave the new fusion disabled by default and XFAIL its positive dump
checks until a CPU enables it.

gcc/ChangeLog:

	* config/riscv/riscv-fusion.cc (riscv_fuse_mem_pair_p): New
	function.
	(riscv_fuse_ldst_pair_p): Likewise.
	(riscv_fuse_aligned_std): Use riscv_fuse_mem_pair_p.
	(riscv_fuse_ldst_pair_inc): New function.
	(riscv_fusion_table): Add RISCV_FUSE_LDST_PAIR_INC.
	* config/riscv/riscv-protos.h (enum riscv_fusion_pairs): Add
	RISCV_FUSE_LDST_PAIR_INC.

gcc/testsuite/ChangeLog:

	* gcc.target/riscv/fusion-ldst-pair-inc.c: New test.
	* gcc.target/riscv/fusion-memory-rtl-shapes-ldst-pair.c: Likewise.

Signed-off-by: Jin Ma <[email protected]>
---
 gcc/config/riscv/riscv-fusion.cc              | 109 +++++++++++++++---
 gcc/config/riscv/riscv-protos.h               |   1 +
 .../gcc.target/riscv/fusion-ldst-pair-inc.c   |  33 ++++++
 .../fusion-memory-rtl-shapes-ldst-pair.c      |  33 ++++++
 4 files changed, 162 insertions(+), 14 deletions(-)
 create mode 100644 gcc/testsuite/gcc.target/riscv/fusion-ldst-pair-inc.c
 create mode 100644 gcc/testsuite/gcc.target/riscv/fusion-memory-rtl-shapes-ldst-pair.c

diff --git a/gcc/config/riscv/riscv-fusion.cc b/gcc/config/riscv/riscv-fusion.cc
index 530c24482b2..419628a5038 100644
--- a/gcc/config/riscv/riscv-fusion.cc
+++ b/gcc/config/riscv/riscv-fusion.cc
@@ -839,6 +839,74 @@ riscv_fuse_indexed_mem_p (rtx_insn *prev, rtx_insn *curr,
   return !riscv_fuse_same_reg_p (SET_SRC (mem_set), update_dest);
 }
 
+/* Extract a pair of scalar loads or stores with the same mode and base
+   register.  Store whether they are loads in *LOAD_P.  */
+
+static bool
+riscv_fuse_mem_pair_p (rtx_insn *prev, rtx_insn *curr,
+			 struct riscv_fusion_mem_info *prev_mem,
+			 struct riscv_fusion_mem_info *curr_mem,
+			 bool *load_p)
+{
+  if (!riscv_fuse_sets_p (prev, curr)
+      || !riscv_fuse_mem_p (prev, prev_mem)
+      || !riscv_fuse_mem_p (curr, curr_mem))
+    return false;
+
+  bool prev_load_p = prev_mem->type != SCHED_FUSION_ST;
+  bool curr_load_p = curr_mem->type != SCHED_FUSION_ST;
+  if (prev_load_p != curr_load_p
+      || prev_mem->fp_p != curr_mem->fp_p
+      || prev_mem->mode != curr_mem->mode
+      || prev_mem->addr.type != ADDRESS_REG
+      || curr_mem->addr.type != ADDRESS_REG
+      || !CONST_INT_P (prev_mem->addr.offset)
+      || !CONST_INT_P (curr_mem->addr.offset)
+      || !riscv_fuse_same_reg_p (prev_mem->addr.reg,
+				   curr_mem->addr.reg))
+    return false;
+
+  *load_p = prev_load_p;
+  return true;
+}
+
+/* Check common adjacent load/store-pair constraints.  INC_P selects ascending
+   offsets and FP_P selects floating-point rather than integer accesses.  */
+
+static bool
+riscv_fuse_ldst_pair_p (rtx_insn *prev, rtx_insn *curr,
+			  bool inc_p, bool fp_p)
+{
+  struct riscv_fusion_mem_info prev_mem, curr_mem;
+  bool load_p;
+  if (!riscv_fuse_mem_pair_p (prev, curr, &prev_mem, &curr_mem,
+				&load_p)
+      || prev_mem.fp_p != fp_p
+      || prev_mem.type == SCHED_FUSION_LD_ZERO_EXTEND
+      || curr_mem.type == SCHED_FUSION_LD_ZERO_EXTEND)
+    return false;
+
+  HOST_WIDE_INT access_size = GET_MODE_SIZE (prev_mem.mode).to_constant ();
+  if (access_size != 4 && access_size != 8)
+    return false;
+
+  if (load_p)
+    {
+      rtx prev_dest = SET_DEST (single_set (prev));
+      rtx curr_dest = SET_DEST (single_set (curr));
+      if (riscv_fuse_same_reg_p (prev_dest, curr_dest)
+	  || riscv_fuse_same_reg_p (prev_mem.addr.reg, prev_dest))
+	return false;
+    }
+
+  HOST_WIDE_INT diff = inc_p
+		       ? INTVAL (curr_mem.addr.offset)
+			 - INTVAL (prev_mem.addr.offset)
+		       : INTVAL (prev_mem.addr.offset)
+			 - INTVAL (curr_mem.addr.offset);
+  return diff == access_size;
+}
+
 /* Check the common RTL for ZEXTW, ZEXTWS and ZEXTH fusion.  */
 
 static bool
@@ -1422,22 +1490,13 @@ riscv_fuse_auipc_ld (rtx_insn *prev, rtx_insn *curr)
 static bool
 riscv_fuse_aligned_std (rtx_insn *prev, rtx_insn *curr)
 {
-  if (!riscv_fuse_sets_p (prev, curr))
-    return false;
-
   struct riscv_fusion_mem_info prev_mem, curr_mem;
-
-  if (!riscv_fuse_mem_p (prev, &prev_mem)
-      || !riscv_fuse_mem_p (curr, &curr_mem)
-      || prev_mem.type != SCHED_FUSION_ST
-      || curr_mem.type != SCHED_FUSION_ST
+  bool load_p;
+  if (!riscv_fuse_mem_pair_p (prev, curr, &prev_mem, &curr_mem,
+				&load_p)
+      || load_p
       || prev_mem.fp_p
-      || curr_mem.fp_p
-      || !SCALAR_INT_MODE_P (prev_mem.mode)
-      || prev_mem.mode != curr_mem.mode
-      || prev_mem.addr.type != ADDRESS_REG
-      || curr_mem.addr.type != ADDRESS_REG
-      || !riscv_fuse_same_reg_p (prev_mem.addr.reg, curr_mem.addr.reg))
+      || !SCALAR_INT_MODE_P (prev_mem.mode))
     return false;
 
   unsigned int mode_size
@@ -1454,6 +1513,26 @@ riscv_fuse_aligned_std (rtx_insn *prev, rtx_insn *curr)
   return false;
 }
 
+/* Check for RISCV_FUSE_LDST_PAIR_INC fusion.
+   prev/curr (one of the following pairs):
+     prev (lw/ld) == (set (reg rd1) (mem (rs1, offset1)))
+     curr (lw/ld) == (set (reg rd2) (mem (rs1, offset2)))
+
+     prev (sw/sd) == (set (mem (rs1, offset1)) (reg rs2))
+     curr (sw/sd) == (set (mem (rs1, offset2)) (reg rs3))
+
+   Constraints:
+     access size is 4 or 8 bytes
+     offset2 - offset1 equals the access size
+     loads are not zero-extending
+     for loads, rd1 != rd2 and rd1 != rs1.  */
+
+static bool
+riscv_fuse_ldst_pair_inc (rtx_insn *prev, rtx_insn *curr)
+{
+  return riscv_fuse_ldst_pair_p (prev, curr, true, false);
+}
+
 /* Check for RISCV_FUSE_BFEXT fusion.
    prev (slli) == (set (reg rd1)
 		       (ashift (reg rs1) (const_int shamt1)))
@@ -1797,6 +1876,8 @@ static const struct riscv_fusion_entry riscv_fusion_table[] =
     riscv_fuse_auipc_ld, "RISCV_FUSE_AUIPC_LD" },
   { RISCV_FUSE_ALIGNED_STD,
     riscv_fuse_aligned_std, "RISCV_FUSE_ALIGNED_STD" },
+  { RISCV_FUSE_LDST_PAIR_INC,
+    riscv_fuse_ldst_pair_inc, "RISCV_FUSE_LDST_PAIR_INC" },
   { RISCV_FUSE_BFEXT,
     riscv_fuse_bfext, "RISCV_FUSE_BFEXT" },
   { RISCV_FUSE_SLLI_SRLI,
diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
index 445d1f7f257..de4476a2d66 100644
--- a/gcc/config/riscv/riscv-protos.h
+++ b/gcc/config/riscv/riscv-protos.h
@@ -876,6 +876,7 @@ enum riscv_fusion_pairs
   RISCV_FUSE_PREINDEX_ST = HOST_WIDE_INT_1U << 20,
   RISCV_FUSE_POSTINDEX_LD = HOST_WIDE_INT_1U << 21,
   RISCV_FUSE_POSTINDEX_ST = HOST_WIDE_INT_1U << 22,
+  RISCV_FUSE_LDST_PAIR_INC = HOST_WIDE_INT_1U << 23,
 };
 
 extern bool riscv_macro_fusion_p (void);
diff --git a/gcc/testsuite/gcc.target/riscv/fusion-ldst-pair-inc.c b/gcc/testsuite/gcc.target/riscv/fusion-ldst-pair-inc.c
new file mode 100644
index 00000000000..44ad262bf17
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/fusion-ldst-pair-inc.c
@@ -0,0 +1,33 @@
+/* { dg-do compile { target { rv64 } } } */
+/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-O3" "-O[sgz]" "-flto" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -mtune=xt-c9501fdvt -O2 -fdump-rtl-sched2-details" } */
+/* { dg-final { scan-rtl-dump-times "RISCV_FUSE_LDST_PAIR_INC" 4 "sched2" { xfail *-*-* } } } */
+
+typedef int int32_t;
+typedef long int64_t;
+
+int64_t
+test_ld_pair_inc (int64_t *p)
+{
+  return p[0] + p[1];
+}
+
+int64_t
+test_lw_pair_inc (int32_t *p)
+{
+  return (int64_t) p[0] + p[1];
+}
+
+void
+test_sd_pair_inc (int64_t *p, int64_t a, int64_t b)
+{
+  p[0] = a;
+  p[1] = b;
+}
+
+void
+test_sw_pair_inc (int32_t *p, int32_t a, int32_t b)
+{
+  p[0] = a;
+  p[1] = b;
+}
diff --git a/gcc/testsuite/gcc.target/riscv/fusion-memory-rtl-shapes-ldst-pair.c b/gcc/testsuite/gcc.target/riscv/fusion-memory-rtl-shapes-ldst-pair.c
new file mode 100644
index 00000000000..2a19c6f3d97
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/fusion-memory-rtl-shapes-ldst-pair.c
@@ -0,0 +1,33 @@
+/* { dg-do compile { target { rv64 } } } */
+/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-O3" "-O[sgz]" "-flto" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -O2 -mtune=xt-c9501fdvt -fdump-rtl-sched2-details" } */
+/* { dg-final { scan-rtl-dump-times "RISCV_FUSE_LDST_PAIR_INC" 1 "sched2" { xfail *-*-* } } } */
+
+/* The second load destination may be the shared base register.  */
+long __RTL (startwith ("sched2"))
+test_load_pair_inc_second_destination_is_base (void)
+{
+(function "test_load_pair_inc_second_destination_is_base"
+  (insn-chain
+    (block 2
+      (edge-from entry (flags "FALLTHRU"))
+      (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+      (cnote 2 NOTE_INSN_FUNCTION_BEG)
+      (cinsn 3 (set (reg:DI a1)
+                    (mem:DI (reg:DI a0) [0 S8 A64])))
+      (cinsn 4 (set (reg:DI a0)
+                    (mem:DI
+                      (plus:DI (reg:DI a0)
+                               (const_int 8)) [0 S8 A64])))
+      (cinsn 5 (use (reg:DI a1)))
+      (cinsn 6 (use (reg:DI a0)))
+      (cjump_insn 7 (simple_return))
+      (edge-to exit)
+    ) ;; block 2
+    (cbarrier 8)
+  ) ;; insn-chain
+  (crtl
+    (return_rtx (reg/i:DI a0))
+  ) ;; crtl
+) ;; function "test_load_pair_inc_second_destination_is_base"
+}
-- 
2.52.0