[PATCH 20/23] target/riscv: Use FLATTEN rather than ALWAYS_INLINE for vector ldst

It turns out there are fewer places that need annotation if we
just flatten from the outer helper macros instead of everything
along the call stack.

Signed-off-by: Richard Henderson <[email protected]>
---
 target/riscv/tcg/vector_helper.c | 143 +++++++++++++++----------------
 1 file changed, 67 insertions(+), 76 deletions(-)

diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 473b112426..2a24e15a31 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -158,7 +158,7 @@ static inline void vext_set_elem_mask(void *v0, int index,
     ((uint64_t *)v0)[idx] = deposit64(old, pos, 1, value);
 }
 
-static inline MemOpIdx vext_oi(uint32_t desc, MemOp memop)
+static MemOpIdx vext_oi(uint32_t desc, MemOp memop)
 {
     uint32_t data = simd_data(desc);
 
@@ -179,17 +179,15 @@ typedef void vext_ldst_elem_fn_host(void *vd, void *host,
                                     uint32_t idx, uint32_t evl);
 
 #define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF)                     \
-static inline QEMU_ALWAYS_INLINE                                        \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,         \
-                void *vd, MemOpIdx oi, uintptr_t retaddr)               \
+static void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,  \
+                       void *vd, MemOpIdx oi, uintptr_t retaddr)        \
 {                                                                       \
     ETYPE *cur = ((ETYPE *)vd + H(idx));                                \
     *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr);                   \
 }
 
 #define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF)                    \
-static inline QEMU_ALWAYS_INLINE                                        \
-void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl)      \
+static void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
 {                                                                       \
     do {                                                                \
         ETYPE *cur = (ETYPE *)vd + H(idx);                              \
@@ -206,8 +204,7 @@ GEN_VEXT_TLB_LD_ELEM(lde_d, uint64_t, H8, ldq)
 #if HOST_BIG_ENDIAN
 GEN_VEXT_HOST_LD_ELEM(lde_b, uint8_t,  H1, ldub)
 #else
-static inline QEMU_ALWAYS_INLINE
-void lde_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
+static void lde_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
 {
     memcpy(vd + idx, host, evl - idx);
 }
@@ -218,8 +215,7 @@ GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le)
 GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le)
 
 #define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF)                     \
-static inline QEMU_ALWAYS_INLINE                                        \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,         \
+static void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,  \
                 void *vd, MemOpIdx oi, uintptr_t retaddr)               \
 {                                                                       \
     ETYPE data = *((ETYPE *)vd + H(idx));                               \
@@ -227,8 +223,7 @@ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,         \
 }
 
 #define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF)                    \
-static inline QEMU_ALWAYS_INLINE                                        \
-void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl)      \
+static void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \
 {                                                                       \
     do {                                                                \
         ETYPE data = *((ETYPE *)vd + H(idx));                           \
@@ -245,8 +240,7 @@ GEN_VEXT_TLB_ST_ELEM(ste_d, uint64_t, H8, stq)
 #if HOST_BIG_ENDIAN
 GEN_VEXT_HOST_ST_ELEM(ste_b, uint8_t,  H1, stb)
 #else
-static inline QEMU_ALWAYS_INLINE
-void ste_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
+static void ste_b_host(void *vd, void *host, uint32_t idx, uint32_t evl)
 {
     memcpy(host, vd + idx, evl - idx);
 }
@@ -361,7 +355,7 @@ GEN_VEXT_ST_STRIDE(vsse64_v, int64_t, ste_d_tlb)
  */
 
 /* unmasked unit-stride load and store operation */
-static inline QEMU_ALWAYS_INLINE void
+static void
 vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl,
                        uint32_t nf, uint32_t log2_esz, uint32_t max_elems,
                        vext_ldst_elem_fn_host *ldst_host)
@@ -378,7 +372,7 @@ vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl,
     }
 }
 
-static inline QEMU_ALWAYS_INLINE void
+static void
 vext_page_ldst_us_tlb(CPURISCVState *env, void *vd, target_ulong addr,
                       uint32_t i, uint32_t evl, uint32_t nf,
                       uint32_t log2_esz, uint32_t max_elems,
@@ -409,7 +403,7 @@ static void vext_test_alignment(CPURISCVState *env, vaddr addr, uint32_t esz,
     }
 }
 
-static inline QEMU_ALWAYS_INLINE void
+static void
 vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
              uint32_t log2_esz, uint32_t nf, uint32_t evl,
              uint32_t max_elems, bool vma,
@@ -511,7 +505,7 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
     }
 }
 
-static inline QEMU_ALWAYS_INLINE void
+static void
 vext_ldst_us_desc(void *vd, target_ulong base, CPURISCVState *env,
                   uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
                   vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
@@ -528,20 +522,19 @@ vext_ldst_us_desc(void *vd, target_ulong base, CPURISCVState *env,
  * stride, stride = NF * sizeof (ETYPE)
  */
 
-#define GEN_VEXT_LD_US(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST)      \
-void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base,     \
-                         CPURISCVState *env, uint32_t desc)         \
-{                                                                   \
-    uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE));         \
-    vext_ldst_stride(vd, v0, base, stride, env, desc, false,        \
-                     LOAD_FN_TLB, ctzl(sizeof(ETYPE)), GETPC());    \
-}                                                                   \
-                                                                    \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,            \
-                  CPURISCVState *env, uint32_t desc)                \
-{                                                                   \
-    vext_ldst_us_desc(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST,    \
-                      ctzl(sizeof(ETYPE)), env->vl, GETPC(), true);      \
+#define GEN_VEXT_LD_US(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST)          \
+QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, void *v0,               \
+    target_ulong base, CPURISCVState *env, uint32_t desc)               \
+{                                                                       \
+    uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE));             \
+    vext_ldst_stride(vd, v0, base, stride, env, desc, false,            \
+                     LOAD_FN_TLB, ctzl(sizeof(ETYPE)), GETPC());        \
+}                                                                       \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,   \
+                               CPURISCVState *env, uint32_t desc)       \
+{                                                                       \
+    vext_ldst_us_desc(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST,   \
+                      ctzl(sizeof(ETYPE)), env->vl, GETPC(), true);     \
 }
 
 GEN_VEXT_LD_US(vle8_v,  int8_t,  lde_b_tlb, lde_b_host)
@@ -549,20 +542,19 @@ GEN_VEXT_LD_US(vle16_v, int16_t, lde_h_tlb, lde_h_host)
 GEN_VEXT_LD_US(vle32_v, int32_t, lde_w_tlb, lde_w_host)
 GEN_VEXT_LD_US(vle64_v, int64_t, lde_d_tlb, lde_d_host)
 
-#define GEN_VEXT_ST_US(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST)         \
-void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base,          \
-                         CPURISCVState *env, uint32_t desc)              \
-{                                                                        \
-    uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE));              \
-    vext_ldst_stride(vd, v0, base, stride, env, desc, false,             \
-                     STORE_FN_TLB, ctzl(sizeof(ETYPE)), GETPC());        \
-}                                                                        \
-                                                                         \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,                 \
-                  CPURISCVState *env, uint32_t desc)                     \
-{                                                                        \
-    vext_ldst_us_desc(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST,  \
-                      ctzl(sizeof(ETYPE)), env->vl, GETPC(), false);     \
+#define GEN_VEXT_ST_US(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST)        \
+QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, void *v0,               \
+    target_ulong base, CPURISCVState *env, uint32_t desc)               \
+{                                                                       \
+    uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE));             \
+    vext_ldst_stride(vd, v0, base, stride, env, desc, false,            \
+                     STORE_FN_TLB, ctzl(sizeof(ETYPE)), GETPC());       \
+}                                                                       \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,   \
+                  CPURISCVState *env, uint32_t desc)                    \
+{                                                                       \
+    vext_ldst_us_desc(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \
+                      ctzl(sizeof(ETYPE)), env->vl, GETPC(), false);    \
 }
 
 GEN_VEXT_ST_US(vse8_v,  int8_t,  ste_b_tlb, ste_b_host)
@@ -573,8 +565,8 @@ GEN_VEXT_ST_US(vse64_v, int64_t, ste_d_tlb, ste_d_host)
 /*
  * unit stride mask load and store, EEW = 1
  */
-void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
-                    CPURISCVState *env, uint32_t desc)
+QEMU_FLATTEN void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
+                                CPURISCVState *env, uint32_t desc)
 {
     /* evl = ceil(vl/8) */
     uint8_t evl = (env->vl + 7) >> 3;
@@ -582,8 +574,8 @@ void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
                       0, evl, GETPC(), true);
 }
 
-void HELPER(vsm_v)(void *vd, void *v0, target_ulong base,
-                    CPURISCVState *env, uint32_t desc)
+QEMU_FLATTEN void HELPER(vsm_v)(void *vd, void *v0, target_ulong base,
+                                CPURISCVState *env, uint32_t desc)
 {
     /* evl = ceil(vl/8) */
     uint8_t evl = (env->vl + 7) >> 3;
@@ -609,7 +601,7 @@ GEN_VEXT_GET_INDEX_ADDR(idx_h, uint16_t, H2)
 GEN_VEXT_GET_INDEX_ADDR(idx_w, uint32_t, H4)
 GEN_VEXT_GET_INDEX_ADDR(idx_d, uint64_t, H8)
 
-static inline void
+static void
 vext_ldst_index(void *vd, void *v0, target_ulong base,
                 void *vs2, CPURISCVState *env, uint32_t desc,
                 vext_get_index_addr get_index_addr,
@@ -646,12 +638,12 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
     }
 }
 
-#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN)                  \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,                   \
-                  void *vs2, CPURISCVState *env, uint32_t desc)            \
-{                                                                          \
-    vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN,                \
-                    LOAD_FN, ctzl(sizeof(ETYPE)), GETPC());                \
+#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN)               \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,   \
+                               void *vs2, CPURISCVState *env, uint32_t desc) \
+{                                                                       \
+    vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN,             \
+                    LOAD_FN, ctzl(sizeof(ETYPE)), GETPC());             \
 }
 
 GEN_VEXT_LD_INDEX(vlxei8_8_v,   int8_t,  idx_b, lde_b_tlb)
@@ -671,13 +663,12 @@ GEN_VEXT_LD_INDEX(vlxei64_16_v, int16_t, idx_d, lde_h_tlb)
 GEN_VEXT_LD_INDEX(vlxei64_32_v, int32_t, idx_d, lde_w_tlb)
 GEN_VEXT_LD_INDEX(vlxei64_64_v, int64_t, idx_d, lde_d_tlb)
 
-#define GEN_VEXT_ST_INDEX(NAME, ETYPE, INDEX_FN, STORE_FN)       \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,         \
-                  void *vs2, CPURISCVState *env, uint32_t desc)  \
-{                                                                \
-    vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN,      \
-                    STORE_FN, ctzl(sizeof(ETYPE)),               \
-                    GETPC());                                    \
+#define GEN_VEXT_ST_INDEX(NAME, ETYPE, INDEX_FN, STORE_FN)              \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,   \
+                               void *vs2, CPURISCVState *env, uint32_t desc) \
+{                                                                       \
+    vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN,             \
+                    STORE_FN, ctzl(sizeof(ETYPE)), GETPC());            \
 }
 
 GEN_VEXT_ST_INDEX(vsxei8_8_v,   int8_t,  idx_b, ste_b_tlb)
@@ -700,7 +691,7 @@ GEN_VEXT_ST_INDEX(vsxei64_64_v, int64_t, idx_d, ste_d_tlb)
 /*
  * unit-stride fault-only-fisrt load instructions
  */
-static inline void
+static void
 vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
           uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
           vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra)
@@ -933,12 +924,12 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
     }
 }
 
-#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST)   \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,        \
-                  CPURISCVState *env, uint32_t desc)            \
-{                                                               \
-    vext_ldff(vd, v0, base, env, desc, LOAD_FN_TLB,             \
-              LOAD_FN_HOST, ctzl(sizeof(ETYPE)), GETPC());      \
+#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST)           \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,   \
+                               CPURISCVState *env, uint32_t desc)       \
+{                                                                       \
+    vext_ldff(vd, v0, base, env, desc, LOAD_FN_TLB,                     \
+              LOAD_FN_HOST, ctzl(sizeof(ETYPE)), GETPC());              \
 }
 
 GEN_VEXT_LDFF(vle8ff_v,  int8_t,  lde_b_tlb, lde_b_host)
@@ -959,7 +950,7 @@ GEN_VEXT_LDFF(vle64ff_v, int64_t, lde_d_tlb, lde_d_host)
 /*
  * load and store whole register instructions
  */
-static inline QEMU_ALWAYS_INLINE void
+static void
 vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
                 vext_ldst_elem_fn_tlb *ldst_tlb,
                 vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
@@ -973,8 +964,8 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
 }
 
 #define GEN_VEXT_LD_WHOLE(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST)   \
-void HELPER(NAME)(void *vd, target_ulong base, CPURISCVState *env,  \
-                  uint32_t desc)                                    \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base,         \
+                               CPURISCVState *env, uint32_t desc)   \
 {                                                                   \
     vext_ldst_whole(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \
                     ctzl(sizeof(ETYPE)), GETPC(), true);            \
@@ -998,8 +989,8 @@ GEN_VEXT_LD_WHOLE(vl8re32_v, int32_t, lde_w_tlb, lde_w_host)
 GEN_VEXT_LD_WHOLE(vl8re64_v, int64_t, lde_d_tlb, lde_d_host)
 
 #define GEN_VEXT_ST_WHOLE(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST)     \
-void HELPER(NAME)(void *vd, target_ulong base, CPURISCVState *env,      \
-                  uint32_t desc)                                        \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base,             \
+                               CPURISCVState *env, uint32_t desc)       \
 {                                                                       \
     vext_ldst_whole(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST,   \
                     ctzl(sizeof(ETYPE)), GETPC(), false);               \
-- 
2.43.0