[PATCH v2 18/23] target/riscv: Use FLATTEN rather than ALWAYS_INLINE for vector ldst

It turns out there are fewer places that need annotation if we
just flatten from the outer helper macros instead of everything
along the call stack.

Signed-off-by: Richard Henderson <[email protected]>
---
 target/riscv/tcg/vector_helper.c | 147 +++++++++++++++----------------
 1 file changed, 70 insertions(+), 77 deletions(-)

diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index f65da46144..c5b4085f9c 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -158,7 +158,7 @@ static inline void vext_set_elem_mask(void *v0, int index,
     ((uint64_t *)v0)[idx] = deposit64(old, pos, 1, value);
 }
 
-static inline MemOpIdx vext_oi(uint32_t desc, MemOp memop)
+static MemOpIdx vext_oi(uint32_t desc, MemOp memop)
 {
     uint32_t data = simd_data(desc);
 
@@ -178,20 +178,18 @@ typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr,
 typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void *host);
 
 #define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF)                     \
-static inline QEMU_ALWAYS_INLINE                                        \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,         \
-                void *vd, MemOpIdx oi, uintptr_t retaddr)               \
+static void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,  \
+                       void *vd, MemOpIdx oi, uintptr_t retaddr)        \
 {                                                                       \
     ETYPE *cur = ((ETYPE *)vd + H(idx));                                \
     *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr);                   \
 }
 
-#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF)        \
-static inline QEMU_ALWAYS_INLINE                            \
-void NAME##_host(void *vd, uint32_t idx, void *host)        \
-{                                                           \
-    ETYPE *cur = ((ETYPE *)vd + H(idx));                    \
-    *cur = (ETYPE)LDSUF##_p(host);                          \
+#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF)                    \
+static void NAME##_host(void *vd, uint32_t idx, void *host)             \
+{                                                                       \
+    ETYPE *cur = ((ETYPE *)vd + H(idx));                                \
+    *cur = (ETYPE)LDSUF##_p(host);                                      \
 }
 
 GEN_VEXT_TLB_LD_ELEM(lde_b, uint8_t,  H1, ldb)
@@ -205,20 +203,18 @@ GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le)
 GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le)
 
 #define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF)                     \
-static inline QEMU_ALWAYS_INLINE                                        \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,         \
+static void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,  \
                 void *vd, MemOpIdx oi, uintptr_t retaddr)               \
 {                                                                       \
     ETYPE data = *((ETYPE *)vd + H(idx));                               \
     cpu_##STSUF##_mmu(env, addr, data, oi, retaddr);                    \
 }
 
-#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF)        \
-static inline QEMU_ALWAYS_INLINE                            \
-void NAME##_host(void *vd, uint32_t idx, void *host)        \
-{                                                           \
-    ETYPE data = *((ETYPE *)vd + H(idx));                   \
-    STSUF##_p(host, data);                                  \
+#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF)                    \
+static void NAME##_host(void *vd, uint32_t idx, void *host)             \
+{                                                                       \
+    ETYPE data = *((ETYPE *)vd + H(idx));                               \
+    STSUF##_p(host, data);                                              \
 }
 
 GEN_VEXT_TLB_ST_ELEM(ste_b, uint8_t,  H1, stb)
@@ -308,9 +304,9 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride,
 }
 
 #define GEN_VEXT_LD_STRIDE(NAME, ETYPE, LOAD_FN)                        \
-void HELPER(NAME)(void *vd, void * v0, target_ulong base,               \
-                  target_ulong stride, CPURISCVState *env,              \
-                  uint32_t desc)                                        \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void * v0, target_ulong base,  \
+                               target_ulong stride, CPURISCVState *env, \
+                               uint32_t desc)                           \
 {                                                                       \
     uint32_t vm = vext_vm(desc);                                        \
     vext_ldst_stride(vd, v0, base, stride, env, desc, vm, LOAD_FN,      \
@@ -323,9 +319,9 @@ GEN_VEXT_LD_STRIDE(vlse32_v, int32_t, lde_w_tlb)
 GEN_VEXT_LD_STRIDE(vlse64_v, int64_t, lde_d_tlb)
 
 #define GEN_VEXT_ST_STRIDE(NAME, ETYPE, STORE_FN)                       \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,                \
-                  target_ulong stride, CPURISCVState *env,              \
-                  uint32_t desc)                                        \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,   \
+                               target_ulong stride, CPURISCVState *env, \
+                               uint32_t desc)                           \
 {                                                                       \
     uint32_t vm = vext_vm(desc);                                        \
     vext_ldst_stride(vd, v0, base, stride, env, desc, vm, STORE_FN,     \
@@ -342,7 +338,7 @@ GEN_VEXT_ST_STRIDE(vsse64_v, int64_t, ste_d_tlb)
  */
 
 /* unmasked unit-stride load and store operation */
-static inline QEMU_ALWAYS_INLINE void
+static void
 vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl,
                        uint32_t nf, uint32_t log2_esz, uint32_t max_elems,
                        vext_ldst_elem_fn_host *ldst_host, bool is_load)
@@ -372,7 +368,7 @@ vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl,
     }
 }
 
-static inline QEMU_ALWAYS_INLINE void
+static void
 vext_page_ldst_us_tlb(CPURISCVState *env, void *vd, target_ulong addr,
                       uint32_t i, uint32_t evl, uint32_t nf,
                       uint32_t log2_esz, uint32_t max_elems,
@@ -403,7 +399,7 @@ static void vext_test_alignment(CPURISCVState *env, vaddr addr, uint32_t esz,
     }
 }
 
-static inline QEMU_ALWAYS_INLINE void
+static void
 vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env,
                     uint32_t log2_esz, uint32_t nf, uint32_t evl,
                     uint32_t max_elems, bool vma,
@@ -499,7 +495,7 @@ vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env,
     }
 }
 
-static inline QEMU_ALWAYS_INLINE void
+static void
 vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
              vext_ldst_elem_fn_tlb *ldst_tlb,
              vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
@@ -521,20 +517,19 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
  * stride, stride = NF * sizeof (ETYPE)
  */
 
-#define GEN_VEXT_LD_US(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST)      \
-void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base,     \
-                         CPURISCVState *env, uint32_t desc)         \
-{                                                                   \
-    uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE));         \
-    vext_ldst_stride(vd, v0, base, stride, env, desc, false,        \
-                     LOAD_FN_TLB, ctzl(sizeof(ETYPE)), GETPC());    \
-}                                                                   \
-                                                                    \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,            \
-                  CPURISCVState *env, uint32_t desc)                \
-{                                                                   \
-    vext_ldst_us(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST,    \
-                 ctzl(sizeof(ETYPE)), env->vl, GETPC(), true);      \
+#define GEN_VEXT_LD_US(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST)          \
+QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, void *v0,               \
+    target_ulong base, CPURISCVState *env, uint32_t desc)               \
+{                                                                       \
+    uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE));             \
+    vext_ldst_stride(vd, v0, base, stride, env, desc, false,            \
+                     LOAD_FN_TLB, ctzl(sizeof(ETYPE)), GETPC());        \
+}                                                                       \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,   \
+                               CPURISCVState *env, uint32_t desc)       \
+{                                                                       \
+    vext_ldst_us(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST,        \
+                 ctzl(sizeof(ETYPE)), env->vl, GETPC(), true);          \
 }
 
 GEN_VEXT_LD_US(vle8_v,  int8_t,  lde_b_tlb, lde_b_host)
@@ -543,16 +538,15 @@ GEN_VEXT_LD_US(vle32_v, int32_t, lde_w_tlb, lde_w_host)
 GEN_VEXT_LD_US(vle64_v, int64_t, lde_d_tlb, lde_d_host)
 
 #define GEN_VEXT_ST_US(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST)         \
-void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base,          \
-                         CPURISCVState *env, uint32_t desc)              \
+QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, void *v0,                \
+    target_ulong base, CPURISCVState *env, uint32_t desc)                \
 {                                                                        \
     uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE));              \
     vext_ldst_stride(vd, v0, base, stride, env, desc, false,             \
                      STORE_FN_TLB, ctzl(sizeof(ETYPE)), GETPC());        \
 }                                                                        \
-                                                                         \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,                 \
-                  CPURISCVState *env, uint32_t desc)                     \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,    \
+                               CPURISCVState *env, uint32_t desc)        \
 {                                                                        \
     vext_ldst_us(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST,       \
                  ctzl(sizeof(ETYPE)), env->vl, GETPC(), false);          \
@@ -566,8 +560,8 @@ GEN_VEXT_ST_US(vse64_v, int64_t, ste_d_tlb, ste_d_host)
 /*
  * unit stride mask load and store, EEW = 1
  */
-void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
-                    CPURISCVState *env, uint32_t desc)
+QEMU_FLATTEN void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
+                                CPURISCVState *env, uint32_t desc)
 {
     /* evl = ceil(vl/8) */
     uint8_t evl = (env->vl + 7) >> 3;
@@ -575,8 +569,8 @@ void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
                  0, evl, GETPC(), true);
 }
 
-void HELPER(vsm_v)(void *vd, void *v0, target_ulong base,
-                    CPURISCVState *env, uint32_t desc)
+QEMU_FLATTEN void HELPER(vsm_v)(void *vd, void *v0, target_ulong base,
+                                CPURISCVState *env, uint32_t desc)
 {
     /* evl = ceil(vl/8) */
     uint8_t evl = (env->vl + 7) >> 3;
@@ -602,7 +596,7 @@ GEN_VEXT_GET_INDEX_ADDR(idx_h, uint16_t, H2)
 GEN_VEXT_GET_INDEX_ADDR(idx_w, uint32_t, H4)
 GEN_VEXT_GET_INDEX_ADDR(idx_d, uint64_t, H8)
 
-static inline void
+static void
 vext_ldst_index(void *vd, void *v0, target_ulong base,
                 void *vs2, CPURISCVState *env, uint32_t desc,
                 vext_get_index_addr get_index_addr,
@@ -637,12 +631,12 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
     vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
 }
 
-#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN)                  \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,                   \
-                  void *vs2, CPURISCVState *env, uint32_t desc)            \
-{                                                                          \
-    vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN,                \
-                    LOAD_FN, ctzl(sizeof(ETYPE)), GETPC());                \
+#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN)               \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,   \
+                               void *vs2, CPURISCVState *env, uint32_t desc) \
+{                                                                       \
+    vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN,             \
+                    LOAD_FN, ctzl(sizeof(ETYPE)), GETPC());             \
 }
 
 GEN_VEXT_LD_INDEX(vlxei8_8_v,   int8_t,  idx_b, lde_b_tlb)
@@ -662,13 +656,12 @@ GEN_VEXT_LD_INDEX(vlxei64_16_v, int16_t, idx_d, lde_h_tlb)
 GEN_VEXT_LD_INDEX(vlxei64_32_v, int32_t, idx_d, lde_w_tlb)
 GEN_VEXT_LD_INDEX(vlxei64_64_v, int64_t, idx_d, lde_d_tlb)
 
-#define GEN_VEXT_ST_INDEX(NAME, ETYPE, INDEX_FN, STORE_FN)       \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,         \
-                  void *vs2, CPURISCVState *env, uint32_t desc)  \
-{                                                                \
-    vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN,      \
-                    STORE_FN, ctzl(sizeof(ETYPE)),               \
-                    GETPC());                                    \
+#define GEN_VEXT_ST_INDEX(NAME, ETYPE, INDEX_FN, STORE_FN)              \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,   \
+                               void *vs2, CPURISCVState *env, uint32_t desc) \
+{                                                                       \
+    vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN,             \
+                    STORE_FN, ctzl(sizeof(ETYPE)), GETPC());            \
 }
 
 GEN_VEXT_ST_INDEX(vsxei8_8_v,   int8_t,  idx_b, ste_b_tlb)
@@ -691,7 +684,7 @@ GEN_VEXT_ST_INDEX(vsxei64_64_v, int64_t, idx_d, ste_d_tlb)
 /*
  * unit-stride fault-only-fisrt load instructions
  */
-static inline void
+static void
 vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
           uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
           vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra)
@@ -922,12 +915,12 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
     vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
 }
 
-#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST)   \
-void HELPER(NAME)(void *vd, void *v0, target_ulong base,        \
-                  CPURISCVState *env, uint32_t desc)            \
-{                                                               \
-    vext_ldff(vd, v0, base, env, desc, LOAD_FN_TLB,             \
-              LOAD_FN_HOST, ctzl(sizeof(ETYPE)), GETPC());      \
+#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST)           \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base,   \
+                               CPURISCVState *env, uint32_t desc)       \
+{                                                                       \
+    vext_ldff(vd, v0, base, env, desc, LOAD_FN_TLB,                     \
+              LOAD_FN_HOST, ctzl(sizeof(ETYPE)), GETPC());              \
 }
 
 GEN_VEXT_LDFF(vle8ff_v,  int8_t,  lde_b_tlb, lde_b_host)
@@ -948,7 +941,7 @@ GEN_VEXT_LDFF(vle64ff_v, int64_t, lde_d_tlb, lde_d_host)
 /*
  * load and store whole register instructions
  */
-static inline QEMU_ALWAYS_INLINE void
+static void
 vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
                 vext_ldst_elem_fn_tlb *ldst_tlb,
                 vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
@@ -963,8 +956,8 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
 }
 
 #define GEN_VEXT_LD_WHOLE(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST)   \
-void HELPER(NAME)(void *vd, target_ulong base, CPURISCVState *env,  \
-                  uint32_t desc)                                    \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base,         \
+                               CPURISCVState *env, uint32_t desc)   \
 {                                                                   \
     vext_ldst_whole(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \
                     ctzl(sizeof(ETYPE)), GETPC(), true);            \
@@ -988,8 +981,8 @@ GEN_VEXT_LD_WHOLE(vl8re32_v, int32_t, lde_w_tlb, lde_w_host)
 GEN_VEXT_LD_WHOLE(vl8re64_v, int64_t, lde_d_tlb, lde_d_host)
 
 #define GEN_VEXT_ST_WHOLE(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST)     \
-void HELPER(NAME)(void *vd, target_ulong base, CPURISCVState *env,      \
-                  uint32_t desc)                                        \
+QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base,             \
+                               CPURISCVState *env, uint32_t desc)       \
 {                                                                       \
     vext_ldst_whole(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST,   \
                     ctzl(sizeof(ETYPE)), GETPC(), false);               \
-- 
2.43.0