[PATCH v2 17/23] target/riscv: Pass MemOpIdx to vext_ldst_elem_fn_tlb

Avoids continually looking up mmu_index with every operation.

Signed-off-by: Richard Henderson <[email protected]>
---
 target/riscv/tcg/vector_helper.c | 91 +++++++++++++++-----------------
 1 file changed, 43 insertions(+), 48 deletions(-)

diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 98a0a5d025..f65da46144 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -171,32 +171,20 @@ static inline MemOpIdx vext_oi(uint32_t desc, MemOp memop)
     return make_memop_idx(memop, FIELD_EX32(data, VDATA, MEM_IDX));
 }
 
-static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size)
-{
-    int mmu_idx = riscv_env_mmu_index(env, false);
-    MemOp memop = size_memop(size) | mo_endian_env(env);
-
-    if (!riscv_cpu_cfg(env)->ext_zicclsm) {
-        memop |= MO_ALIGN;
-    }
-
-    return make_memop_idx(memop, mmu_idx);
-}
-
 /* elements operations for load and store */
 typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr,
-                                   uint32_t idx, void *vd, uintptr_t retaddr);
+                                   uint32_t idx, void *vd,
+                                   MemOpIdx oi, uintptr_t retaddr);
 typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void *host);
 
-#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF)         \
-static inline QEMU_ALWAYS_INLINE                            \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr,           \
-                uint32_t idx, void *vd, uintptr_t retaddr)  \
-{                                                           \
-    ETYPE *cur = ((ETYPE *)vd + H(idx));                    \
-    MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE));  \
-    *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr);       \
-}                                                           \
+#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF)                     \
+static inline QEMU_ALWAYS_INLINE                                        \
+void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,         \
+                void *vd, MemOpIdx oi, uintptr_t retaddr)               \
+{                                                                       \
+    ETYPE *cur = ((ETYPE *)vd + H(idx));                                \
+    *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr);                   \
+}
 
 #define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF)        \
 static inline QEMU_ALWAYS_INLINE                            \
@@ -216,15 +204,14 @@ GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, lduw_le)
 GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le)
 GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le)
 
-#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF)         \
-static inline QEMU_ALWAYS_INLINE                            \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr,           \
-                uint32_t idx, void *vd, uintptr_t retaddr)  \
-{                                                           \
-    ETYPE data = *((ETYPE *)vd + H(idx));                   \
-    MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE));  \
-    cpu_##STSUF##_mmu(env, addr, data, oi, retaddr);        \
-}                                                           \
+#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF)                     \
+static inline QEMU_ALWAYS_INLINE                                        \
+void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx,         \
+                void *vd, MemOpIdx oi, uintptr_t retaddr)               \
+{                                                                       \
+    ETYPE data = *((ETYPE *)vd + H(idx));                               \
+    cpu_##STSUF##_mmu(env, addr, data, oi, retaddr);                    \
+}
 
 #define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF)        \
 static inline QEMU_ALWAYS_INLINE                            \
@@ -282,10 +269,11 @@ static void vext_ldst_nf_host(void *vd, void *host, uint32_t i, uint32_t nf,
 static void vext_ldst_nf_tlb(CPURISCVState *env, void *vd, target_ulong addr,
                              uint32_t i, uint32_t nf,
                              uint32_t esz, uint32_t max_elems,
-                             vext_ldst_elem_fn_tlb *ldst_tlb, uintptr_t ra)
+                             vext_ldst_elem_fn_tlb *ldst_tlb,
+                             MemOpIdx oi, uintptr_t ra)
 {
     for (uint32_t k = 0; k < nf; k++, addr += esz) {
-        ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+        ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems, vd, oi, ra);
     }
 }
 
@@ -302,13 +290,14 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride,
     uint32_t max_elems = vext_max_elems(desc, log2_esz);
     uint32_t esz = 1 << log2_esz;
     uint32_t vma = vext_vma(desc);
+    MemOpIdx oi = vext_oi(desc, log2_esz);
 
     VSTART_CHECK_EARLY_EXIT(env, env->vl);
 
     for (uint32_t i = env->vstart; i < env->vl; env->vstart = ++i) {
         if (vm || vext_elem_mask(v0, i)) {
             vext_ldst_nf_tlb(env, vd, base + stride * i, i, nf,
-                             esz, max_elems, ldst_elem, ra);
+                             esz, max_elems, ldst_elem, oi, ra);
         } else if (vma) {
             vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
         }
@@ -388,12 +377,13 @@ vext_page_ldst_us_tlb(CPURISCVState *env, void *vd, target_ulong addr,
                       uint32_t i, uint32_t evl, uint32_t nf,
                       uint32_t log2_esz, uint32_t max_elems,
                       vext_ldst_elem_fn_tlb *ldst_tlb,
-                      int mmu_index, uintptr_t ra)
+                      MemOpIdx oi, uintptr_t ra)
 {
     uint32_t esz = 1 << log2_esz;
     uint32_t msize = nf << log2_esz;
     do {
-        vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
+        vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
+                         max_elems, ldst_tlb, oi, ra);
         addr += msize;
         env->vstart = ++i;
     } while (i < evl);
@@ -443,7 +433,7 @@ vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env,
      */
     if (nf == 1 && (evl << log2_esz) <= 6) {
         for (; i < evl; env->vstart = ++i, addr += esz) {
-            ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
+            ldst_tlb(env, adjust_addr(env, addr), i, vd, oi, ra);
         }
         return;
     }
@@ -472,7 +462,7 @@ vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env,
         } else {
             vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
                                   log2_esz, max_elems,
-                                  ldst_tlb, mmu_index, ra);
+                                  ldst_tlb, oi, ra);
         }
 
         /* Usually the first page contains the entire vector. */
@@ -485,7 +475,8 @@ vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env,
 
     /* Cross page element */
     if (unlikely(page_split % msize)) {
-        vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz, max_elems, ldst_tlb, ra);
+        vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz,
+                         max_elems, ldst_tlb, oi, ra);
         if (i == evl) {
             return;
         }
@@ -504,7 +495,7 @@ vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env,
                                max_elems, ldst_host, is_load);
     } else {
         vext_page_ldst_us_tlb(env, vd, addr, i, evl, nf, log2_esz, max_elems,
-                              ldst_tlb, mmu_index, ra);
+                              ldst_tlb, oi, ra);
     }
 }
 
@@ -623,6 +614,7 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
     uint32_t max_elems = vext_max_elems(desc, log2_esz);
     uint32_t esz = 1 << log2_esz;
     uint32_t vma = vext_vma(desc);
+    MemOpIdx oi = vext_oi(desc, log2_esz);
 
     VSTART_CHECK_EARLY_EXIT(env, env->vl);
 
@@ -636,7 +628,8 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
 
         for (uint32_t k = 0; k < nf; ++k) {
             abi_ptr addr = get_index_addr(base, i, vs2) + (k << log2_esz);
-            ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+            ldst_elem(env, adjust_addr(env, addr),
+                      i + k * max_elems, vd, oi, ra);
         }
     }
 
@@ -753,7 +746,8 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
      * If it also crosses the page, just go ahead and load it.
      */
     if (unlikely(page_split < msize) && first_active) {
-        vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz, max_elems, ldst_tlb, ra);
+        vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
+                         max_elems, ldst_tlb, oi, ra);
         env->vstart = i = 1;
         addr += msize;
         page_split = -(addr | TARGET_PAGE_MASK);
@@ -798,7 +792,7 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
              */
             if (first_active) {
                 vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
-                                 max_elems, ldst_tlb, ra);
+                                 max_elems, ldst_tlb, oi, ra);
                 i = 1;
                 addr += msize;
             }
@@ -813,12 +807,12 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
             if (vm) {
                 vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
                                       log2_esz, max_elems,
-                                      ldst_tlb, mmu_index, ra);
+                                      ldst_tlb, oi, ra);
             } else {
                 do {
                     if (vext_elem_mask(v0, i)) {
                         vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
-                                         esz, max_elems, ldst_tlb, ra);
+                                         esz, max_elems, ldst_tlb, oi, ra);
                     } else if (vma) {
                         vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
                     }
@@ -868,7 +862,8 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
             goto tail;
         }
 
-        vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
+        vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
+                         max_elems, ldst_tlb, oi, ra);
         if (++i == vl) {
             goto tail;
         }
@@ -909,12 +904,12 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
         if (vm) {
             vext_page_ldst_us_tlb(env, vd, addr, i, vl, nf,
                                   log2_esz, max_elems,
-                                  ldst_tlb, mmu_index, ra);
+                                  ldst_tlb, oi, ra);
         } else {
             do {
                 if (vext_elem_mask(v0, i)) {
                     vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
-                                     esz, max_elems, ldst_tlb, ra);
+                                     esz, max_elems, ldst_tlb, oi, ra);
                 } else if (vma) {
                     vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
                 }
-- 
2.43.0