[PATCH v2 07/23] target/riscv: Split out vext_page_ldst_us_{host, tlb}

Split out the tail of vext_page_ldst_us, after probing.
Merge vext_continuous_ldst_host into vext_page_ldst_us.

Signed-off-by: Richard Henderson <[email protected]>
---
 target/riscv/tcg/vector_helper.c | 101 ++++++++++++++++---------------
 1 file changed, 51 insertions(+), 50 deletions(-)

diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 18840336cb..505f0331f2 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -275,33 +275,6 @@ GEN_VEXT_HOST_ST_ELEM(ste_h, uint16_t, H2, stw_le)
 GEN_VEXT_HOST_ST_ELEM(ste_w, uint32_t, H4, stl_le)
 GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le)
 
-static inline QEMU_ALWAYS_INLINE void
-vext_continuous_ldst_host(CPURISCVState *env, vext_ldst_elem_fn_host *ldst_host,
-                        void *vd, uint32_t evl, uint32_t reg_start, void *host,
-                        uint32_t esz, bool is_load)
-{
-    if (HOST_BIG_ENDIAN) {
-        for (; reg_start < evl; reg_start++, host += esz) {
-            ldst_host(vd, reg_start, host);
-        }
-    } else {
-        if (esz == 1) {
-            uint32_t byte_offset = reg_start * esz;
-            uint32_t size = (evl - reg_start) * esz;
-
-            if (is_load) {
-                memcpy(vd + byte_offset, host, size);
-            } else {
-                memcpy(host, vd + byte_offset, size);
-            }
-        } else {
-            for (; reg_start < evl; reg_start++, host += esz) {
-                ldst_host(vd, reg_start, host);
-            }
-        }
-    }
-}
-
 static void vext_set_tail_elems_1s(uint32_t vl, void *vd,
                                    uint32_t desc, uint32_t nf,
                                    uint32_t esz, uint32_t max_elems)
@@ -411,6 +384,52 @@ GEN_VEXT_ST_STRIDE(vsse64_v, int64_t, ste_d_tlb)
  */
 
 /* unmasked unit-stride load and store operation */
+static inline QEMU_ALWAYS_INLINE void
+vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl,
+                       uint32_t nf, uint32_t log2_esz, uint32_t max_elems,
+                       vext_ldst_elem_fn_host *ldst_host, bool is_load)
+{
+    uint32_t esz = 1 << log2_esz;
+
+    if (nf == 1) {
+        if (!HOST_BIG_ENDIAN && esz == 1) {
+            size_t size = evl - i;
+            if (is_load) {
+                memcpy(vd + i, host, size);
+            } else {
+                memcpy(host, vd + i, size);
+            }
+        } else {
+            do {
+                ldst_host(vd, i, host);
+                host += esz;
+            } while (++i < evl);
+        }
+    } else {
+        uint32_t msize = nf << log2_esz;
+        do {
+            vext_ldst_nf_host(vd, host, i, nf, esz, max_elems, ldst_host);
+            host += msize;
+        } while (++i < evl);
+    }
+}
+
+static inline QEMU_ALWAYS_INLINE void
+vext_page_ldst_us_tlb(CPURISCVState *env, void *vd, target_ulong addr,
+                      uint32_t i, uint32_t evl, uint32_t nf,
+                      uint32_t log2_esz, uint32_t max_elems,
+                      vext_ldst_elem_fn_tlb *ldst_tlb,
+                      int mmu_index, uintptr_t ra)
+{
+    uint32_t esz = 1 << log2_esz;
+    uint32_t msize = nf << log2_esz;
+    do {
+        vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
+        addr += msize;
+        env->vstart = ++i;
+    } while (i < evl);
+}
+
 static inline QEMU_ALWAYS_INLINE void
 vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
                   uint32_t elems, uint32_t nf, uint32_t max_elems,
@@ -420,39 +439,21 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr,
 {
     void *host;
     int flags;
-    uint32_t esz = 1 << log2_esz;
     uint32_t size = (elems * nf) << log2_esz;
-    uint32_t msize = nf * esz;
-    uint32_t evl = env->vstart + elems;
+    uint32_t i = env->vstart;
     MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
 
-    /*
-     * Maximum vector length is VLMAX == 2^16 == LMUL * VL / SEW, and
-     * occurs for LMUL == 8, SEW == 8, VL == 2^16.
-     */
-    g_assert(env->vstart < UINT16_MAX && UINT16_MAX - env->vstart >= elems);
-
     /* Check page permission/pmp/watchpoint/etc. */
     probe_pages(env, addr, size, ra, access_type, mmu_index, &host, &flags,
                 true);
 
     if (flags == 0) {
-        if (nf == 1) {
-            vext_continuous_ldst_host(env, ldst_host, vd, evl, env->vstart,
-                                      host, esz, is_load);
-        } else {
-            for (uint32_t i = env->vstart; i < evl; ++i) {
-                vext_ldst_nf_host(vd, host, i, nf, esz, max_elems, ldst_host);
-                host += msize;
-            }
-        }
+        vext_page_ldst_us_host(vd, host, i, i + elems, nf, log2_esz,
+                               max_elems, ldst_host, is_load);
         env->vstart += elems;
     } else {
-        for (uint32_t i = env->vstart; i < evl; env->vstart = ++i) {
-            vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
-                             max_elems, ldst_tlb, ra);
-            addr += msize;
-        }
+        vext_page_ldst_us_tlb(env, vd, addr, i, i + elems, nf, log2_esz,
+                              max_elems, ldst_tlb, mmu_index, ra);
     }
 }
 
-- 
2.43.0