[PATCH v2 08/23] target/riscv: Rewrite vext_ldst_us

We already computed the page split here, which means we
were quite ready to call probe_access directly.  Do so,
then use vext_ldst_nf_{host,tlb} to load the elements.

Signed-off-by: Richard Henderson <[email protected]>
---
 target/riscv/tcg/vector_helper.c | 88 ++++++++++++++++++++------------
 1 file changed, 56 insertions(+), 32 deletions(-)

diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 505f0331f2..36e64465f2 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -477,17 +477,19 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
              vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
              uint32_t evl, uintptr_t ra, bool is_load)
 {
-    target_ulong page_split, elems, addr;
+    target_ulong elems, addr, last, last_in_page, page_split;
     uint32_t nf = vext_nf(desc);
     uint32_t max_elems = vext_max_elems(desc, log2_esz);
     uint32_t esz = 1 << log2_esz;
     uint32_t msize = nf * esz;
     int mmu_index = riscv_env_mmu_index(env, false);
     MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
+    uint32_t i = env->vstart;
+    void *host;
 
     VSTART_CHECK_EARLY_EXIT(env, evl);
 
-    addr = base + env->vstart * msize;
+    addr = base + i * msize;
 
     /* Recognize alignment fault before memory protection fault. */
     vext_test_alignment(env, addr, esz, access_type, mmu_index, ra);
@@ -498,50 +500,72 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc,
      * by simply calling ldst_tlb.
      */
     if (nf == 1 && (evl << log2_esz) <= 6) {
-        for (uint32_t i = env->vstart; i < evl;
-             env->vstart = ++i, addr += esz) {
+        for (; i < evl; env->vstart = ++i, addr += esz) {
             ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
         }
-        env->vstart = 0;
-        vext_set_tail_elems_1s(evl, vd, desc, nf, esz, max_elems);
-        return;
+        goto tail;
     }
 #endif
 
-    /* Calculate the page range of first page */
+    /* Calculate the page range of first page. */
+    last = base + evl * msize - 1;
+    last_in_page = addr | ~TARGET_PAGE_MASK;
     page_split = -(addr | TARGET_PAGE_MASK);
-    /* Get number of elements */
-    elems = page_split / msize;
-    if (unlikely(env->vstart + elems >= evl)) {
-        elems = evl - env->vstart;
-    }
+
+    /* Validate the first page is accessible. */
+    host = probe_access(env, adjust_addr(env, addr),
+                        MIN(last, last_in_page) - addr + 1,
+                        access_type, mmu_index, ra);
+
+    /* Get number of complete elements in the first page. */
+    elems = MIN(page_split / msize, evl - i);
 
     /* Load/store elements in the first page */
     if (likely(elems)) {
-        vext_page_ldst_us(env, vd, addr, elems, nf, max_elems, log2_esz,
-                          is_load, mmu_index, ldst_tlb, ldst_host, ra);
-    }
+        uint32_t page_evl = i + elems;
 
-    /* Load/store elements in the second page */
-    if (unlikely(env->vstart < evl)) {
-        addr = base + env->vstart * msize;
-
-        /* Cross page element */
-        if (unlikely(page_split % msize)) {
-            vext_ldst_nf_tlb(env, vd, addr, env->vstart,
-                             nf, esz, max_elems, ldst_tlb, ra);
-            env->vstart++;
-            addr += msize;
+        if (host) {
+            vext_page_ldst_us_host(vd, host, i, page_evl, nf, log2_esz,
+                                   max_elems, ldst_host, is_load);
+        } else {
+            vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
+                                  log2_esz, max_elems,
+                                  ldst_tlb, mmu_index, ra);
         }
 
-        /* Get number of elements of second page */
-        elems = evl - env->vstart;
-
-        /* Load/store elements in the second page */
-        vext_page_ldst_us(env, vd, addr, elems, nf, max_elems, log2_esz,
-                          is_load, mmu_index, ldst_tlb, ldst_host, ra);
+        /* Usually the first page contains the entire vector. */
+        if (likely(page_evl == evl)) {
+            goto tail;
+        }
+        env->vstart = i = page_evl;
+        addr += elems * msize;
     }
 
+    /* Cross page element */
+    if (unlikely(page_split % msize)) {
+        vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz, max_elems, ldst_tlb, ra);
+        if (i == evl) {
+            goto tail;
+        }
+        env->vstart = i;
+        addr += msize;
+    }
+
+    /* Validate the second page is accessible. */
+    assert(i < evl);
+    elems = evl - i;
+    host = probe_access(env, adjust_addr(env, addr), elems * msize,
+                        access_type, mmu_index, ra);
+
+    if (host) {
+        vext_page_ldst_us_host(vd, host, i, evl, nf, log2_esz,
+                               max_elems, ldst_host, is_load);
+    } else {
+        vext_page_ldst_us_tlb(env, vd, addr, i, evl, nf, log2_esz, max_elems,
+                              ldst_tlb, mmu_index, ra);
+    }
+
+ tail:
     env->vstart = 0;
     vext_set_tail_elems_1s(evl, vd, desc, nf, esz, max_elems);
 }
-- 
2.43.0