[PATCH 11/23] target/riscv: Rewrite vext_ldff

Do not call probe_pages for every active element.
We can make do with no more than 2 such calls for
the two pages the insn might reference.

Signed-off-by: Richard Henderson <[email protected]>
---
 target/riscv/tcg/vector_helper.c | 258 ++++++++++++++++++++-----------
 1 file changed, 172 insertions(+), 86 deletions(-)

diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 5b03e23fc3..b482537f40 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -759,135 +759,221 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env,
           uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
           vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra)
 {
-    uint32_t i, vl = 0;
+    uint32_t i = env->vstart, vl = env->vl;
     uint32_t nf = vext_nf(desc);
     uint32_t vm = vext_vm(desc);
     uint32_t max_elems = vext_max_elems(desc, log2_esz);
     uint32_t esz = 1 << log2_esz;
     uint32_t msize = nf * esz;
     uint32_t vma = vext_vma(desc);
-    target_ulong addr, addr_i, offset, remain, page_split, elems;
+    target_ulong addr, last, last_in_page, page_split, elems;
     int mmu_index = riscv_env_mmu_index(env, false);
+    bool first_active;
     int flags;
     void *host;
 
     VSTART_CHECK_EARLY_EXIT(env, env->vl);
 
-    /* Search for the first active element. */
-    if (!vm) {
-        for (i = env->vstart; i < env->vl; ++i) {
+    /* Skip forward to the first active element. */
+    if (vm) {
+        first_active = i == 0;
+    } else if (i == 0 && vext_elem_mask(v0, 0)) {
+        first_active = true;
+    } else {
+        first_active = false;
+        while (1) {
             if (vext_elem_mask(v0, i)) {
                 break;
             }
             if (vma) {
                 vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
             }
+            if (++i == vl) {
+                goto tail;
+            }
         }
-        if (i == env->vl) {
-            goto tail;
-        }
-        env->vstart = i;
     }
-
-    addr = base + ((env->vstart * nf) << log2_esz);
+    addr = base + i * msize;
 
     /* Recognize alignment fault before memory protection fault. */
     vext_test_alignment(env, addr, esz, MMU_DATA_LOAD, mmu_index, ra);
 
-    page_split = -(addr | TARGET_PAGE_MASK);
-    /* Get number of elements */
-    elems = page_split / msize;
-    if (unlikely(env->vstart + elems >= env->vl)) {
-        elems = env->vl - env->vstart;
+    /* Calculate the page range of first page. */
+    last = base + vl * msize - 1;
+    last_in_page = addr | ~TARGET_PAGE_MASK;
+    page_split = last_in_page - addr;
+
+    /*
+     * If the first element is active, it must succeed.
+     * If it also crosses the page, just go ahead and load it.
+     */
+    if (unlikely(page_split < msize) && first_active) {
+        vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz, max_elems, ldst_tlb, ra);
+        env->vstart = i = 1;
+        addr += msize;
+        page_split = -(addr | TARGET_PAGE_MASK);
+        first_active = false;
     }
 
-    /* Check page permission/pmp/watchpoint/etc. */
-    probe_pages(env, addr, (env->vl - env->vstart) * msize, ra, MMU_DATA_LOAD,
-                mmu_index, &host, &flags, true);
+    /*
+     * Test whether the first page is accessible.
+     * If the first element is active, it must succeed.
+     */
+    flags = probe_access_flags(env, adjust_addr(env, addr),
+                               MIN(last, last_in_page) - addr + 1,
+                               MMU_DATA_LOAD, mmu_index, !first_active,
+                               &host, ra);
 
-    if (flags & ~TLB_WATCHPOINT) {
-        /* probe every access */
-        for (i = env->vstart; i < env->vl; i++) {
-            if (!vm && !vext_elem_mask(v0, i)) {
-                continue;
-            }
-            addr_i = adjust_addr(env, base + i * (nf << log2_esz));
-            if (i == 0) {
-                /* Allow fault on first element. */
-                probe_pages(env, addr_i, nf << log2_esz, ra, MMU_DATA_LOAD,
-                            mmu_index, &host, NULL, false);
+    /* Get number of complete elements in the first page. */
+    elems = MIN(page_split / msize, vl - i);
+
+    /* Load complete elements from the first page. */
+    if (likely(elems)) {
+        uint32_t page_evl = i + elems;
+
+        if (flags == 0) {
+            if (vm) {
+                vext_page_ldst_us_host(vd, host, i, page_evl, nf,
+                                       log2_esz, max_elems, ldst_host);
             } else {
-                remain = nf << log2_esz;
-                while (remain > 0) {
-                    offset = -(addr_i | TARGET_PAGE_MASK);
-
-                    /* Probe nonfault on subsequent elements. */
-                    probe_pages(env, addr_i, offset, 0, MMU_DATA_LOAD,
-                                mmu_index, &host, &flags, true);
-
-                    /*
-                     * Stop if invalid (unmapped) or mmio (transaction may
-                     * fail). Do not stop if watchpoint, as the spec says that
-                     * first-fault should continue to access the same
-                     * elements regardless of any watchpoint.
-                     */
-                    if (flags & ~TLB_WATCHPOINT) {
-                        vl = i;
-                        goto ProbeSuccess;
+                host -= addr - base;
+                do {
+                    if (vext_elem_mask(v0, i)) {
+                        vext_ldst_nf_host(vd, host + i * msize, i, nf, esz,
+                                          max_elems, ldst_host);
+                    } else if (vma) {
+                        vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
                     }
-                    if (remain <= offset) {
-                        break;
+                } while (++i < page_evl);
+            }
+        } else {
+            /*
+             * If the first element is active, it must succeed.
+             * This will load from MMIO or fault from INVALID.
+             */
+            if (first_active) {
+                vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
+                                 max_elems, ldst_tlb, ra);
+                i = 1;
+                addr += msize;
+            }
+
+            /* Stop if invalid (unmapped) or mmio (transaction may fail). */
+            if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
+                env->vl = i;
+                goto tail;
+            }
+
+            /* None of these ldst_tlb calls may fault. */
+            if (vm) {
+                vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
+                                      log2_esz, max_elems,
+                                      ldst_tlb, mmu_index, ra);
+            } else {
+                do {
+                    if (vext_elem_mask(v0, i)) {
+                        vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
+                                         esz, max_elems, ldst_tlb, ra);
+                    } else if (vma) {
+                        vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
                     }
-                    remain -= offset;
-                    addr_i = adjust_addr(env, addr_i + offset);
-                }
+                } while (++i < page_evl);
+            }
+        }
+
+        /* Usually the first page contains the entire vector. */
+        if (likely(page_evl == vl)) {
+            goto tail;
+        }
+        i = page_evl;
+    }
+
+    /* Skip forward to the next active element. */
+    if (!vm) {
+        while (1) {
+            if (vext_elem_mask(v0, i)) {
+                break;
+            }
+            if (vma) {
+                vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
+            }
+            if (++i == vl) {
+                goto tail;
             }
         }
     }
-ProbeSuccess:
-    /* load bytes from guest memory */
-    if (vl != 0) {
-        env->vl = vl;
+
+    addr = base + i * msize;
+    page_split = -(addr | TARGET_PAGE_MASK);
+
+    /* Validate the second page is accessible. */
+    if (unlikely(page_split < msize)) {
+        /*
+         * Cross page element which isn't first.
+         * We have not yet advanced addr to the next page.
+         */
+        target_ulong next_page = addr + page_split;
+        flags |= probe_access_flags(env, adjust_addr(env, next_page),
+                                    last - next_page + 1, MMU_DATA_LOAD,
+                                    mmu_index, true, &host, ra);
+
+        /* Stop if invalid (unmapped) or mmio (transaction may fail). */
+        if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
+            env->vl = i;
+            goto tail;
+        }
+
+        vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
+        if (++i == vl) {
+            goto tail;
+        }
+        addr += msize;
+        if (host) {
+            host += addr - next_page;
+        }
+    } else {
+        flags = probe_access_flags(env, adjust_addr(env, addr),
+                                   last - addr + 1, MMU_DATA_LOAD,
+                                   mmu_index, true, &host, ra);
+
+        /* Stop if invalid (unmapped) or mmio (transaction may fail). */
+        if (flags & (TLB_INVALID_MASK | TLB_MMIO)) {
+            env->vl = i;
+            goto tail;
+        }
     }
 
-    if (env->vstart < env->vl) {
+    /* Load complete elements from the second page. */
+    if (flags == 0) {
         if (vm) {
-            /* Load/store elements in the first page */
-            if (likely(elems)) {
-                vext_page_ldst_us(env, vd, addr, elems, nf, max_elems,
-                                  log2_esz, true, mmu_index, ldst_tlb,
-                                  ldst_host, ra);
-            }
-
-            /* Load/store elements in the second page */
-            if (unlikely(env->vstart < env->vl)) {
-                addr = base + env->vstart * msize;
-
-                /* Cross page element */
-                if (unlikely(page_split % msize)) {
-                    vext_ldst_nf_tlb(env, vd, addr, env->vstart, nf,
-                                     esz, max_elems, ldst_tlb, ra);
-                    env->vstart++;
-                    addr += msize;
-                }
-
-                /* Get number of elements of second page */
-                elems = env->vl - env->vstart;
-
-                /* Load/store elements in the second page */
-                vext_page_ldst_us(env, vd, addr, elems, nf, max_elems,
-                                  log2_esz, true, mmu_index, ldst_tlb,
-                                  ldst_host, ra);
-            }
+            vext_page_ldst_us_host(vd, host, i, vl, nf,
+                                   log2_esz, max_elems, ldst_host);
         } else {
-            for (i = env->vstart; i < env->vl; i++) {
+            host -= addr - base;
+            do {
+                if (vext_elem_mask(v0, i)) {
+                    vext_ldst_nf_host(vd, host + i * msize, i, nf, esz,
+                                      max_elems, ldst_host);
+                } else if (vma) {
+                    vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
+                }
+            } while (++i < vl);
+        }
+    } else {
+        /* None of these ldst_tlb calls may fault. */
+        if (vm) {
+            vext_page_ldst_us_tlb(env, vd, addr, i, vl, nf,
+                                  log2_esz, max_elems,
+                                  ldst_tlb, mmu_index, ra);
+        } else {
+            do {
                 if (vext_elem_mask(v0, i)) {
                     vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
                                      esz, max_elems, ldst_tlb, ra);
                 } else if (vma) {
                     vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
                 }
-            }
+            } while (++i < vl);
         }
     }
 
-- 
2.43.0