[PATCH 16/17] target/arm: Create helper_dc_zva_mte

Reorganize helper_mte_check_zva into helper_dc_zva_mte so that
we only require one function call from tcg generated code.

Signed-off-by: Richard Henderson <[email protected]>
---
 target/arm/tcg/helper-a64-defs.h |   2 +-
 target/arm/tcg/mte_helper.c      | 210 ++++++++++++++++---------------
 target/arm/tcg/translate-a64.c   |   8 +-
 3 files changed, 113 insertions(+), 107 deletions(-)

diff --git a/target/arm/tcg/helper-a64-defs.h b/target/arm/tcg/helper-a64-defs.h
index ac8f460ffb..518e74c8aa 100644
--- a/target/arm/tcg/helper-a64-defs.h
+++ b/target/arm/tcg/helper-a64-defs.h
@@ -81,6 +81,7 @@ DEF_HELPER_3(vfp_ah_maxs, f32, f32, f32, fpst)
 DEF_HELPER_3(vfp_ah_maxd, f64, f64, f64, fpst)
 
 DEF_HELPER_FLAGS_3(dc_zva, TCG_CALL_NO_WG, void, env, i64, i32)
+DEF_HELPER_FLAGS_3(dc_zva_mte, TCG_CALL_NO_WG, void, env, i64, i32)
 
 DEF_HELPER_FLAGS_3(pacia, TCG_CALL_NO_WG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_3(pacib, TCG_CALL_NO_WG, i64, env, i64, i64)
@@ -99,7 +100,6 @@ DEF_HELPER_FLAGS_2(xpaci, TCG_CALL_NO_RWG_SE, i64, env, i64)
 DEF_HELPER_FLAGS_2(xpacd, TCG_CALL_NO_RWG_SE, i64, env, i64)
 
 DEF_HELPER_FLAGS_3(mte_check, TCG_CALL_NO_WG, i64, env, i32, i64)
-DEF_HELPER_FLAGS_3(mte_check_zva, TCG_CALL_NO_WG, i64, env, i32, i64)
 DEF_HELPER_FLAGS_3(irg, TCG_CALL_NO_RWG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_4(addsubg, TCG_CALL_NO_RWG_SE, i64, env, i64, s32, i32)
 DEF_HELPER_FLAGS_4(ldg, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c
index 4fa6066cbc..4161f4eb32 100644
--- a/target/arm/tcg/mte_helper.c
+++ b/target/arm/tcg/mte_helper.c
@@ -91,6 +91,12 @@ typedef struct AllocationTagMem {
 typedef enum {
     /* Trap on missing pages, invalid tag access, or watchpoints. */
     ATM_NORMAL,
+    /*
+     * Trap on missing pages, invalid tag access.
+     * Align the pointer after initial DATA_ACCESS faults.
+     * Do not check for watchpoints.
+     */
+    ATM_ZVA,
     /* Gracefully return no tag memory for invalid pages. */
     ATM_PROBE_PAGES,
     /*
@@ -104,7 +110,7 @@ typedef enum {
 
 static AllocationTagMem
 allocation_tag_mem_internal(CPUARMState *env, int ptr_mmu_idx,
-                            vaddr ptr, MMUAccessType ptr_access,
+                            vaddr ptr_orig, MMUAccessType ptr_access,
                             int ptr_size, MMUAccessType tag_access,
                             uintptr_t ra, AllocationTagMemKind atm_kind)
 {
@@ -113,7 +119,7 @@ allocation_tag_mem_internal(CPUARMState *env, int ptr_mmu_idx,
     assert((atm_kind >= ATM_PROBE_PAGES) == (ra == 0));
 
 #ifdef CONFIG_USER_ONLY
-    vaddr clean_ptr = useronly_clean_ptr(ptr);
+    vaddr clean_ptr = useronly_clean_ptr(ptr_orig);
     int flags = page_get_flags(clean_ptr);
     int req_flags = ptr_access == MMU_DATA_STORE ? PAGE_WRITE_ORG : PAGE_READ;
 
@@ -123,25 +129,29 @@ allocation_tag_mem_internal(CPUARMState *env, int ptr_mmu_idx,
             ret.flags = TLB_INVALID_MASK;
             goto fini;
         case ATM_NORMAL:
-            cpu_loop_exit_sigsegv(env_cpu(env), ptr, ptr_access,
+        case ATM_ZVA:
+            cpu_loop_exit_sigsegv(env_cpu(env), ptr_orig, ptr_access,
                                   !(flags & PAGE_VALID), ra);
         default:
             g_assert_not_reached();
         }
     }
+
     if (atm_kind == ATM_NORMAL) {
-        int in_page = -(ptr | TARGET_PAGE_MASK);
+        int in_page = -(ptr_orig | TARGET_PAGE_MASK);
         if (unlikely(ptr_size > in_page)) {
-            probe_access(env, ptr + in_page, ptr_size - in_page,
+            probe_access(env, ptr_orig + in_page, ptr_size - in_page,
                          ptr_access, MMU_USER_IDX, ra);
         }
+    } else if (atm_kind == ATM_ZVA) {
+        clean_ptr &= -ptr_size;
     }
 
     /* Require both MAP_ANON and PROT_MTE for the page. */
     if ((flags & PAGE_ANON) && (flags & PAGE_MTE)) {
         size_t page_data_size = TARGET_PAGE_SIZE >> (LOG2_TAG_GRANULE + 1);
         uint8_t *tags = page_get_target_data(clean_ptr, page_data_size);
-        uintptr_t index = extract64(ptr, LOG2_TAG_GRANULE + 1,
+        uintptr_t index = extract64(clean_ptr, LOG2_TAG_GRANULE + 1,
                                     TARGET_PAGE_BITS - LOG2_TAG_GRANULE - 1);
         ret.tag_mem = tags + index;
     }
@@ -149,6 +159,7 @@ allocation_tag_mem_internal(CPUARMState *env, int ptr_mmu_idx,
     ret.ptr_mem = g2h_untagged_vaddr(clean_ptr);
     ret.attrs = MEMTXATTRS_UNSPECIFIED;
 #else
+    uint64_t ptr = ptr_orig;
     CPUTLBEntryFull *full;
     hwaddr ptr_paddr, tag_paddr, xlat;
     MemoryRegion *mr;
@@ -169,6 +180,15 @@ allocation_tag_mem_internal(CPUARMState *env, int ptr_mmu_idx,
         goto fini;
     }
 
+    /*
+     * For DC_ZVA and friends, the initial page lookup and trap is performed
+     * with the original address, but we need to align the further results.
+     */
+    if (atm_kind == ATM_ZVA) {
+        ptr &= -ptr_size;
+        ret.ptr_mem = (void *)((uintptr_t)ret.ptr_mem & -ptr_size);
+    }
+
     /*
      * Remember these across the second lookup below,
      * which may invalidate this pointer via tlb resize.
@@ -204,7 +224,8 @@ allocation_tag_mem_internal(CPUARMState *env, int ptr_mmu_idx,
                 ret.flags = TLB_INVALID_MASK;
                 goto fini;
             }
-            mte_perm_check_fail(env, ptr, ra, tag_access == MMU_DATA_STORE);
+            mte_perm_check_fail(env, ptr_orig, ra,
+                                tag_access == MMU_DATA_STORE);
         }
         /* fall through */
 
@@ -1146,118 +1167,103 @@ void HELPER(dc_zva)(CPUARMState *env, uint64_t addr, uint32_t desc)
     do_dczva_0(env, addr, len, mem, mmu_idx, flags, ra);
 }
 
-/*
- * Perform an MTE checked access for DC_ZVA.
- */
-uint64_t HELPER(mte_check_zva)(CPUARMState *env, uint32_t desc, uint64_t ptr)
+void HELPER(dc_zva_mte)(CPUARMState *env, uintptr_t ptr_orig, uint32_t desc)
 {
+    int dcz_bytes = FIELD_EX32(desc, MTEDESC, SIZEM1) + 1;
+    int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
     uintptr_t ra = GETPC();
-    int log2_dcz_bytes, log2_tag_bytes;
-    int mmu_idx, bit55;
-    intptr_t dcz_bytes, tag_bytes, i;
-    void *mem;
-    uint64_t ptr_tag, mem_tag, align_ptr;
+    uint64_t ptr = ptr_orig, ptr_tag, mem_tag;
+    int bit55;
+
+    AllocationTagMem r =
+        allocation_tag_mem_internal(env, mmu_idx, ptr, MMU_DATA_STORE,
+                                    dcz_bytes, MMU_DATA_LOAD, ra, ATM_ZVA);
+
+    r.flags = do_dcxva_traps(env, ptr, dcz_bytes, mmu_idx,
+                             r.flags, r.attrs, ra);
 
     bit55 = extract64(ptr, 55, 1);
-
-    /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
-    if (unlikely(!tbi_or_mtx_check(desc, bit55))) {
-        return ptr;
-    }
-
     ptr_tag = allocation_tag_from_addr(ptr);
+    ptr &= -dcz_bytes;
 
-    if (tcma_check(desc, bit55, ptr_tag)) {
-        goto done;
-    }
+    if (unlikely(!tbi_or_mtx_check(desc, bit55)) ||
+        tcma_check(desc, bit55, ptr_tag)) {
+        /* If TBI is disabled, or TCMA disabled, the access is unchecked. */
+    } else if (r.tag_mem) {
+        /*
+         * In arm_cpu_realizefn, we asserted that dcz > LOG2_TAG_GRANULE+1,
+         * i.e. 32 bytes, which is an unreasonably small dcz anyway, to make
+         * sure that we can access one complete tag byte here.
+         */
+        int tag_bytes = dcz_bytes / (TAG_GRANULE * 2);
 
-    /*
-     * In arm_cpu_realizefn, we asserted that dcz > LOG2_TAG_GRANULE+1,
-     * i.e. 32 bytes, which is an unreasonably small dcz anyway, to make
-     * sure that we can access one complete tag byte here.
-     */
-    log2_dcz_bytes = get_dczid_bs(env_archcpu(env)) + 2;
-    log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1);
-    dcz_bytes = (intptr_t)1 << log2_dcz_bytes;
-    tag_bytes = (intptr_t)1 << log2_tag_bytes;
-    align_ptr = ptr & -dcz_bytes;
+        /*
+         * Unlike the reasoning for checkN, DC_ZVA is always aligned, and thus
+         * it is quite easy to perform all of the comparisons at once without
+         * any extra masking.
+         *
+         * The most common zva block size is 64; some of the thunderx cpus use
+         * a block size of 128.  For user-only, aarch64_max_initfn will set the
+         * block size to 512.  Fill out the other cases for future-proofing.
+         *
+         * In order to be able to find the first miscompare later, we want the
+         * tag bytes to be in little-endian order.
+         */
+        switch (tag_bytes) {
+        case 1: /* zva_blocksize 32 */
+            mem_tag = *(uint8_t *)r.tag_mem;
+            ptr_tag *= 0x11u;
+            goto cmp_tag;
+        case 2: /* zva_blocksize 64 */
+            mem_tag = cpu_to_le16(*(uint16_t *)r.tag_mem);
+            ptr_tag *= 0x1111u;
+            goto cmp_tag;
+        case 4: /* zva_blocksize 128 */
+            mem_tag = cpu_to_le32(*(uint32_t *)r.tag_mem);
+            ptr_tag *= 0x11111111u;
+            goto cmp_tag;
+        case 8: /* zva_blocksize 256 */
+            mem_tag = cpu_to_le64(*(uint64_t *)r.tag_mem);
+            ptr_tag *= 0x1111111111111111ull;
+        cmp_tag:
+            if (unlikely(mem_tag != ptr_tag)) {
+                goto fail;
+            }
+            break;
 
-    /*
-     * Trap if accessing an invalid page.  DC_ZVA requires that we supply
-     * the original pointer for an invalid page.  But watchpoints require
-     * that we probe the actual space.  So do both.
-     */
-    mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
-    (void) probe_write(env, ptr, 1, mmu_idx, ra);
-    mem = allocation_tag_mem(env, mmu_idx, align_ptr, MMU_DATA_STORE,
-                             dcz_bytes, MMU_DATA_LOAD, ra);
-    if (!mem) {
+        case 16: /* zva_blocksize 512 */
+        case 32: /* zva_blocksize 1024 */
+        case 64: /* zva_blocksize 2048 */
+            ptr_tag *= 0x1111111111111111ull;
+            for (int i = 0; i < tag_bytes; i += 8, ptr += 16 * TAG_GRANULE) {
+                mem_tag = cpu_to_le64(*(uint64_t *)(r.tag_mem + i));
+                if (unlikely(mem_tag != ptr_tag)) {
+                    goto fail;
+                }
+            }
+            break;
+
+        default:
+            /* Values above 2KiB are architecturally invalid. */
+            g_assert_not_reached();
+
+        fail:
+            /* Locate the first nibble that differs. */
+            ptr += (ctz64(mem_tag ^ ptr_tag) >> 4) * TAG_GRANULE;
+            mte_check_fail(env, desc, ptr, ra);
+        }
+    } else {
         /*
          * If mtx is enabled, then the access is MemTag_CanonicallyTagged,
          * otherwise it is Untagged. See AArch64.S1DecodeMemAttrs and
          * AArch64.S1DisabledOutput.
          */
         if (mtx_check(desc, bit55) && !tag_is_canonical(ptr_tag, bit55)) {
-            mte_check_fail(env, desc, ptr, ra);
+            mte_check_fail(env, desc, ptr_orig, ra);
         }
-        goto done;
     }
 
-    /*
-     * Unlike the reasoning for checkN, DC_ZVA is always aligned, and thus
-     * it is quite easy to perform all of the comparisons at once without
-     * any extra masking.
-     *
-     * The most common zva block size is 64; some of the thunderx cpus use
-     * a block size of 128.  For user-only, aarch64_max_initfn will set the
-     * block size to 512.  Fill out the other cases for future-proofing.
-     *
-     * In order to be able to find the first miscompare later, we want the
-     * tag bytes to be in little-endian order.
-     */
-    switch (log2_tag_bytes) {
-    case 0: /* zva_blocksize 32 */
-        mem_tag = *(uint8_t *)mem;
-        ptr_tag *= 0x11u;
-        break;
-    case 1: /* zva_blocksize 64 */
-        mem_tag = cpu_to_le16(*(uint16_t *)mem);
-        ptr_tag *= 0x1111u;
-        break;
-    case 2: /* zva_blocksize 128 */
-        mem_tag = cpu_to_le32(*(uint32_t *)mem);
-        ptr_tag *= 0x11111111u;
-        break;
-    case 3: /* zva_blocksize 256 */
-        mem_tag = cpu_to_le64(*(uint64_t *)mem);
-        ptr_tag *= 0x1111111111111111ull;
-        break;
-
-    default: /* zva_blocksize 512, 1024, 2048 */
-        ptr_tag *= 0x1111111111111111ull;
-        i = 0;
-        do {
-            mem_tag = cpu_to_le64(*(uint64_t *)(mem + i));
-            if (unlikely(mem_tag != ptr_tag)) {
-                goto fail;
-            }
-            i += 8;
-            align_ptr += 16 * TAG_GRANULE;
-        } while (i < tag_bytes);
-        goto done;
-    }
-
-    if (likely(mem_tag == ptr_tag)) {
-        goto done;
-    }
-
- fail:
-    /* Locate the first nibble that differs. */
-    i = ctz64(mem_tag ^ ptr_tag) >> 4;
-    mte_check_fail(env, desc, align_ptr + i * TAG_GRANULE, ra);
-
- done:
-    return useronly_clean_ptr(ptr);
+    do_dczva_0(env, ptr, dcz_bytes, r.ptr_mem, mmu_idx, r.flags, ra);
 }
 
 uint64_t mte_mops_probe(CPUARMState *env, uint64_t ptr, uint64_t size,
diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c
index c3ff46989a..cae8d6826f 100644
--- a/target/arm/tcg/translate-a64.c
+++ b/target/arm/tcg/translate-a64.c
@@ -3139,13 +3139,13 @@ static void handle_sys(DisasContext *s, bool isread,
         /* Writes clear the aligned block of memory which rt points into. */
         {
             TCGv_i32 desc = gen_mtedesc_zva(s);
+
+            tcg_rt = cpu_reg(s, rt);
             if (s->mte_active[0]) {
-                tcg_rt = tcg_temp_new_i64();
-                gen_helper_mte_check_zva(tcg_rt, tcg_env, desc, cpu_reg(s, rt));
+                gen_helper_dc_zva_mte(tcg_env, tcg_rt, desc);
             } else {
-                tcg_rt = cpu_reg(s, rt);
+                gen_helper_dc_zva(tcg_env, tcg_rt, desc);
             }
-            gen_helper_dc_zva(tcg_env, tcg_rt, desc);
         }
         return;
     case ARM_CP_DC_GVA:
-- 
2.43.0