[PATCH 17/17] target/arm: Rewrite DC_GVA, DC_GZVA, STZGM

Perform alignment trap on device memory.  Recognize watchpoints.
Perform both in the correct order wrt MTX faults.

Signed-off-by: Richard Henderson <[email protected]>
---
 target/arm/tcg/helper-a64-defs.h |   5 +-
 target/arm/tcg/mte_helper.c      | 127 ++++++++++++++++++++++---------
 target/arm/tcg/translate-a64.c   |  49 +++++-------
 3 files changed, 115 insertions(+), 66 deletions(-)

diff --git a/target/arm/tcg/helper-a64-defs.h b/target/arm/tcg/helper-a64-defs.h
index 518e74c8aa..0d0830c837 100644
--- a/target/arm/tcg/helper-a64-defs.h
+++ b/target/arm/tcg/helper-a64-defs.h
@@ -82,6 +82,9 @@ DEF_HELPER_3(vfp_ah_maxd, f64, f64, f64, fpst)
 
 DEF_HELPER_FLAGS_3(dc_zva, TCG_CALL_NO_WG, void, env, i64, i32)
 DEF_HELPER_FLAGS_3(dc_zva_mte, TCG_CALL_NO_WG, void, env, i64, i32)
+DEF_HELPER_FLAGS_3(dc_gva, TCG_CALL_NO_WG, void, env, i64, i32)
+DEF_HELPER_FLAGS_3(dc_gva_stub, TCG_CALL_NO_WG, void, env, i64, i32)
+DEF_HELPER_FLAGS_3(dc_gzva, TCG_CALL_NO_WG, void, env, i64, i32)
 
 DEF_HELPER_FLAGS_3(pacia, TCG_CALL_NO_WG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_3(pacib, TCG_CALL_NO_WG, i64, env, i64, i64)
@@ -111,7 +114,7 @@ DEF_HELPER_FLAGS_4(st2g_parallel, TCG_CALL_NO_WG, void, env, i64, i64, i32)
 DEF_HELPER_FLAGS_2(st2g_stub, TCG_CALL_NO_WG, void, env, i64)
 DEF_HELPER_FLAGS_3(ldgm, TCG_CALL_NO_WG, i64, env, i64, i32)
 DEF_HELPER_FLAGS_4(stgm, TCG_CALL_NO_WG, void, env, i64, i64, i32)
-DEF_HELPER_FLAGS_4(stzgm_tags, TCG_CALL_NO_WG, void, env, i64, i64, i32)
+DEF_HELPER_FLAGS_4(stzgm, TCG_CALL_NO_WG, void, env, i64, i64, i32)
 
 DEF_HELPER_FLAGS_4(arm_unaligned_access, TCG_CALL_NO_WG,
                    noreturn, env, i64, i32, i32)
diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c
index 4161f4eb32..25adc8c714 100644
--- a/target/arm/tcg/mte_helper.c
+++ b/target/arm/tcg/mte_helper.c
@@ -671,36 +671,6 @@ void HELPER(stgm)(CPUARMState *env, uint64_t ptr, uint64_t val, uint32_t mtx)
     }
 }
 
-void HELPER(stzgm_tags)(CPUARMState *env, uint64_t ptr, uint64_t val,
-                        uint32_t mtx)
-{
-    uintptr_t ra = GETPC();
-    int mmu_idx = arm_env_mmu_index(env);
-    int log2_dcz_bytes, log2_tag_bytes;
-    intptr_t dcz_bytes, tag_bytes;
-    uint8_t *mem;
-
-    /*
-     * In arm_cpu_realizefn, we assert that dcz > LOG2_TAG_GRANULE+1,
-     * i.e. 32 bytes, which is an unreasonably small dcz anyway,
-     * to make sure that we can access one complete tag byte here.
-     */
-    log2_dcz_bytes = get_dczid_bs(env_archcpu(env)) + 2;
-    log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1);
-    dcz_bytes = (intptr_t)1 << log2_dcz_bytes;
-    tag_bytes = (intptr_t)1 << log2_tag_bytes;
-    ptr &= -dcz_bytes;
-
-    mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, dcz_bytes,
-                             MMU_DATA_STORE, ra);
-    if (mem) {
-        int tag_pair = (val & 0xf) * 0x11;
-        memset(mem, tag_pair, tag_bytes);
-    } else if (raw_mtx_check(mtx, extract64(ptr, 55, 1))) {
-        canonical_tag_write_fail(env, ptr, ra);
-    }
-}
-
 static void mte_sync_check_fail(CPUARMState *env, uint32_t desc,
                                 uint64_t dirty_ptr, uintptr_t ra)
 {
@@ -1092,8 +1062,9 @@ bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr)
 }
 
 /* Traps for DC_ZVA, DC_GVA, and friends, after successful page lookup. */
-static int do_dcxva_traps(CPUARMState *env, vaddr addr, size_t len, int mmu_idx,
-                          int flags, MemTxAttrs attrs, uintptr_t ra)
+static int do_dcxva_traps(CPUARMState *env, vaddr addr, size_t len,
+                          int mmu_idx, int flags, MemTxAttrs attrs,
+                          uint32_t mtx_write, uintptr_t ra)
 {
 #ifndef CONFIG_USER_ONLY
     /*
@@ -1106,7 +1077,17 @@ static int do_dcxva_traps(CPUARMState *env, vaddr addr, size_t len, int mmu_idx,
         arm_cpu_do_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
                                     mmu_idx, ra);
     }
+#endif
 
+    /*
+     * MTX write fault is via AArch64_S1CheckPermissions, which happens
+     * after AArch64_S1HasAlignmentFaultDueToMemType.
+     */
+    if (mtx_check(mtx_write, extract64(addr, 55, 1))) {
+        canonical_tag_write_fail(env, addr, ra);
+    }
+
+#ifndef CONFIG_USER_ONLY
     /* Watchpoints have lower priority than alignment faults. */
     if (unlikely(flags & TLB_WATCHPOINT)) {
         cpu_check_watchpoint(env_cpu(env), addr & -len, len,
@@ -1158,7 +1139,7 @@ void HELPER(dc_zva)(CPUARMState *env, uint64_t addr, uint32_t desc)
     attrs = full->attrs;
 #endif
 
-    flags = do_dcxva_traps(env, addr, len, mmu_idx, flags, attrs, ra);
+    flags = do_dcxva_traps(env, addr, len, mmu_idx, flags, attrs, 0, ra);
 
     /* After traps, treat as aligned blocks. */
     addr = addr & -len;
@@ -1180,7 +1161,7 @@ void HELPER(dc_zva_mte)(CPUARMState *env, uintptr_t ptr_orig, uint32_t desc)
                                     dcz_bytes, MMU_DATA_LOAD, ra, ATM_ZVA);
 
     r.flags = do_dcxva_traps(env, ptr, dcz_bytes, mmu_idx,
-                             r.flags, r.attrs, ra);
+                             r.flags, r.attrs, 0, ra);
 
     bit55 = extract64(ptr, 55, 1);
     ptr_tag = allocation_tag_from_addr(ptr);
@@ -1266,6 +1247,84 @@ void HELPER(dc_zva_mte)(CPUARMState *env, uintptr_t ptr_orig, uint32_t desc)
     do_dczva_0(env, ptr, dcz_bytes, r.ptr_mem, mmu_idx, r.flags, ra);
 }
 
+static void do_stzgm_tags(void *mem, int dcz_bytes, int tag)
+{
+    /*
+     * In arm_cpu_realizefn, we asserted that dcz > LOG2_TAG_GRANULE+1,
+     * i.e. 32 bytes, which is an unreasonably small dcz anyway, to make
+     * sure that we can access one complete tag byte here.
+     */
+    int tag_bytes = dcz_bytes / (TAG_GRANULE * 2);
+    int tag_pair = (tag & 0xf) * 0x11;
+    memset(mem, tag_pair, tag_bytes);
+}
+
+void HELPER(dc_gva)(CPUARMState *env, uint64_t ptr, uint32_t desc)
+{
+    int dcz_bytes = FIELD_EX32(desc, MTEDESC, SIZEM1) + 1;
+    int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
+    uintptr_t ra = GETPC();
+
+    AllocationTagMem r =
+        allocation_tag_mem_internal(env, mmu_idx, ptr, MMU_DATA_STORE,
+                                    dcz_bytes, MMU_DATA_STORE, ra, ATM_ZVA);
+
+    do_dcxva_traps(env, ptr, dcz_bytes, mmu_idx, r.flags, r.attrs,
+                   r.tag_mem ? 0 : desc, ra);
+
+    if (r.tag_mem) {
+        do_stzgm_tags(r.tag_mem, dcz_bytes, allocation_tag_from_addr(ptr));
+    }
+}
+
+/* DC GVA when tag access is disabled -- we still want all of the traps. */
+void HELPER(dc_gva_stub)(CPUARMState *env, uint64_t ptr, uint32_t desc)
+{
+    uintptr_t ra = GETPC();
+    int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
+
+#ifdef CONFIG_USER_ONLY
+    probe_write(env, ptr, 0, mmu_idx, ra);
+#else
+    int len = FIELD_EX32(desc, MTEDESC, SIZEM1) + 1;
+    void *mem;
+    CPUTLBEntryFull *full;
+    int flags = probe_access_full(env, ptr, 0, MMU_DATA_STORE, mmu_idx,
+                                  false, &mem, &full, ra);
+    do_dcxva_traps(env, ptr, len, mmu_idx, flags, full->attrs, 0, ra);
+#endif
+}
+
+static void do_gzva_stzgm(CPUARMState *env, uint64_t ptr, uint64_t tag,
+                          uint32_t desc, uintptr_t ra)
+{
+    int dcz_bytes = FIELD_EX32(desc, MTEDESC, SIZEM1) + 1;
+    int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
+
+    AllocationTagMem r =
+        allocation_tag_mem_internal(env, mmu_idx, ptr, MMU_DATA_STORE,
+                                    dcz_bytes, MMU_DATA_STORE, ra, ATM_ZVA);
+
+    r.flags = do_dcxva_traps(env, ptr, dcz_bytes, mmu_idx, r.flags, r.attrs,
+                             r.tag_mem ? 0 : desc, ra);
+
+    if (r.tag_mem) {
+        do_stzgm_tags(r.tag_mem, dcz_bytes, tag);
+    }
+    do_dczva_0(env, ptr, dcz_bytes, r.ptr_mem, mmu_idx, r.flags, ra);
+}
+
+void HELPER(dc_gzva)(CPUARMState *env, uint64_t ptr, uint32_t desc)
+{
+    do_gzva_stzgm(env, ptr, allocation_tag_from_addr(ptr), desc, GETPC());
+}
+
+void HELPER(stzgm)(CPUARMState *env, uint64_t ptr, uint64_t tag, uint32_t desc)
+{
+    int dcz_bytes = FIELD_EX32(desc, MTEDESC, SIZEM1) + 1;
+    do_gzva_stzgm(env, ptr & -dcz_bytes, tag, desc, GETPC());
+}
+
 uint64_t mte_mops_probe(CPUARMState *env, uint64_t ptr, uint64_t size,
                         uint32_t desc)
 {
diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c
index cae8d6826f..6ce123eafc 100644
--- a/target/arm/tcg/translate-a64.c
+++ b/target/arm/tcg/translate-a64.c
@@ -3150,41 +3150,25 @@ static void handle_sys(DisasContext *s, bool isread,
         return;
     case ARM_CP_DC_GVA:
         {
-            TCGv_i64 clean_addr, tag;
+            TCGv_i32 desc = gen_mtedesc_zva(s);
 
-            /*
-             * DC_GVA, like DC_ZVA, requires that we supply the original
-             * pointer for an invalid page.  Probe that address first.
-             */
             tcg_rt = cpu_reg(s, rt);
-            clean_addr = clean_data_tbi(s, tcg_rt);
-            gen_probe_access(s, clean_addr, MMU_DATA_STORE, MO_8);
-
             if (s->ata[0]) {
-                /* Extract the tag from the register to match STZGM.  */
-                tag = tcg_temp_new_i64();
-                tcg_gen_shri_i64(tag, tcg_rt, 56);
-                gen_helper_stzgm_tags(tcg_env, clean_addr, tag,
-                                      tcg_constant_i32(s->mtx));
+                gen_helper_dc_gva(tcg_env, tcg_rt, desc);
+            } else {
+                gen_helper_dc_gva_stub(tcg_env, tcg_rt, desc);
             }
         }
         return;
     case ARM_CP_DC_GZVA:
         {
-            TCGv_i64 clean_addr, tag;
             TCGv_i32 desc = gen_mtedesc_zva(s);
 
-            /* For DC_GZVA, we can rely on DC_ZVA for the proper fault. */
             tcg_rt = cpu_reg(s, rt);
-            clean_addr = clean_data_tbi(s, tcg_rt);
-            gen_helper_dc_zva(tcg_env, clean_addr, desc);
-
             if (s->ata[0]) {
-                /* Extract the tag from the register to match STZGM.  */
-                tag = tcg_temp_new_i64();
-                tcg_gen_shri_i64(tag, tcg_rt, 56);
-                gen_helper_stzgm_tags(tcg_env, clean_addr, tag,
-                                      tcg_constant_i32(s->mtx));
+                gen_helper_dc_gzva(tcg_env, tcg_rt, desc);
+            } else {
+                gen_helper_dc_zva(tcg_env, tcg_rt, desc);
             }
         }
         return;
@@ -4767,6 +4751,7 @@ static bool trans_LD_single_repl(DisasContext *s, arg_LD_single_repl *a)
 static bool trans_STZGM(DisasContext *s, arg_ldst_tag *a)
 {
     TCGv_i64 addr, tcg_rt;
+    TCGv_i32 desc;
 
     if (!dc_isar_feature(aa64_mte, s)) {
         return false;
@@ -4781,17 +4766,19 @@ static bool trans_STZGM(DisasContext *s, arg_ldst_tag *a)
 
     addr = read_cpu_reg_sp(s, a->rn, true);
     tcg_gen_addi_i64(addr, addr, a->imm);
-    tcg_rt = cpu_reg(s, a->rt);
 
+    desc = gen_mtedesc_zva(s);
     if (s->ata[0]) {
-        gen_helper_stzgm_tags(tcg_env, addr, tcg_rt, tcg_constant_i32(s->mtx));
+        tcg_rt = cpu_reg(s, a->rt);
+        gen_helper_stzgm(tcg_env, addr, tcg_rt, desc);
+    } else {
+        /*
+         * The non-tags portion of STZGM is mostly like DC_ZVA,
+         * except the alignment happens before the access.
+         */
+        tcg_gen_andi_i64(addr, addr, -s->dcz_blocksize);
+        gen_helper_dc_zva(tcg_env, addr, desc);
     }
-    /*
-     * The non-tags portion of STZGM is mostly like DC_ZVA,
-     * except the alignment happens before the access.
-     */
-    tcg_gen_andi_i64(addr, addr, -s->dcz_blocksize);
-    gen_helper_dc_zva(tcg_env, addr, gen_mtedesc_zva(s));
     return true;
 }
 
-- 
2.43.0