[XEN][RESEND][PATCH 2/3] xen/arm: optimize stage-1,2 combined TLBI in presence of FEAT_nTLBPA

From: Haseeb Ashraf <[email protected]>

FEAT_nTLBPA guarantees that intermediate caching of translation table
walks does not retain non-coherent cached copies of previously valid
translation table entries since the last completed applicable TLBI.

When FEAT_nTLBPA is present, a stage-2 invalidation no longer needs to
be paired with an additional stage-1 invalidation solely to discard
such side effects. Use this to avoid redundant stage-1 invalidation in
the combined guest TLBI paths.

Update the stage-1-only guest TLBI helpers so that they become no-ops
when FEAT_nTLBPA is present. This allows existing call sites that
conservatively issue stage-1 invalidation after stage-2 invalidation to
benefit without further restructuring.

Also update the Arm64 IPA-based invalidation helper so that:
  - stage-2 invalidation by IPA is performed explicitly,
  - stage-1 invalidation is issued separately only when FEAT_nTLBPA is
    absent,
  - completion of the broadcast TLBI sequence remains explicit.

This makes the architectural intent clearer and avoids redundant
invalidations on systems that implement FEAT_nTLBPA.

Suggested-by: Mohamed Mediouni <[email protected]>
Signed-off-by: Haseeb Ashraf <[email protected]>

Changes in v4:
- Reworked the Arm64 IPA-based invalidation helper so that stage-2,
  optional stage-1, and final TLBI completion are expressed
  separately and explicitly.
- Simplified the FEAT_nTLBPA optimization by reusing the explicit TLBI
  completion helper instead of open-coding a special final IPA TLBI
  sequence.
- Refined comments and commit message wording.

Changes in v3:
- This commit has no functional change in v3, only rebasing changes
  due to updates in commit-1.

Changes in v2:
- This commit is implemented in v2 and is splitted from commit-1 in
  v1. This is implemented by using CPU capability.
---
 xen/arch/arm/cpufeature.c                 | 19 ++++++++++
 xen/arch/arm/include/asm/arm32/flushtlb.h | 14 ++++----
 xen/arch/arm/include/asm/arm64/flushtlb.h | 44 +++++++++++++++++++----
 xen/arch/arm/include/asm/cpufeature.h     | 24 +++++++++++--
 xen/arch/arm/include/asm/processor.h      |  7 ++++
 5 files changed, 93 insertions(+), 15 deletions(-)

diff --git a/xen/arch/arm/cpufeature.c b/xen/arch/arm/cpufeature.c
index 94d14fb6a9..86720c10a3 100644
--- a/xen/arch/arm/cpufeature.c
+++ b/xen/arch/arm/cpufeature.c
@@ -17,7 +17,19 @@ DECLARE_BITMAP(cpu_hwcaps, ARM_NCAPS);
 
 struct cpuinfo_arm __read_mostly domain_cpuinfo;
 
+#ifdef CONFIG_ARM_32
+static bool has_ntlbpa(const struct arm_cpu_capabilities *entry)
+{
+    return system_cpuinfo.mm32.ntlbpa == MM32_NTLBPA_SUPPORT_IMP;
+}
+#endif
+
 #ifdef CONFIG_ARM_64
+static bool has_ntlbpa(const struct arm_cpu_capabilities *entry)
+{
+    return system_cpuinfo.mm64.ntlbpa == MM64_NTLBPA_SUPPORT_IMP;
+}
+
 static bool has_sb_instruction(const struct arm_cpu_capabilities *entry)
 {
     return system_cpuinfo.isa64.sb;
@@ -25,6 +37,13 @@ static bool has_sb_instruction(const struct arm_cpu_capabilities *entry)
 #endif
 
 static const struct arm_cpu_capabilities arm_features[] = {
+#if defined(CONFIG_ARM_32) || defined(CONFIG_ARM_64)
+    {
+        .desc = "Intermediate caching of translation table walks (nTLBPA)",
+        .capability = ARM_HAS_NTLBPA,
+        .matches = has_ntlbpa,
+    },
+#endif
 #ifdef CONFIG_ARM_64
     {
         .desc = "Speculation barrier instruction (SB)",
diff --git a/xen/arch/arm/include/asm/arm32/flushtlb.h b/xen/arch/arm/include/asm/arm32/flushtlb.h
index 1497f16ba5..30df161323 100644
--- a/xen/arch/arm/include/asm/arm32/flushtlb.h
+++ b/xen/arch/arm/include/asm/arm32/flushtlb.h
@@ -49,8 +49,8 @@ TLB_HELPER(flush_xen_tlb_local, TLBIALLH, nsh)
  * Flush TLB of local processor. Use when flush for only stage-1 is intended.
  *
  * The following function should be used where intention is to clear only
- * stage-1 TLBs. This would be helpful in future in identifying which stage-1
- * TLB flushes can be skipped such as in present of FEAT_nTLBPA.
+ * stage-1 TLBs. This would be helpful in identifying which stage-1 TLB flushes
+ * can be skipped such as in present of FEAT_nTLBPA.
  */
 static inline void flush_guest_tlb_s1_local(void)
 {
@@ -60,7 +60,8 @@ static inline void flush_guest_tlb_s1_local(void)
      *
      * See ARMv8 (DDI 0487L.b): G5-11698 Table G5-23.
      */
-    return flush_guest_tlb_local();
+    if ( !cpus_have_const_cap(ARM_HAS_NTLBPA) )
+        flush_guest_tlb_local();
 }
 
 /*
@@ -68,8 +69,8 @@ static inline void flush_guest_tlb_s1_local(void)
  * stage-1 is intended.
  *
  * The following function should be used where intention is to clear only
- * stage-1 TLBs. This would be helpful in future in identifying which stage-1
- * TLB flushes can be skipped such as in present of FEAT_nTLBPA.
+ * stage-1 TLBs. This would be helpful in identifying which stage-1 TLB flushes
+ * can be skipped such as in present of FEAT_nTLBPA.
  */
 static inline void flush_guest_tlb_s1(void)
 {
@@ -79,7 +80,8 @@ static inline void flush_guest_tlb_s1(void)
      *
      * See ARMv8 (DDI 0487L.b): G5-11698 Table G5-23.
      */
-    return flush_guest_tlb();
+    if ( !cpus_have_const_cap(ARM_HAS_NTLBPA) )
+        flush_guest_tlb();
 }
 
 /* Flush TLB of local processor for address va. */
diff --git a/xen/arch/arm/include/asm/arm64/flushtlb.h b/xen/arch/arm/include/asm/arm64/flushtlb.h
index cbe2a0da57..de266b0e11 100644
--- a/xen/arch/arm/include/asm/arm64/flushtlb.h
+++ b/xen/arch/arm/include/asm/arm64/flushtlb.h
@@ -63,6 +63,36 @@ static inline void name(void)                         \
         : : : "memory"); \
 }
 
+#define TLB_HELPER_NTLBPA_LOCAL(name, tlbop)     \
+static inline void name(void)                    \
+{                                                \
+    if ( !cpus_have_const_cap(ARM_HAS_NTLBPA) )  \
+        asm_inline volatile (                    \
+            "dsb  nshst;"                        \
+            "tlbi "  # tlbop  ";"                \
+            "dsb  nsh;"                          \
+            "isb;"                               \
+            : : : "memory");                     \
+}
+
+#define TLB_HELPER_NTLBPA(name, tlbop)                \
+static inline void name(void)                         \
+{                                                     \
+    if ( !cpus_have_const_cap(ARM_HAS_NTLBPA) )       \
+        asm_inline volatile (                         \
+            "dsb  ishst;"                             \
+            "tlbi "  # tlbop  ";"                     \
+            ALTERNATIVE(                              \
+                "nop; nop;",                          \
+                "dsb  ish;"                           \
+                "tlbi vale2is, xzr;",                 \
+                ARM64_WORKAROUND_REPEAT_TLBI,         \
+                CONFIG_ARM64_WORKAROUND_REPEAT_TLBI)  \
+            "dsb  ish;"                               \
+            "isb;"                                    \
+            : : : "memory");                          \
+}
+
 /* Flush local TLBs, current VMID only. */
 TLB_HELPER_LOCAL(flush_guest_tlb_local, vmalls12e1)
 
@@ -70,10 +100,10 @@ TLB_HELPER_LOCAL(flush_guest_tlb_local, vmalls12e1)
 TLB_HELPER(flush_guest_tlb, vmalls12e1is)
 
 /* Flush local TLBs, current VMID, stage-1 only */
-TLB_HELPER(flush_guest_tlb_s1_local, vmalle1, nsh)
+TLB_HELPER_NTLBPA_LOCAL(flush_guest_tlb_s1_local, vmalle1)
 
 /* Flush innershareable TLBs, current VMID, stage-1 only */
-TLB_HELPER(flush_guest_tlb_s1, vmalle1is, ish)
+TLB_HELPER_NTLBPA(flush_guest_tlb_s1, vmalle1is)
 
 /* Flush local TLBs, all VMIDs, non-hypervisor mode */
 TLB_HELPER_LOCAL(flush_all_guests_tlb_local, alle1)
@@ -86,6 +116,8 @@ TLB_HELPER_LOCAL(flush_xen_tlb_local, alle2)
 
 #undef TLB_HELPER_LOCAL
 #undef TLB_HELPER
+#undef TLB_HELPER_NTLBPA_LOCAL
+#undef TLB_HELPER_NTLBPA
 
 /*
  * FLush TLB by VA. This will likely be used in a loop, so the caller
@@ -169,11 +201,11 @@ static inline void flush_guest_tlb_range_ipa(paddr_t ipa, unsigned long size)
     }
 
     /*
-     * Invalidate stage-1 entries for the current VMID as well. Keep this
-     * separate from completion of the broadcast TLBI sequence so that the
-     * final barriers/workaround are explicit.
+     * Without FEAT_nTLBPA, invalidate corresponding stage-1 side effects
+     * separately. Completion of the broadcast TLBI sequence remains explicit.
      */
-    asm_inline volatile("tlbi vmalle1is" : : : "memory");
+    if ( !cpus_have_const_cap(ARM_HAS_NTLBPA) )
+        asm_inline volatile("tlbi vmalle1is" : : : "memory");
 
     tlbi_complete_broadcast();
 }
diff --git a/xen/arch/arm/include/asm/cpufeature.h b/xen/arch/arm/include/asm/cpufeature.h
index bf902a3970..3e56dd27ef 100644
--- a/xen/arch/arm/include/asm/cpufeature.h
+++ b/xen/arch/arm/include/asm/cpufeature.h
@@ -76,8 +76,9 @@
 #define ARM_WORKAROUND_BHB_SMCC_3 15
 #define ARM_HAS_SB 16
 #define ARM64_WORKAROUND_1508412 17
+#define ARM_HAS_NTLBPA 18
 
-#define ARM_NCAPS           18
+#define ARM_NCAPS           19
 
 #ifndef __ASSEMBLER__
 
@@ -276,7 +277,8 @@ struct cpuinfo_arm {
             unsigned long ets:4;
             unsigned long __res2:4;
             unsigned long afp:4;
-            unsigned long __res3:12;
+            unsigned long ntlbpa:4;
+            unsigned long __res3:8;
             unsigned long ecbhb:4;
 
             /* MMFR2 */
@@ -437,8 +439,24 @@ struct cpuinfo_arm {
         register_t bits[1];
     } aux32;
 
-    struct {
+    union {
         register_t bits[6];
+        struct {
+            /* MMFR0 */
+            unsigned long __res0:32;
+            /* MMFR1 */
+            unsigned long __res1:32;
+            /* MMFR2 */
+            unsigned long __res2:32;
+            /* MMFR3 */
+            unsigned long __res3:32;
+            /* MMFR4 */
+            unsigned long __res4:32;
+            /* MMFR5 */
+            unsigned long __res5:4;
+            unsigned long ntlbpa:4;
+            unsigned long __res6:24;
+        };
     } mm32;
 
     struct {
diff --git a/xen/arch/arm/include/asm/processor.h b/xen/arch/arm/include/asm/processor.h
index a3753c317f..dbf9471a7a 100644
--- a/xen/arch/arm/include/asm/processor.h
+++ b/xen/arch/arm/include/asm/processor.h
@@ -484,9 +484,16 @@
 /* FSR long format */
 #define FSRL_STATUS_DEBUG       (_AC(0x22,UL)<<0)
 
+#ifdef CONFIG_ARM_32
+#define MM32_NTLBPA_SUPPORT_NI      0x0
+#define MM32_NTLBPA_SUPPORT_IMP     0x1
+#endif
+
 #ifdef CONFIG_ARM_64
 #define MM64_VMID_8_BITS_SUPPORT    0x0
 #define MM64_VMID_16_BITS_SUPPORT   0x2
+#define MM64_NTLBPA_SUPPORT_NI      0x0
+#define MM64_NTLBPA_SUPPORT_IMP     0x1
 #endif
 
 #define MM64_MSA_PMSA_SUPPORT       0xf
-- 
2.43.0