RISC-V: Limit the scope of TLB shootdowns

Web:        https://git.kernel.org/torvalds/c/f1b65f20fb05d1dd94656904848b96cc6df52bc0
Commit:     f1b65f20fb05d1dd94656904848b96cc6df52bc0
Parent:     fe9b842f72921fb18b93cf47a255f374289ef242
Refname:    refs/heads/master
Author:     Andrew Waterman <[email protected]>
AuthorDate: Thu Oct 26 01:53:40 2017 -0700
Committer:  Palmer Dabbelt <[email protected]>
CommitDate: Tue Jan 30 19:13:33 2018 -0800

    RISC-V: Limit the scope of TLB shootdowns
    
    RISC-V systems perform TLB shootdows via the SBI, which currently
    performs an IPI to each of the remote harts which then performs a local
    TLB flush.  This process is a bit on the slow side, but we can at least
    speed it up for some common cases by restricting the set of harts to
    shoot down to the actual set of harts that are currently participating
    in the given mm context, as opposed to the entire system.
    
    This should provide a measurable performance increase, but we haven't
    measured it.  Regardless, it seems like obviously the right thing to do
    here.
    
    Signed-off-by: Andrew Waterman <[email protected]>
    Signed-off-by: Palmer Dabbelt <[email protected]>
---
 arch/riscv/include/asm/tlbflush.h | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
index 7b9c24ebdf52..7b209aec355d 100644
--- a/arch/riscv/include/asm/tlbflush.h
+++ b/arch/riscv/include/asm/tlbflush.h
@@ -36,7 +36,14 @@ static inline void local_flush_tlb_page(unsigned long addr)
 
 #define flush_tlb_all() local_flush_tlb_all()
 #define flush_tlb_page(vma, addr) local_flush_tlb_page(addr)
-#define flush_tlb_range(vma, start, end) local_flush_tlb_all()
+
+static inline void flush_tlb_range(struct vm_area_struct *vma,
+		unsigned long start, unsigned long end)
+{
+	local_flush_tlb_all();
+}
+
+#define flush_tlb_mm(mm) flush_tlb_all()
 
 #else /* CONFIG_SMP */
 
@@ -45,16 +52,13 @@ static inline void local_flush_tlb_page(unsigned long addr)
 #define flush_tlb_all() sbi_remote_sfence_vma(0, 0, -1)
 #define flush_tlb_page(vma, addr) flush_tlb_range(vma, addr, 0)
 #define flush_tlb_range(vma, start, end) \
-	sbi_remote_sfence_vma(0, start, (end) - (start))
+	sbi_remote_sfence_vma(mm_cpumask((vma)->vm_mm)->bits, \
+			      start, (end) - (start))
+#define flush_tlb_mm(mm) \
+	sbi_remote_sfence_vma(mm_cpumask(mm)->bits, 0, -1)
 
 #endif /* CONFIG_SMP */
 
-/* Flush the TLB entries of the specified mm context */
-static inline void flush_tlb_mm(struct mm_struct *mm)
-{
-	flush_tlb_all();
-}