KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by always inlining iterator helper methods

Web:        https://git.kernel.org/torvalds/c/928a4c39484281f8ca366f53a1db79330d058401
Commit:     928a4c39484281f8ca366f53a1db79330d058401
Parent:     f208820a321f9b23d77d7eed89945d862d62a3ed
Refname:    refs/heads/master
Author:     David Woodhouse <[email protected]>
AuthorDate: Sat Feb 10 23:39:24 2018 +0000
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue Feb 13 08:59:45 2018 +0100

    KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by always inlining iterator helper methods
    
    With retpoline, tight loops of "call this function for every XXX" are
    very much pessimised by taking a prediction miss *every* time. This one
    is by far the biggest contributor to the guest launch time with retpoline.
    
    By marking the iterator slot_handle_…() functions always_inline, we can
    ensure that the indirect function call can be optimised away into a
    direct call and it actually generates slightly smaller code because
    some of the other conditionals can get optimised away too.
    
    Performance is now pretty close to what we see with nospectre_v2 on
    the command line.
    
    Suggested-by: Linus Torvalds <[email protected]>
    Tested-by: Filippo Sironi <[email protected]>
    Signed-off-by: David Woodhouse <[email protected]>
    Reviewed-by: Filippo Sironi <[email protected]>
    Acked-by: Paolo Bonzini <[email protected]>
    Cc: Andy Lutomirski <[email protected]>
    Cc: Arjan van de Ven <[email protected]>
    Cc: Borislav Petkov <[email protected]>
    Cc: Dan Williams <[email protected]>
    Cc: Dave Hansen <[email protected]>
    Cc: David Woodhouse <[email protected]>
    Cc: Greg Kroah-Hartman <[email protected]>
    Cc: Josh Poimboeuf <[email protected]>
    Cc: Peter Zijlstra <[email protected]>
    Cc: Thomas Gleixner <[email protected]>
    Cc: [email protected]
    Cc: [email protected]
    Cc: [email protected]
    Cc: [email protected]
    Cc: [email protected]
    Cc: [email protected]
    Link: http://lkml.kernel.org/r/[email protected]
    Signed-off-by: Ingo Molnar <[email protected]>
---
 arch/x86/kvm/mmu.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 2b8eb4da4d08..cc83bdcb65d1 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -5058,7 +5058,7 @@ void kvm_mmu_uninit_vm(struct kvm *kvm)
 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head);
 
 /* The caller should hold mmu-lock before calling this function. */
-static bool
+static __always_inline bool
 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
 			slot_level_handler fn, int start_level, int end_level,
 			gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
@@ -5088,7 +5088,7 @@ slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
 	return flush;
 }
 
-static bool
+static __always_inline bool
 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 		  slot_level_handler fn, int start_level, int end_level,
 		  bool lock_flush_tlb)
@@ -5099,7 +5099,7 @@ slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 			lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 		      slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5107,7 +5107,7 @@ slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 				 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 			slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5115,7 +5115,7 @@ slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 				 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
 		 slot_level_handler fn, bool lock_flush_tlb)
 {
--
To unsubscribe from this list: send the line "unsubscribe git-commits-head" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html