[PATCH 3/8] KVM: x86/pmu: Add support for hardware-switched PMU

Implement the hardware-switched mode of mediated PMU. The key design
differences from the software-switched mode are listed below.
  * Hardware handles context-switching of the guest PMU state.
  * The guest PMU state resides in a vendor save area (such as VMCB or
    VMCS) instead of struct kvm_pmu.

When enabled, the load and put functionality of mediated PMU simplifies
to just scheduling the active host events in and out.

Event filtering and instruction emulation require the ability to change
the guest PMU state in software. Since struct kvm_pmu is not guaranteed
to always have the correct state, make use of the new vendor state sync
ops to access MSR states directly from a vendor save area. These are
also used to read the latest counter value when RDPMC is intercepted
for legacy guests with fewer counters.

The software-switched mode continues to be the default for hardware
lacking the features to support this capability.

Signed-off-by: Sandipan Das <[email protected]>
---
 arch/x86/kvm/pmu.c           | 97 +++++++++++++++++++++++++++++-------
 arch/x86/kvm/pmu.h           |  7 +++
 arch/x86/kvm/svm/pmu.c       |  2 +
 arch/x86/kvm/svm/svm.c       |  3 +-
 arch/x86/kvm/vmx/pmu_intel.c |  2 +
 arch/x86/kvm/vmx/vmx.c       |  3 +-
 6 files changed, 95 insertions(+), 19 deletions(-)

diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 5ede3ee968e3..4c918aa6d6f3 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -130,6 +130,24 @@ void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops)
 #undef __KVM_X86_PMU_OP
 }
 
+static void kvm_pmu_get_vendor_state(struct kvm_vcpu *vcpu, u32 msr)
+{
+	if (!kvm_vcpu_has_mediated_pmu_caps(vcpu, KVM_MEDIATED_PMU_CAP_HW_SWITCHED))
+		return;
+
+	if (kvm_pmu_call(get_vendor_state)(vcpu, msr))
+		kvm_pmu_warn_vendor_state(msr);
+}
+
+static void kvm_pmu_set_vendor_state(struct kvm_vcpu *vcpu, u32 msr)
+{
+	if (!kvm_vcpu_has_mediated_pmu_caps(vcpu, KVM_MEDIATED_PMU_CAP_HW_SWITCHED))
+		return;
+
+	if (kvm_pmu_call(set_vendor_state)(vcpu, msr))
+		kvm_pmu_warn_vendor_state(msr);
+}
+
 void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops)
 {
 	bool is_intel = boot_cpu_data.x86_vendor == X86_VENDOR_INTEL;
@@ -200,6 +218,27 @@ void kvm_handle_guest_mediated_pmi(void)
 	kvm_make_request(KVM_REQ_PMI, vcpu);
 }
 
+static __always_inline u32 fixed_counter_msr(u32 idx)
+{
+	return kvm_pmu_ops.FIXED_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE;
+}
+
+static __always_inline u32 gp_counter_msr(u32 idx)
+{
+	return kvm_pmu_ops.GP_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE;
+}
+
+static __always_inline u32 gp_eventsel_msr(u32 idx)
+{
+	return kvm_pmu_ops.GP_EVENTSEL_BASE + idx * kvm_pmu_ops.MSR_STRIDE;
+}
+
+static __always_inline u32 pmc_counter_msr(struct kvm_pmc *pmc)
+{
+	return pmc_is_gp(pmc) ? gp_counter_msr(pmc->idx) :
+				fixed_counter_msr(pmc->idx - KVM_FIXED_PMC_BASE_IDX);
+}
+
 static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi)
 {
 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
@@ -553,18 +592,23 @@ static void kvm_mediated_pmu_refresh_event_filter(struct kvm_pmc *pmc)
 {
 	bool allowed = pmc_is_locally_enabled(pmc) && pmc_is_event_allowed(pmc);
 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
+	struct kvm_vcpu *vcpu = pmc->vcpu;
 
 	if (pmc_is_gp(pmc)) {
 		pmc->eventsel_hw &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
 		if (allowed)
 			pmc->eventsel_hw |= pmc->eventsel &
 					    ARCH_PERFMON_EVENTSEL_ENABLE;
+
+		kvm_pmu_set_vendor_state(vcpu, gp_eventsel_msr(pmc->idx));
 	} else {
 		u64 mask = intel_fixed_bits_by_idx(pmc->idx - KVM_FIXED_PMC_BASE_IDX, 0xf);
 
 		pmu->fixed_ctr_ctrl_hw &= ~mask;
 		if (allowed)
 			pmu->fixed_ctr_ctrl_hw |= pmu->fixed_ctr_ctrl & mask;
+
+		kvm_pmu_set_vendor_state(vcpu, kvm_pmu_ops.FIXED_COUNTER_CTRL);
 	}
 }
 
@@ -772,6 +816,8 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
 	    kvm_is_cr0_bit_set(vcpu, X86_CR0_PE))
 		return 1;
 
+	kvm_pmu_get_vendor_state(vcpu, pmc_counter_msr(pmc));
+
 	*data = pmc_read_counter(pmc) & mask;
 	return 0;
 }
@@ -855,10 +901,12 @@ int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	switch (msr) {
 	case MSR_CORE_PERF_GLOBAL_STATUS:
 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
+		kvm_pmu_get_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_STATUS);
 		msr_info->data = pmu->global_status;
 		break;
 	case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
 	case MSR_CORE_PERF_GLOBAL_CTRL:
+		kvm_pmu_get_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_CTRL);
 		msr_info->data = pmu->global_ctrl;
 		break;
 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
@@ -898,6 +946,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			return 1;
 
 		pmu->global_status = data;
+		kvm_pmu_set_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_STATUS);
 		break;
 	case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
 		data &= ~pmu->global_ctrl_rsvd;
@@ -917,6 +966,8 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		 */
 		if (kvm_vcpu_has_mediated_pmu(vcpu))
 			kvm_pmu_call(write_global_ctrl)(data);
+
+		kvm_pmu_set_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_CTRL);
 		break;
 	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
 		/*
@@ -927,12 +978,18 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			return 1;
 		fallthrough;
 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
-		if (!msr_info->host_initiated)
+		if (!msr_info->host_initiated) {
+			kvm_pmu_get_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_STATUS);
 			pmu->global_status &= ~data;
+			kvm_pmu_set_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_STATUS);
+		}
 		break;
 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET:
-		if (!msr_info->host_initiated)
+		if (!msr_info->host_initiated) {
+			kvm_pmu_get_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_STATUS);
 			pmu->global_status |= data & ~pmu->global_status_rsvd;
+			kvm_pmu_set_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_STATUS);
+		}
 		break;
 	default:
 		kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
@@ -1020,6 +1077,8 @@ void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
 	if (kvm_vcpu_has_mediated_pmu(vcpu))
 		kvm_pmu_call(write_global_ctrl)(pmu->global_ctrl);
 
+	kvm_pmu_set_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_CTRL);
+
 	bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters);
 	bitmap_set(pmu->all_valid_pmc_idx, KVM_FIXED_PMC_BASE_IDX,
 		   pmu->nr_arch_fixed_counters);
@@ -1142,6 +1201,9 @@ static void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu,
 	if (bitmap_empty(event_pmcs, X86_PMC_IDX_MAX))
 		return;
 
+	kvm_pmu_get_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_CTRL);
+	kvm_pmu_get_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_STATUS);
+
 	if (!kvm_pmu_has_perf_global_ctrl(pmu))
 		bitmap_copy(bitmap, event_pmcs, X86_PMC_IDX_MAX);
 	else if (!bitmap_and(bitmap, event_pmcs,
@@ -1150,11 +1212,17 @@ static void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu,
 
 	idx = srcu_read_lock(&vcpu->kvm->srcu);
 	kvm_for_each_pmc(pmu, pmc, i, bitmap) {
+		kvm_pmu_get_vendor_state(vcpu, pmc_counter_msr(pmc));
+
 		if (!pmc_is_event_allowed(pmc) || !cpl_is_matched(pmc))
 			continue;
 
 		kvm_pmu_incr_counter(pmc);
+		kvm_pmu_set_vendor_state(vcpu, pmc_counter_msr(pmc));
 	}
+
+	kvm_pmu_set_vendor_state(vcpu, kvm_pmu_ops.PERF_GLOBAL_STATUS);
+
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 }
 
@@ -1313,21 +1381,6 @@ int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp)
 	return r;
 }
 
-static __always_inline u32 fixed_counter_msr(u32 idx)
-{
-	return kvm_pmu_ops.FIXED_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE;
-}
-
-static __always_inline u32 gp_counter_msr(u32 idx)
-{
-	return kvm_pmu_ops.GP_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE;
-}
-
-static __always_inline u32 gp_eventsel_msr(u32 idx)
-{
-	return kvm_pmu_ops.GP_EVENTSEL_BASE + idx * kvm_pmu_ops.MSR_STRIDE;
-}
-
 static void kvm_pmu_load_guest_pmcs(struct kvm_vcpu *vcpu)
 {
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
@@ -1364,6 +1417,10 @@ void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu)
 
 	perf_load_guest_context();
 
+	/* Guest PMU state is restored by hardware */
+	if (kvm_vcpu_has_mediated_pmu_caps(vcpu, KVM_MEDIATED_PMU_CAP_HW_SWITCHED))
+		return;
+
 	/*
 	 * Explicitly clear PERF_GLOBAL_CTRL, as "loading" the guest's context
 	 * disables all individual counters (if any were enabled), but doesn't
@@ -1423,6 +1480,12 @@ void kvm_mediated_pmu_put(struct kvm_vcpu *vcpu)
 
 	lockdep_assert_irqs_disabled();
 
+	/* Guest PMU state is saved by hardware */
+	if (kvm_vcpu_has_mediated_pmu_caps(vcpu, KVM_MEDIATED_PMU_CAP_HW_SWITCHED)) {
+		perf_put_guest_context();
+		return;
+	}
+
 	/*
 	 * Defer handling of PERF_GLOBAL_CTRL to vendor code.  On Intel, it's
 	 * atomically cleared on VM-Exit, i.e. doesn't need to be clear here.
diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
index 5d6f6863ad40..42d33072ece9 100644
--- a/arch/x86/kvm/pmu.h
+++ b/arch/x86/kvm/pmu.h
@@ -50,8 +50,10 @@ struct kvm_pmu_ops {
 	const int MIN_NR_GP_COUNTERS;
 
 	const u32 PERF_GLOBAL_CTRL;
+	const u32 PERF_GLOBAL_STATUS;
 	const u32 GP_EVENTSEL_BASE;
 	const u32 GP_COUNTER_BASE;
+	const u32 FIXED_COUNTER_CTRL;
 	const u32 FIXED_COUNTER_BASE;
 	const u32 MSR_STRIDE;
 };
@@ -110,6 +112,11 @@ do {									\
 	(kvm)->arch.mediated_pmu_caps &= ~(caps);			\
 } while (0)
 
+static inline void kvm_pmu_warn_vendor_state(u32 msr)
+{
+	WARN_ONCE(1, "accessing unsupported vendor save slot for MSR 0x%x\n", msr);
+}
+
 /*
  * KVM tracks all counters in 64-bit bitmaps, with general purpose counters
  * mapped to bits 31:0 and fixed counters mapped to 63:32, e.g. fixed counter 0
diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c
index c18286545a7a..5dccf8776368 100644
--- a/arch/x86/kvm/svm/pmu.c
+++ b/arch/x86/kvm/svm/pmu.c
@@ -321,8 +321,10 @@ struct kvm_pmu_ops amd_pmu_ops __initdata = {
 	.MIN_NR_GP_COUNTERS = AMD64_NUM_COUNTERS,
 
 	.PERF_GLOBAL_CTRL = MSR_AMD64_PERF_CNTR_GLOBAL_CTL,
+	.PERF_GLOBAL_STATUS = MSR_AMD64_PERF_CNTR_GLOBAL_STATUS,
 	.GP_EVENTSEL_BASE = MSR_F15H_PERF_CTL0,
 	.GP_COUNTER_BASE = MSR_F15H_PERF_CTR0,
+	.FIXED_COUNTER_CTRL = 0,
 	.FIXED_COUNTER_BASE = 0,
 	.MSR_STRIDE = 2,
 };
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 9d607b98bd06..2e39b5e752cd 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -4618,7 +4618,8 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 
 	kvm_clear_available_registers(vcpu, SVM_REGS_LAZY_LOAD_SET);
 
-	if (!msr_write_intercepted(svm, MSR_AMD64_PERF_CNTR_GLOBAL_CTL))
+	if (!kvm_vcpu_has_mediated_pmu_caps(vcpu, KVM_MEDIATED_PMU_CAP_HW_SWITCHED) &&
+	    !msr_write_intercepted(svm, MSR_AMD64_PERF_CNTR_GLOBAL_CTL))
 		rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, vcpu_to_pmu(vcpu)->global_ctrl);
 
 	trace_kvm_exit(vcpu, KVM_ISA_SVM);
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index bfa8612fb450..d0723814410e 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -853,8 +853,10 @@ struct kvm_pmu_ops intel_pmu_ops __initdata = {
 	.MIN_NR_GP_COUNTERS = 1,
 
 	.PERF_GLOBAL_CTRL = MSR_CORE_PERF_GLOBAL_CTRL,
+	.PERF_GLOBAL_STATUS = MSR_CORE_PERF_GLOBAL_STATUS,
 	.GP_EVENTSEL_BASE = MSR_P6_EVNTSEL0,
 	.GP_COUNTER_BASE = MSR_IA32_PMC0,
+	.FIXED_COUNTER_CTRL = MSR_CORE_PERF_FIXED_CTR_CTRL,
 	.FIXED_COUNTER_BASE = MSR_CORE_PERF_FIXED_CTR0,
 	.MSR_STRIDE = 1,
 };
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index e3bfe6aca1a0..7d8c032c1897 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -7383,7 +7383,8 @@ static void vmx_refresh_guest_perf_global_control(struct kvm_vcpu *vcpu)
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 
-	if (msr_write_intercepted(vmx, MSR_CORE_PERF_GLOBAL_CTRL))
+	if (kvm_vcpu_has_mediated_pmu_caps(vcpu, KVM_MEDIATED_PMU_CAP_HW_SWITCHED) ||
+	    msr_write_intercepted(vmx, MSR_CORE_PERF_GLOBAL_CTRL))
 		return;
 
 	if (!cpu_has_save_perf_global_ctrl()) {
-- 
2.53.0