[PATCH v3 9/9] KVM: selftests: Add nested SVM DecodeAssists test

Add a nested SVM selftest for DecodeAssists.  Verify that KVM exposes
the feature to L1 and provides architectural exit state for MOV CR/DR,
CLTS, LMSW, SMSW, INTn, INVLPG, and INVLPGA intercepts.

For data #NPF and intercepted #PF exits, cover instruction bytes from
hardware, the emulator fetch cache, and on-demand fetching from L2 RIP.
Verify that instruction-fetch #PF reports no bytes, that fetching stops
at an unreadable page or non-canonical address, and that 32-bit linear
addresses wrap correctly.

Add regressions for a synthesized #NPF that follows a hardware #NPF in
the same emulated instruction, and for a userspace-injected #PF while an
emulated MMIO read is awaiting completion.

The synthesized OUTSB #NPF and userspace-injected #PF paths run by
default.  The remaining synthesized paths are covered when
kvm.force_emulation_prefix=1 is enabled.

Signed-off-by: Tina Zhang <[email protected]>
---
 tools/testing/selftests/kvm/Makefile.kvm      |   1 +
 .../selftests/kvm/include/x86/processor.h     |   1 +
 .../kvm/x86/svm_nested_decode_assists_test.c  | 791 ++++++++++++++++++
 3 files changed, 793 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/x86/svm_nested_decode_assists_test.c

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index 88c6c8046dde..e31e3a2412ae 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -116,6 +116,7 @@ TEST_GEN_PROGS_x86 += x86/vmx_preemption_timer_test
 TEST_GEN_PROGS_x86 += x86/svm_vmcall_test
 TEST_GEN_PROGS_x86 += x86/svm_int_ctl_test
 TEST_GEN_PROGS_x86 += x86/svm_nested_clear_efer_svme
+TEST_GEN_PROGS_x86 += x86/svm_nested_decode_assists_test
 TEST_GEN_PROGS_x86 += x86/svm_nested_shutdown_test
 TEST_GEN_PROGS_x86 += x86/svm_nested_soft_inject_test
 TEST_GEN_PROGS_x86 += x86/svm_nested_vmcb12_gpa
diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h
index b161174ece45..42e009af525e 100644
--- a/tools/testing/selftests/kvm/include/x86/processor.h
+++ b/tools/testing/selftests/kvm/include/x86/processor.h
@@ -219,6 +219,7 @@ struct kvm_x86_cpu_feature {
 #define	X86_FEATURE_LBRV		KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1)
 #define	X86_FEATURE_NRIPS		KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3)
 #define X86_FEATURE_TSCRATEMSR          KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 4)
+#define X86_FEATURE_DECODEASSISTS       KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 7)
 #define X86_FEATURE_PAUSEFILTER         KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 10)
 #define X86_FEATURE_PFTHRESHOLD         KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 12)
 #define	X86_FEATURE_V_VMSAVE_VMLOAD	KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 15)
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_decode_assists_test.c b/tools/testing/selftests/kvm/x86/svm_nested_decode_assists_test.c
new file mode 100644
index 000000000000..13511ca8473b
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86/svm_nested_decode_assists_test.c
@@ -0,0 +1,791 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Test KVM's virtualization of SVM DecodeAssists for nested guests.
+ */
+
+#include "test_util.h"
+#include "kvm_util.h"
+#include "processor.h"
+#include "svm_util.h"
+
+#define TEST_INT_VECTOR 0x81
+
+/* Any canonical virtual address that is never mapped by the selftest VM. */
+#define PF_TEST_GVA BIT_ULL(40)
+#define PF_FETCH_TEST_GVA BIT_ULL(41)
+
+#define OUTSB_OPCODE 0x6e
+#define BOUNDARY_OUTSB_CODE_SIZE 15
+#define LINEAR_WRAP_CODE_GVA (BIT_ULL(32) - PAGE_SIZE)
+#define LINEAR_WRAP_OUTSB_OFFSET (PAGE_SIZE - 8)
+#define LINEAR_WRAP_SETUP_SIZE 9
+#define LINEAR_WRAP_SETUP_OFFSET \
+	(LINEAR_WRAP_OUTSB_OFFSET - LINEAR_WRAP_SETUP_SIZE)
+#define LINEAR_WRAP_CS_BASE PAGE_SIZE
+#define TEST_IOPM_SIZE (3 * PAGE_SIZE)
+
+static u8 npf_target[PAGE_SIZE] __aligned(PAGE_SIZE);
+static u8 mmio_source __aligned(PAGE_SIZE);
+static u8 boundary_outsb_code[2 * PAGE_SIZE] __aligned(PAGE_SIZE);
+static const u8 linear_wrap_insn_bytes[15] = {
+	OUTSB_OPCODE, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+	0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee,
+};
+
+static void l2_read_code(void)
+{
+	asm volatile("mov (%0), %%rax" : : "r"(&npf_target) : "rax", "memory");
+	GUEST_FAIL("L2 read did not cause a nested page fault");
+}
+
+static void l2_outsb_code(void)
+{
+	asm volatile("mov %0, %%rsi\n\t"
+		     "mov $0x80, %%dx\n\t"
+		     "outsb"
+		     : : "r"(&npf_target) : "rsi", "rdx", "memory");
+	GUEST_FAIL("L2 OUTSB did not cause a nested page fault");
+}
+
+static void l2_movsb_code(void)
+{
+	asm volatile("mov %0, %%rsi\n\t"
+		     "mov %1, %%rdi\n\t"
+		     "movsb"
+		     : : "r"(&mmio_source), "r"(&npf_target)
+		     : "rsi", "rdi", "memory");
+	GUEST_FAIL("L2 MOVSB did not cause a nested page fault");
+}
+
+static void l2_pf_code(void)
+{
+	asm volatile("mov (%0), %%rax"
+		     : : "r"(PF_TEST_GVA) : "rax", "memory");
+	GUEST_FAIL("L2 access to an unmapped VA did not #PF");
+}
+
+static void l2_fep_pf_code(void)
+{
+	asm volatile(KVM_FEP "mov (%0), %%rax"
+		     : : "r"(PF_TEST_GVA) : "rax", "memory");
+	GUEST_FAIL("L2 forced-emulated access to an unmapped VA did not #PF");
+}
+
+static void l2_stale_emulator_pf_code(void)
+{
+	asm volatile("movb (%0), %%al\n\t"
+		     "nop"
+		     : : "r"(&mmio_source) : "rax", "memory");
+	GUEST_FAIL("Userspace-injected #PF was not intercepted by L1");
+}
+
+static void l2_mov_from_cr4_code(void)
+{
+	asm volatile("mov %%cr4, %%r10" : : : "r10");
+	GUEST_FAIL("L2 MOV-from-CR4 was not intercepted");
+}
+
+static void l2_fep_mov_from_cr4_code(void)
+{
+	asm volatile(KVM_FEP "mov %%cr4, %%r10" : : : "r10");
+	GUEST_FAIL("L2 forced-emulated MOV-from-CR4 was not intercepted");
+}
+
+static void l2_mov_to_cr4_code(void)
+{
+	asm volatile("mov %%cr4, %%rax\n\t"
+		     "mov %%rax, %%cr4" : : : "rax");
+	GUEST_FAIL("L2 MOV-to-CR4 was not intercepted");
+}
+
+static void l2_fep_mov_to_cr4_code(void)
+{
+	asm volatile("mov %%cr4, %%rax\n\t"
+		     KVM_FEP "mov %%rax, %%cr4" : : : "rax");
+	GUEST_FAIL("L2 forced-emulated MOV-to-CR4 was not intercepted");
+}
+
+static void l2_mov_to_dr7_code(void)
+{
+	asm volatile("mov %%dr7, %%rax\n\t"
+		     "mov %%rax, %%rbx\n\t"
+		     "mov %%rbx, %%dr7" : : : "rax", "rbx");
+	GUEST_FAIL("L2 MOV-to-DR7 was not intercepted");
+}
+
+static void l2_fep_mov_to_dr7_code(void)
+{
+	asm volatile("mov %%dr7, %%rax\n\t"
+		     "mov %%rax, %%rbx\n\t"
+		     KVM_FEP "mov %%rbx, %%dr7" : : : "rax", "rbx");
+	GUEST_FAIL("L2 forced-emulated MOV-to-DR7 was not intercepted");
+}
+
+static void l2_mov_from_dr7_code(void)
+{
+	asm volatile("mov %%dr7, %%r10" : : : "r10");
+	GUEST_FAIL("L2 MOV-from-DR7 was not intercepted");
+}
+
+static void l2_fep_mov_from_dr7_code(void)
+{
+	asm volatile(KVM_FEP "mov %%dr7, %%r10" : : : "r10");
+	GUEST_FAIL("L2 forced-emulated MOV-from-DR7 was not intercepted");
+}
+
+static void l2_clts_code(void)
+{
+	asm volatile("clts" : : : "memory");
+	GUEST_FAIL("L2 CLTS was not intercepted");
+}
+
+static void l2_fep_clts_code(void)
+{
+	asm volatile(KVM_FEP "clts" : : : "memory");
+	GUEST_FAIL("L2 forced-emulated CLTS was not intercepted");
+}
+
+static void l2_lmsw_code(void)
+{
+	asm volatile("smsw %%ax\n\t"
+		     "lmsw %%ax" : : : "rax", "memory");
+	GUEST_FAIL("L2 LMSW was not intercepted");
+}
+
+static void l2_fep_lmsw_code(void)
+{
+	asm volatile("smsw %%ax\n\t"
+		     KVM_FEP "lmsw %%ax" : : : "rax", "memory");
+	GUEST_FAIL("L2 forced-emulated LMSW was not intercepted");
+}
+
+static void l2_smsw_code(void)
+{
+	asm volatile("smsw %%ax" : : : "rax", "memory");
+	GUEST_FAIL("L2 SMSW was not intercepted");
+}
+
+static void l2_fep_smsw_code(void)
+{
+	asm volatile(KVM_FEP "smsw %%ax" : : : "rax", "memory");
+	GUEST_FAIL("L2 forced-emulated SMSW was not intercepted");
+}
+
+static void l2_int_code(void)
+{
+	asm volatile("int %0" : : "i"(TEST_INT_VECTOR));
+	GUEST_FAIL("L2 INTn was not intercepted");
+}
+
+static void l2_fep_int_code(void)
+{
+	asm volatile(KVM_FEP "int %0" : : "i"(TEST_INT_VECTOR));
+	GUEST_FAIL("L2 forced-emulated INTn was not intercepted");
+}
+
+static void l2_invlpg_code(void)
+{
+	asm volatile("invlpg (%0)" : : "r"(&npf_target) : "memory");
+	GUEST_FAIL("L2 INVLPG was not intercepted");
+}
+
+static void l2_fep_invlpg_code(void)
+{
+	asm volatile(KVM_FEP "invlpg (%0)" : : "r"(&npf_target) : "memory");
+	GUEST_FAIL("L2 forced-emulated INVLPG was not intercepted");
+}
+
+static void l2_invlpga_code(void)
+{
+	asm volatile("invlpga" : : "a"(&npf_target), "c"(0) : "memory");
+	GUEST_FAIL("L2 INVLPGA was not intercepted");
+}
+
+static void l2_fep_invlpga_code(void)
+{
+	asm volatile(KVM_FEP "invlpga"
+		     : : "a"(&npf_target), "c"(0) : "memory");
+	GUEST_FAIL("L2 forced-emulated INVLPGA was not intercepted");
+}
+
+struct instruction_intercept_test {
+	const char *name;
+	void (*code)(void);
+	void (*fep_code)(void);
+	u64 intercept;
+	u32 intercept_cr;
+	u32 intercept_dr;
+	u64 exit_code;
+	u64 exit_info_1;
+	u64 exit_info_1_mask;
+	bool check_rax;
+	u64 rax;
+};
+
+static const struct instruction_intercept_test instruction_intercept_tests[] = {
+	{
+		.name = "MOV-to-CR4",
+		.code = l2_mov_to_cr4_code,
+		.fep_code = l2_fep_mov_to_cr4_code,
+		.intercept_cr = BIT(INTERCEPT_CR4_WRITE),
+		.exit_code = SVM_EXIT_WRITE_CR4,
+		.exit_info_1 = BIT_ULL(63),
+		.exit_info_1_mask = ~0ULL,
+	}, {
+		.name = "MOV-from-CR4",
+		.code = l2_mov_from_cr4_code,
+		.fep_code = l2_fep_mov_from_cr4_code,
+		.intercept_cr = BIT(INTERCEPT_CR4_READ),
+		.exit_code = SVM_EXIT_READ_CR4,
+		.exit_info_1 = BIT_ULL(63) | 10,
+		.exit_info_1_mask = ~0ULL,
+	}, {
+		.name = "MOV-to-DR7",
+		.code = l2_mov_to_dr7_code,
+		.fep_code = l2_fep_mov_to_dr7_code,
+		.intercept_dr = BIT(INTERCEPT_DR7_WRITE),
+		.exit_code = SVM_EXIT_WRITE_DR7,
+		.exit_info_1 = 3,
+		.exit_info_1_mask = ~0ULL,
+	}, {
+		.name = "MOV-from-DR7",
+		.code = l2_mov_from_dr7_code,
+		.fep_code = l2_fep_mov_from_dr7_code,
+		.intercept_dr = BIT(INTERCEPT_DR7_READ),
+		.exit_code = SVM_EXIT_READ_DR7,
+		.exit_info_1 = 10,
+		.exit_info_1_mask = ~0ULL,
+	}, {
+		.name = "CLTS",
+		.code = l2_clts_code,
+		.fep_code = l2_fep_clts_code,
+		.intercept_cr = BIT(INTERCEPT_CR0_WRITE),
+		.exit_code = SVM_EXIT_WRITE_CR0,
+		.exit_info_1_mask = BIT_ULL(63),
+	}, {
+		.name = "LMSW",
+		.code = l2_lmsw_code,
+		.fep_code = l2_fep_lmsw_code,
+		.intercept_cr = BIT(INTERCEPT_CR0_WRITE),
+		.exit_code = SVM_EXIT_WRITE_CR0,
+		.exit_info_1_mask = BIT_ULL(63),
+	}, {
+		.name = "SMSW",
+		.code = l2_smsw_code,
+		.fep_code = l2_fep_smsw_code,
+		.intercept_cr = BIT(INTERCEPT_CR0_READ),
+		.exit_code = SVM_EXIT_READ_CR0,
+		.exit_info_1_mask = BIT_ULL(63),
+	}, {
+		.name = "INTn",
+		.code = l2_int_code,
+		.fep_code = l2_fep_int_code,
+		.intercept = BIT_ULL(INTERCEPT_INTn),
+		.exit_code = SVM_EXIT_SWINT,
+		.exit_info_1 = TEST_INT_VECTOR,
+		.exit_info_1_mask = ~0ULL,
+	}, {
+		.name = "INVLPG",
+		.code = l2_invlpg_code,
+		.fep_code = l2_fep_invlpg_code,
+		.intercept = BIT_ULL(INTERCEPT_INVLPG),
+		.exit_code = SVM_EXIT_INVLPG,
+		.exit_info_1 = (u64)&npf_target,
+		.exit_info_1_mask = ~0ULL,
+	}, {
+		.name = "INVLPGA",
+		.code = l2_invlpga_code,
+		.fep_code = l2_fep_invlpga_code,
+		.intercept = BIT_ULL(INTERCEPT_INVLPGA),
+		.exit_code = SVM_EXIT_INVLPGA,
+		.exit_info_1_mask = ~0ULL,
+		.check_rax = true,
+		.rax = (u64)&npf_target,
+	},
+};
+
+static void assert_decode_assist_insn_bytes(struct vmcb *vmcb)
+{
+	GUEST_ASSERT(vmcb->control.insn_len);
+	GUEST_ASSERT(vmcb->control.insn_len <=
+		     sizeof(vmcb->control.insn_bytes));
+	GUEST_ASSERT(!memcmp(vmcb->control.insn_bytes,
+			     (void *)vmcb->save.rip,
+			     vmcb->control.insn_len));
+}
+
+static void assert_full_decode_assist_insn_bytes(struct vmcb *vmcb)
+{
+	GUEST_ASSERT_EQ(vmcb->control.insn_len,
+			sizeof(vmcb->control.insn_bytes));
+	assert_decode_assist_insn_bytes(vmcb);
+}
+
+static void prepare_l2_for_vmrun(struct svm_test_data *svm, gva_t rip)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	vmcb->save.rip = rip;
+	vmcb->save.rsp = (u64)svm->stack;
+}
+
+static void run_intercept_test(struct svm_test_data *svm,
+			       const struct instruction_intercept_test *test,
+			       bool synthesized)
+{
+	struct vmcb *vmcb = svm->vmcb;
+	struct vmcb_control_area *control = &vmcb->control;
+	const char *source = synthesized ? "synthesized" : "hardware";
+	u64 expected_exit_info_1 = test->exit_info_1 & test->exit_info_1_mask;
+
+	control->intercept |= test->intercept;
+	control->intercept_cr |= test->intercept_cr;
+	control->intercept_dr |= test->intercept_dr;
+
+	if (synthesized) {
+		control->exit_info_1 = ~0ULL;
+		control->exit_info_2 = ~0ULL;
+		prepare_l2_for_vmrun(svm, (u64)test->fep_code);
+	} else {
+		prepare_l2_for_vmrun(svm, (u64)test->code);
+	}
+
+	run_guest(vmcb, svm->vmcb_gpa);
+
+	__GUEST_ASSERT(control->exit_code == test->exit_code,
+		       "%s (%s): expected exit code %#lx, got %#lx",
+		       test->name, source, (unsigned long)test->exit_code,
+		       (unsigned long)control->exit_code);
+	__GUEST_ASSERT((control->exit_info_1 & test->exit_info_1_mask) ==
+		       expected_exit_info_1,
+		       "%s (%s): expected EXITINFO1 %#lx with mask %#lx, got %#lx",
+		       test->name, source, (unsigned long)expected_exit_info_1,
+		       (unsigned long)test->exit_info_1_mask,
+		       (unsigned long)control->exit_info_1);
+	__GUEST_ASSERT(!control->insn_len,
+		       "%s (%s): expected no instruction bytes, got %u",
+		       test->name, source, control->insn_len);
+
+	if (test->check_rax)
+		__GUEST_ASSERT(vmcb->save.rax == test->rax,
+			       "%s (%s): expected rAX %#lx, got %#lx",
+			       test->name, source, (unsigned long)test->rax,
+			       (unsigned long)vmcb->save.rax);
+
+	if (synthesized)
+		__GUEST_ASSERT(!control->exit_info_2,
+			       "%s (%s): expected EXITINFO2 to be clear, got %#lx",
+			       test->name, source,
+			       (unsigned long)control->exit_info_2);
+
+	control->intercept &= ~test->intercept;
+	control->intercept_cr &= ~test->intercept_cr;
+	control->intercept_dr &= ~test->intercept_dr;
+}
+
+static void test_instruction_intercepts(struct svm_test_data *svm)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(instruction_intercept_tests); i++) {
+		run_intercept_test(svm, &instruction_intercept_tests[i], false);
+
+		if (is_forced_emulation_enabled)
+			run_intercept_test(svm, &instruction_intercept_tests[i],
+					   true);
+	}
+}
+
+static void test_hardware_npf(struct svm_test_data *svm, gpa_t npf_gpa)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	prepare_l2_for_vmrun(svm, (u64)l2_read_code);
+	run_guest(vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_NPF);
+	GUEST_ASSERT_EQ(vmcb->control.exit_info_2, npf_gpa);
+	assert_decode_assist_insn_bytes(vmcb);
+}
+
+/*
+ * The IOIO intercept causes L0 to emulate OUTSB before accessing its source
+ * operand.  The emulated read then faults on L1's NPT, resulting in a
+ * KVM-synthesized #NPF.
+ */
+static void test_synthesized_npf(struct svm_test_data *svm, gpa_t npf_gpa)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	prepare_l2_for_vmrun(svm, (u64)l2_outsb_code);
+	run_guest(vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_NPF);
+	GUEST_ASSERT_EQ(vmcb->control.exit_info_2, npf_gpa);
+	assert_full_decode_assist_insn_bytes(vmcb);
+}
+
+/*
+ * MOVSB first reads from MMIO, causing a hardware #NPF that L0 emulates.
+ * After userspace completes the read, the emulated destination write faults
+ * on L1's NPT.  The new #NPF must not reuse the original hardware exit's GPA.
+ */
+static void test_synthesized_npf_after_hardware_npf(struct svm_test_data *svm,
+						    gpa_t npf_gpa)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	prepare_l2_for_vmrun(svm, (u64)l2_movsb_code);
+	run_guest(vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_NPF);
+	GUEST_ASSERT_EQ(vmcb->control.exit_info_2, npf_gpa);
+	assert_full_decode_assist_insn_bytes(vmcb);
+}
+
+/*
+ * OUTSB is the final byte of a mapped code page, and the following page is
+ * not present in L2's page tables.  DecodeAssist byte fetching must stop at
+ * the page boundary and report only the OUTSB opcode.
+ */
+static void test_synthesized_npf_truncated(struct svm_test_data *svm,
+					   gpa_t npf_gpa)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	prepare_l2_for_vmrun(svm,
+			     (u64)&boundary_outsb_code[PAGE_SIZE -
+						       BOUNDARY_OUTSB_CODE_SIZE]);
+	run_guest(vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_NPF);
+	GUEST_ASSERT_EQ(vmcb->control.exit_info_2, npf_gpa);
+	GUEST_ASSERT_EQ(vmcb->save.rip,
+			(u64)&boundary_outsb_code[PAGE_SIZE - 1]);
+	GUEST_ASSERT_EQ(vmcb->control.insn_len, 1);
+	GUEST_ASSERT_EQ(vmcb->control.insn_bytes[0], OUTSB_OPCODE);
+}
+
+/*
+ * OUTSB is the final byte in the lower canonical address range.  The first
+ * byte after OUTSB is non-canonical, and must terminate DecodeAssist fetching
+ * even though the corresponding high canonical address is mapped.
+ */
+static void test_synthesized_npf_canonical_boundary(struct svm_test_data *svm,
+						    gpa_t npf_gpa,
+						    gva_t code_gva)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	prepare_l2_for_vmrun(svm, code_gva + PAGE_SIZE -
+				  BOUNDARY_OUTSB_CODE_SIZE);
+	run_guest(vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_NPF);
+	GUEST_ASSERT_EQ(vmcb->control.exit_info_2, npf_gpa);
+	GUEST_ASSERT_EQ(vmcb->save.rip, code_gva + PAGE_SIZE - 1);
+	GUEST_ASSERT_EQ(vmcb->control.insn_len, 1);
+	GUEST_ASSERT_EQ(vmcb->control.insn_bytes[0], OUTSB_OPCODE);
+}
+
+/*
+ * A nonzero CS.base makes the DecodeAssist byte window cross the 32-bit
+ * linear-address boundary without crossing the CS limit.
+ */
+static void test_synthesized_npf_linear_wrap(struct svm_test_data *svm,
+					     gpa_t npf_gpa)
+{
+	struct vmcb *vmcb = svm->vmcb;
+	u16 cs_attrib = vmcb->save.cs.attrib;
+	u64 cs_base = vmcb->save.cs.base;
+	u32 cs_limit = vmcb->save.cs.limit;
+	u32 outsb_eip = LINEAR_WRAP_CODE_GVA + LINEAR_WRAP_OUTSB_OFFSET -
+			LINEAR_WRAP_CS_BASE;
+	u32 setup_eip = LINEAR_WRAP_CODE_GVA + LINEAR_WRAP_SETUP_OFFSET -
+			LINEAR_WRAP_CS_BASE;
+
+	vmcb->save.cs.attrib &= ~SVM_SELECTOR_L_MASK;
+	vmcb->save.cs.attrib |= SVM_SELECTOR_DB_MASK;
+	vmcb->save.cs.base = LINEAR_WRAP_CS_BASE;
+	vmcb->save.cs.limit = UINT32_MAX;
+	prepare_l2_for_vmrun(svm, setup_eip);
+	run_guest(vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_NPF);
+	GUEST_ASSERT_EQ(vmcb->control.exit_info_2, npf_gpa);
+	GUEST_ASSERT_EQ(vmcb->save.rip, outsb_eip);
+	GUEST_ASSERT_EQ(vmcb->control.insn_len,
+			sizeof(linear_wrap_insn_bytes));
+	GUEST_ASSERT(!memcmp(vmcb->control.insn_bytes,
+			     linear_wrap_insn_bytes,
+			     sizeof(linear_wrap_insn_bytes)));
+	vmcb->save.cs.attrib = cs_attrib;
+	vmcb->save.cs.base = cs_base;
+	vmcb->save.cs.limit = cs_limit;
+}
+
+static void test_hardware_intercepted_pf(struct svm_test_data *svm)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	prepare_l2_for_vmrun(svm, (u64)l2_pf_code);
+	run_guest(vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_EXCP_BASE + PF_VECTOR);
+	GUEST_ASSERT_EQ(vmcb->control.exit_info_2, PF_TEST_GVA);
+	GUEST_ASSERT(!(vmcb->control.exit_info_1 & PFERR_PRESENT_MASK));
+	GUEST_ASSERT(!(vmcb->control.exit_info_1 & PFERR_FETCH_MASK));
+	assert_decode_assist_insn_bytes(vmcb);
+}
+
+static void test_hardware_intercepted_fetch_pf(struct svm_test_data *svm)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	prepare_l2_for_vmrun(svm, PF_FETCH_TEST_GVA);
+	run_guest(vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_EXCP_BASE + PF_VECTOR);
+	GUEST_ASSERT_EQ(vmcb->control.exit_info_2, PF_FETCH_TEST_GVA);
+	GUEST_ASSERT(!(vmcb->control.exit_info_1 & PFERR_PRESENT_MASK));
+	GUEST_ASSERT(vmcb->control.exit_info_1 & PFERR_FETCH_MASK);
+	GUEST_ASSERT_EQ(vmcb->control.insn_len, 0);
+}
+
+static void test_synthesized_pf(struct svm_test_data *svm)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	if (!is_forced_emulation_enabled)
+		return;
+
+	prepare_l2_for_vmrun(svm, (u64)l2_fep_pf_code);
+	run_guest(vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_EXCP_BASE + PF_VECTOR);
+	GUEST_ASSERT_EQ(vmcb->control.exit_info_2, PF_TEST_GVA);
+	GUEST_ASSERT(!(vmcb->control.exit_info_1 & PFERR_PRESENT_MASK));
+	GUEST_ASSERT(!(vmcb->control.exit_info_1 & PFERR_FETCH_MASK));
+	assert_full_decode_assist_insn_bytes(vmcb);
+}
+
+/*
+ * Inject #PF while an emulated MMIO read is awaiting completion.  KVM
+ * completes the instruction before constructing the nested VM-Exit, so the
+ * old emulator fetch cache must not be reported for the following instruction.
+ */
+static void test_userspace_injected_pf_during_emulation(struct svm_test_data *svm)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	prepare_l2_for_vmrun(svm, (u64)l2_stale_emulator_pf_code);
+	run_guest(vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_EXCP_BASE + PF_VECTOR);
+	GUEST_ASSERT_EQ(vmcb->control.exit_info_2, PF_TEST_GVA);
+	GUEST_ASSERT(!(vmcb->control.exit_info_1 & PFERR_FETCH_MASK));
+	assert_full_decode_assist_insn_bytes(vmcb);
+}
+
+static void l1_guest_code(struct svm_test_data *svm, gpa_t npf_gpa,
+			  gpa_t iopm_gpa, gva_t canonical_code_gva)
+{
+	struct vmcb *vmcb = svm->vmcb;
+
+	GUEST_ASSERT(this_cpu_has(X86_FEATURE_DECODEASSISTS));
+
+	generic_svm_setup(svm, l2_read_code);
+	vmcb->control.iopm_base_pa = iopm_gpa;
+
+	vmcb->control.intercept |= BIT_ULL(INTERCEPT_IOIO_PROT);
+	vmcb->control.intercept_exceptions |= 1U << PF_VECTOR;
+
+	test_hardware_npf(svm, npf_gpa);
+	test_synthesized_npf(svm, npf_gpa);
+	test_synthesized_npf_after_hardware_npf(svm, npf_gpa);
+	test_synthesized_npf_truncated(svm, npf_gpa);
+	test_synthesized_npf_canonical_boundary(svm, npf_gpa,
+						canonical_code_gva);
+	test_synthesized_npf_linear_wrap(svm, npf_gpa);
+	test_hardware_intercepted_pf(svm);
+	test_hardware_intercepted_fetch_pf(svm);
+	test_synthesized_pf(svm);
+	test_userspace_injected_pf_during_emulation(svm);
+	test_instruction_intercepts(svm);
+
+	GUEST_DONE();
+}
+
+static void build_boundary_outsb_code(u8 *code)
+{
+	u64 source = (u64)&npf_target;
+
+	/* movabs $npf_target, %rsi */
+	code[0] = 0x48;
+	code[1] = 0xbe;
+	memcpy(&code[2], &source, sizeof(source));
+
+	/* mov $0x80, %dx; outsb */
+	code[10] = 0x66;
+	code[11] = 0xba;
+	code[12] = 0x80;
+	code[13] = 0x00;
+	code[14] = OUTSB_OPCODE;
+}
+
+static void prepare_boundary_outsb_code(struct kvm_vm *vm)
+{
+	gva_t code_gva = (gva_t)&boundary_outsb_code[PAGE_SIZE -
+						      BOUNDARY_OUTSB_CODE_SIZE];
+
+	build_boundary_outsb_code(addr_gva2hva(vm, code_gva));
+}
+
+static gva_t prepare_canonical_boundary_outsb_code(struct kvm_vm *vm)
+{
+	gva_t backing_gva = vm_alloc_pages(vm, 2);
+	u8 *code_page = addr_gva2hva(vm, backing_gva);
+	u8 *alias_page = addr_gva2hva(vm, backing_gva + PAGE_SIZE);
+	gpa_t code_gpa = addr_gva2gpa(vm, backing_gva);
+	gpa_t alias_gpa = addr_gva2gpa(vm, backing_gva + PAGE_SIZE);
+	gva_t code_gva = BIT_ULL(vm->va_bits - 1) - PAGE_SIZE;
+	gva_t alias_gva = ~(BIT_ULL(vm->va_bits - 1) - 1);
+	u8 *code = &code_page[PAGE_SIZE - BOUNDARY_OUTSB_CODE_SIZE];
+
+	build_boundary_outsb_code(code);
+	memset(alias_page, 0xcc, PAGE_SIZE);
+	virt_map(vm, code_gva, code_gpa, 1);
+	virt_map(vm, alias_gva, alias_gpa, 1);
+
+	return code_gva;
+}
+
+static void prepare_linear_wrap_outsb_code(struct kvm_vm *vm)
+{
+	gva_t code_gva = vm_alloc_pages(vm, 2);
+	u8 *high_page = addr_gva2hva(vm, code_gva);
+	u8 *low_page = addr_gva2hva(vm, code_gva + PAGE_SIZE);
+	gpa_t high_gpa = addr_gva2gpa(vm, code_gva);
+	gpa_t low_gpa = addr_gva2gpa(vm, code_gva + PAGE_SIZE);
+	u32 source = (u32)(u64)&npf_target;
+	u8 *setup = &high_page[LINEAR_WRAP_SETUP_OFFSET];
+
+	TEST_ASSERT((u64)&npf_target <= UINT32_MAX,
+		    "npf_target must be addressable from compatibility mode");
+
+	/* mov $npf_target, %esi; mov $0x80, %dx */
+	setup[0] = 0xbe;
+	memcpy(&setup[1], &source, sizeof(source));
+	setup[5] = 0x66;
+	setup[6] = 0xba;
+	setup[7] = 0x80;
+	setup[8] = 0x00;
+
+	memcpy(&high_page[LINEAR_WRAP_OUTSB_OFFSET],
+	       linear_wrap_insn_bytes,
+	       PAGE_SIZE - LINEAR_WRAP_OUTSB_OFFSET);
+	memcpy(low_page,
+	       &linear_wrap_insn_bytes[PAGE_SIZE - LINEAR_WRAP_OUTSB_OFFSET],
+	       sizeof(linear_wrap_insn_bytes) -
+	       (PAGE_SIZE - LINEAR_WRAP_OUTSB_OFFSET));
+
+	virt_map(vm, LINEAR_WRAP_CODE_GVA, high_gpa, 1);
+	virt_map(vm, 0, low_gpa, 1);
+}
+
+static void queue_userspace_pf(struct kvm_vcpu *vcpu)
+{
+	struct kvm_vcpu_events events;
+
+	vcpu_events_get(vcpu, &events);
+	TEST_ASSERT(!events.exception.pending && !events.exception.injected,
+		    "Unexpected exception queued before userspace #PF injection");
+	TEST_ASSERT(events.flags & KVM_VCPUEVENT_VALID_PAYLOAD,
+		    "KVM_CAP_EXCEPTION_PAYLOAD was not enabled");
+
+	events.flags |= KVM_VCPUEVENT_VALID_PAYLOAD;
+	events.exception.injected = false;
+	events.exception.pending = true;
+	events.exception.nr = PF_VECTOR;
+	events.exception.has_error_code = true;
+	events.exception.error_code = 0;
+	events.exception_has_payload = true;
+	events.exception_payload = PF_TEST_GVA;
+	vcpu_events_set(vcpu, &events);
+}
+
+static void complete_mmio_read(struct kvm_vcpu *vcpu, gpa_t expected_gpa,
+			       u8 value)
+{
+	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO);
+	TEST_ASSERT(!vcpu->run->mmio.is_write,
+		    "Expected an MMIO read, got a write");
+	TEST_ASSERT_EQ(vcpu->run->mmio.phys_addr, expected_gpa);
+	TEST_ASSERT_EQ(vcpu->run->mmio.len, 1);
+	vcpu->run->mmio.data[0] = value;
+}
+
+static void assert_ucall_done(struct kvm_vcpu *vcpu)
+{
+	struct ucall uc;
+	u64 actual;
+
+	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
+	actual = get_ucall(vcpu, &uc);
+	if (actual == UCALL_ABORT)
+		REPORT_GUEST_ASSERT(uc);
+
+	TEST_ASSERT_EQ(actual, UCALL_DONE);
+}
+
+int main(int argc, char *argv[])
+{
+	gva_t svm_gva, npf_gva, boundary_page_gva, iopm_gva;
+	gva_t canonical_code_gva;
+	gpa_t npf_gpa, mmio_source_gpa, mmio_gpa, iopm_gpa;
+	struct userspace_mem_region *region;
+	struct kvm_vcpu *vcpu;
+	struct kvm_vm *vm;
+	u64 *pte;
+
+	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM));
+	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_NPT));
+	TEST_REQUIRE(this_cpu_has(X86_FEATURE_DECODEASSISTS));
+	TEST_ASSERT(kvm_cpu_has(X86_FEATURE_DECODEASSISTS),
+		    "KVM failed to expose DecodeAssists");
+	TEST_REQUIRE(kvm_has_cap(KVM_CAP_EXCEPTION_PAYLOAD));
+
+	vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
+	vm_enable_cap(vm, KVM_CAP_EXCEPTION_PAYLOAD, 1);
+	prepare_boundary_outsb_code(vm);
+	canonical_code_gva = prepare_canonical_boundary_outsb_code(vm);
+	prepare_linear_wrap_outsb_code(vm);
+	vm_enable_npt(vm);
+	vcpu_alloc_svm(vm, &svm_gva);
+	iopm_gva = vm_alloc_pages(vm, TEST_IOPM_SIZE / PAGE_SIZE);
+	iopm_gpa = addr_gva2gpa(vm, iopm_gva);
+	memset(addr_gva2hva(vm, iopm_gva), 0, TEST_IOPM_SIZE);
+	npf_gva = (gva_t)&npf_target;
+	npf_gpa = addr_gva2gpa(vm, npf_gva);
+
+	tdp_identity_map_default_memslots(vm);
+	pte = tdp_get_pte(vm, npf_gpa);
+	*pte &= ~PTE_PRESENT_MASK(&vm->stage2_mmu);
+	region = memslot2region(vm, 0);
+	mmio_gpa = region->region.guest_phys_addr +
+		   region->region.memory_size + PAGE_SIZE;
+	mmio_source_gpa = addr_gva2gpa(vm, (gva_t)&mmio_source);
+	pte = tdp_get_pte(vm, mmio_source_gpa);
+	*pte = (*pte & ~PHYSICAL_PAGE_MASK) | mmio_gpa;
+
+	boundary_page_gva = (gva_t)&boundary_outsb_code[PAGE_SIZE];
+	pte = vm_get_pte(vm, boundary_page_gva);
+	*pte &= ~PTE_PRESENT_MASK(&vm->mmu);
+
+	vcpu_args_set(vcpu, 4, svm_gva, npf_gpa, iopm_gpa,
+		      canonical_code_gva);
+
+	/* Complete the MMIO source read in the MOVSB #NPF regression test. */
+	vcpu_run(vcpu);
+	complete_mmio_read(vcpu, mmio_gpa, 0xa5);
+
+	/* Leave the second MMIO read pending while injecting #PF. */
+	vcpu_run(vcpu);
+	complete_mmio_read(vcpu, mmio_gpa, 0x5a);
+	queue_userspace_pf(vcpu);
+	vcpu_run(vcpu);
+	assert_ucall_done(vcpu);
+
+	kvm_vm_free(vm);
+	return 0;
+}
-- 
2.43.7