[PATCH v3] hw/i386: Configure proper KVM Suppress EOI Broadcast behavior under split irqchip

Historically, under split irqchip, KVM advertised Suppress EOI Broadcast
(SEOIB) support to the guest, but ignored the guest's decision and
continued broadcasting Local APIC EOIs to the userspace I/O APIC. This was
fixed in kernel commit 6517dfbcc918 ("KVM: x86: Add x2APIC "features" to
control EOI broadcast suppression") by introducing two x2APIC flags:
- KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST: Advertise Suppress EOI
  Broadcast support and honor the guest decision.
- KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST: Do not advertise Suppress
  EOI Broadcast support.

For compatibility, KVM continues to follow the quirky behavior by default
unless any of the above two flags are passed.

This patch fixes the behavior by dynamically configuring KVM based on the
userspace I/O APIC version. If the I/O APIC version is 0x20 (which implements
the EOI register used for directed EOI), enable proper Suppress EOI Broadcast
support by passing KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST. Otherwise, if
the I/O APIC version is 0x11, disable Suppress EOI Broadcast support by
passing KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST.

The fix is required because certain guests (for e.g. Windows 2022 with CG)
fails to boot or exhibits extremely slow boot on legacy KVM behavior.

Changing between the legacy quirky behavior and the ENABLE behavior does
not cause any guest-visible state change; it only affects KVM's internal
behavior. However, changing between the legacy quirky behavior and the
DISABLE behavior does result in a guest-visible change.

Hence, to maintain live migration compatibility, introduce a new x86 machine
property `quirked-seoib`. When enabled, this property leaves the legacy KVM
behavior in place and is enabled by default for PC and Q35 machine types
version 11.0 and older.

Users can explicitly override this default by passing
`-machine quirked-seoib=on` (to force the legacy quirky behavior on newer
machine types) or `-machine quirked-seoib=off` (to enforce the fix on
older machine types).

Signed-off-by: Khushit Shah <[email protected]>
---
Note:
Changes from v2:
- Added a new x86 machine property `quirked-seoib` to control the SEOIB
  behavior.
  v2: https://patchew.org/QEMU/[email protected]/
- This relies on the KVM flags introduced in Linux kernel commit
  6517dfbcc918 ("KVM: x86: Add x2APIC "features" to control EOI
  broadcast suppression"), which landed in v6.18+. For hosts running
  older kernels where this kernel patch is not present, QEMU will
  gracefully exit with a hint to use `-machine quirked-seoib=on`.

- I realise this is very late for 11.1 so will probably need to default
  quirked mode on 11.1 also. Sending the patch anyway for now.
---
 hw/i386/pc_piix.c          |  3 +++
 hw/i386/pc_q35.c           |  3 +++
 hw/i386/x86-common.c       | 38 ++++++++++++++++++++++++++++++++++++++
 hw/i386/x86.c              | 21 +++++++++++++++++++++
 include/hw/i386/x86.h      | 12 ++++++++++++
 target/i386/kvm/kvm.c      | 24 ++++++++++++++++++++++++
 target/i386/kvm/kvm_i386.h |  1 +
 7 files changed, 102 insertions(+)

diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
index 82457bdb16..f8636d0b1a 100644
--- a/hw/i386/pc_piix.c
+++ b/hw/i386/pc_piix.c
@@ -437,9 +437,12 @@ DEFINE_I440FX_MACHINE_AS_LATEST(11, 1);
 
 static void pc_i440fx_machine_11_0_options(MachineClass *m)
 {
+    X86MachineClass *x86mc = X86_MACHINE_CLASS(m);
+
     pc_i440fx_machine_11_1_options(m);
     compat_props_add(m->compat_props, hw_compat_11_0, hw_compat_11_0_len);
     compat_props_add(m->compat_props, pc_compat_11_0, pc_compat_11_0_len);
+    x86mc->quirked_seoib = true;
 }
 
 DEFINE_I440FX_MACHINE(11, 0);
diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
index 6c1e4eff5f..3f045f3582 100644
--- a/hw/i386/pc_q35.c
+++ b/hw/i386/pc_q35.c
@@ -392,9 +392,12 @@ DEFINE_Q35_MACHINE_AS_LATEST(11, 1);
 
 static void pc_q35_machine_11_0_options(MachineClass *m)
 {
+    X86MachineClass *x86mc = X86_MACHINE_CLASS(m);
+
     pc_q35_machine_11_1_options(m);
     compat_props_add(m->compat_props, hw_compat_11_0, hw_compat_11_0_len);
     compat_props_add(m->compat_props, pc_compat_11_0, pc_compat_11_0_len);
+    x86mc->quirked_seoib = true;
 }
 
 DEFINE_Q35_MACHINE(11, 0);
diff --git a/hw/i386/x86-common.c b/hw/i386/x86-common.c
index 8f9419e7d3..4f34360b87 100644
--- a/hw/i386/x86-common.c
+++ b/hw/i386/x86-common.c
@@ -66,6 +66,24 @@ out:
     object_unref(cpu);
 }
 
+static bool x86_ioapic_has_eoi_reg(void)
+{
+    Object *obj;
+    uint64_t version;
+
+    /*
+     * The I/O APIC device does not exist yet, as x86_cpus_init() runs
+     * before ioapic_init_gsi(). Hence, use a throwaway instance to check
+     * I/O APIC version.
+     */
+    obj = object_new(TYPE_IOAPIC);
+    version = object_property_get_uint(obj, "version", &error_abort);
+    object_unref(obj);
+
+    /* Only version 0x20 implements the EOI register used for directed EOI. */
+    return version == 0x20;
+}
+
 void x86_cpus_init(X86MachineState *x86ms, int default_cpu_version)
 {
     int i;
@@ -109,6 +127,26 @@ void x86_cpus_init(X86MachineState *x86ms, int default_cpu_version)
         apic_set_max_apic_id(x86ms->apic_id_limit);
     }
 
+    /*
+     * Under split irqchip KVM advertises x2APIC Suppress EOI Broadcast to
+     * the guest but historically ignored the guest's request and kept
+     * broadcasting LAPIC EOIs to the userspace IOAPIC. For compatibility,
+     * KVM still follows legacy behavior by default.
+     * Based on the I/O APIC version, use proper x2APIC Suppress EOI
+     * Broadcast flags (ENABLE or DISABLE) to fix KVM behavior.
+     * This needs to happen before any vCPUs are created.
+     */
+    if (kvm_enabled() && kvm_irqchip_is_split() && !x86ms->quirked_seoib) {
+        Error *local_err = NULL;
+
+        if (!kvm_configure_x2apic_seoib(x86_ioapic_has_eoi_reg(), &local_err)) {
+            error_append_hint(&local_err, "Use -machine %s=on to keep the "
+                              "legacy behaviour.\n", X86_MACHINE_QUIRKED_SEOIB);
+            error_report_err(local_err);
+            exit(1);
+        }
+    }
+
     possible_cpus = mc->possible_cpu_arch_ids(ms);
     for (i = 0; i < ms->smp.cpus; i++) {
         x86_cpu_new(x86ms, possible_cpus->cpus[i].arch_id, &error_fatal);
diff --git a/hw/i386/x86.c b/hw/i386/x86.c
index dc7f0d56b0..75cc3eb3e3 100644
--- a/hw/i386/x86.c
+++ b/hw/i386/x86.c
@@ -205,6 +205,20 @@ static void x86_machine_set_smm(Object *obj, Visitor *v, const char *name,
     visit_type_OnOffAuto(v, name, &x86ms->smm, errp);
 }
 
+static bool x86_machine_get_quirked_seoib(Object *obj, Error **errp)
+{
+    X86MachineState *x86ms = X86_MACHINE(obj);
+
+    return x86ms->quirked_seoib;
+}
+
+static void x86_machine_set_quirked_seoib(Object *obj, bool value, Error **errp)
+{
+    X86MachineState *x86ms = X86_MACHINE(obj);
+
+    x86ms->quirked_seoib = value;
+}
+
 bool x86_machine_is_acpi_enabled(const X86MachineState *x86ms)
 {
     if (x86ms->acpi == ON_OFF_AUTO_OFF) {
@@ -370,6 +384,7 @@ static void x86_machine_initfn(Object *obj)
     x86ms->oem_table_id = g_strndup(ACPI_BUILD_APPNAME8, 8);
     x86ms->bus_lock_ratelimit = 0;
     x86ms->above_4g_mem_start = 4 * GiB;
+    x86ms->quirked_seoib = X86_MACHINE_GET_CLASS(obj)->quirked_seoib;
 }
 
 static void x86_machine_finalize(Object *obj)
@@ -445,6 +460,12 @@ static void x86_machine_class_init(ObjectClass *oc, const void *data)
         NULL, NULL);
     object_class_property_set_description(oc, "sgx-epc",
         "SGX EPC device");
+
+    object_class_property_add_bool(oc, X86_MACHINE_QUIRKED_SEOIB,
+                                   x86_machine_get_quirked_seoib,
+                                   x86_machine_set_quirked_seoib);
+    object_class_property_set_description(oc, X86_MACHINE_QUIRKED_SEOIB,
+        "Leave KVM's Suppress EOI Broadcast quirky behavior in place");
 }
 
 static const TypeInfo x86_machine_info = {
diff --git a/include/hw/i386/x86.h b/include/hw/i386/x86.h
index 71fe6b5e12..bf3075d6f8 100644
--- a/include/hw/i386/x86.h
+++ b/include/hw/i386/x86.h
@@ -32,6 +32,14 @@ struct X86MachineClass {
 
     /* CPU and apic information: */
     bool apic_xrupt_override;
+
+    /*
+     * Leave KVM's Suppress EOI Broadcast quirk in place rather than
+     * configuring proper behaviour from the userspace IOAPIC version.
+     *
+     * Only applicable under split-irqchip.
+     */
+    bool quirked_seoib;
 };
 
 struct X86MachineState {
@@ -93,6 +101,9 @@ struct X86MachineState {
     uint64_t bus_lock_ratelimit;
 
     IgvmCfg *igvm;
+
+    /* Defaults from X86MachineClass, overridable by the user. */
+    bool quirked_seoib;
 };
 
 #define X86_MACHINE_SMM              "smm"
@@ -102,6 +113,7 @@ struct X86MachineState {
 #define X86_MACHINE_OEM_ID           "x-oem-id"
 #define X86_MACHINE_OEM_TABLE_ID     "x-oem-table-id"
 #define X86_MACHINE_BUS_LOCK_RATELIMIT  "bus-lock-ratelimit"
+#define X86_MACHINE_QUIRKED_SEOIB    "quirked-seoib"
 
 #define TYPE_X86_MACHINE   MACHINE_TYPE_NAME("x86")
 OBJECT_DECLARE_TYPE(X86MachineState, X86MachineClass, X86_MACHINE)
diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
index 4272b6770c..c9348d69e7 100644
--- a/target/i386/kvm/kvm.c
+++ b/target/i386/kvm/kvm.c
@@ -307,6 +307,30 @@ bool kvm_enable_x2apic(void)
              has_x2apic_api);
 }
 
+bool kvm_configure_x2apic_seoib(bool enable, Error **errp)
+{
+    KVMState *s = KVM_STATE(current_accel());
+    uint64_t flag = enable ? KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST
+                           : KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST;
+    int supported, ret;
+
+    supported = kvm_vm_check_extension(s, KVM_CAP_X2APIC_API);
+    if (!(supported & flag)) {
+        error_setg(errp, "KVM does not have x2APIC Suppress EOI Broadcast "
+                   "flags");
+        return false;
+    }
+
+    ret = kvm_vm_enable_cap(s, KVM_CAP_X2APIC_API, 0, flag);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "failed to configure x2APIC Suppress "
+                         "EOI Broadcast flags");
+        return false;
+    }
+
+    return true;
+}
+
 bool kvm_hv_vpindex_settable(void)
 {
     return hv_vpindex_settable;
diff --git a/target/i386/kvm/kvm_i386.h b/target/i386/kvm/kvm_i386.h
index 2b653442f4..1e154e536b 100644
--- a/target/i386/kvm/kvm_i386.h
+++ b/target/i386/kvm/kvm_i386.h
@@ -25,6 +25,7 @@
 
 bool kvm_has_smm(void);
 bool kvm_enable_x2apic(void);
+bool kvm_configure_x2apic_seoib(bool enable, Error **errp);
 bool kvm_hv_vpindex_settable(void);
 bool kvm_enable_hypercall(uint64_t enable_mask);
 
-- 
2.39.3