[PATCH 1/2] KVM: x86: Select lowest vCPU ID for aliased xAPIC IDs

[PATCH 1/2] KVM: x86: Select lowest vCPU ID for aliased xAPIC IDs

[Partha Sarathi Satapathy]

From: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>

When KVM_X2APIC_API_USE_32BIT_IDS is enabled, KVM's x2APIC hotplug
quirk allows events to target a vCPU by its x2APIC ID while the vCPU
is still in xAPIC mode.

In xAPIC mode, APIC IDs are truncated to eight bits. Consequently,
vCPUs with distinct x2APIC IDs can share an effective physical APIC ID.
KVM's x2APIC erratum specifies that an event targeting an aliased
physical APIC ID is delivered only to the vCPU with the lowest vCPU ID.

However, the optimized physical APIC map does not guarantee lowest-ID
selection. When rebuilding the map, KVM retains the first xAPIC-mode
vCPU encountered for an aliased 8-bit xAPIC ID. Since vCPUs can be
added in any order, that vCPU need not have the lowest vCPU ID.

For example, vCPU 19 has x2APIC ID 0x013 and vCPU 275 has x2APIC ID
0x113. Both expose xAPIC ID 0x13 while in xAPIC mode. If vCPU 275 is
created before vCPU 19, the map retains:

  phys_map[0x13] = vCPU 275

An INIT/SIPI targeting physical destination 0x13 then starts vCPU 275
instead of vCPU 19. vCPU 275 executes the secondary startup path for
the firmware CPU entry with APIC ID 0x013. After vCPU 275 enables
x2APIC, Linux observes APIC ID 0x113 and reports:

  Firmware: 13 APIC: 113

KVM hardcodes a vCPU's x2APIC ID to its vCPU ID. When populating an
aliased legacy xAPIC entry, retain the vCPU with the lowest vCPU ID.
This makes optimized-map selection independent of vCPU-add order. The
separate full-ID entry remains unaffected, and destination 0x113
continues to select vCPU 275.

Signed-off-by: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>
Signed-off-by: Partha Sarathi Satapathy <partha.satapathy@oracle.com>
---
 arch/x86/kvm/lapic.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 1597dd0b0cc6..5dedffddebec 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -245,7 +245,9 @@ static int kvm_recalculate_phys_map(struct kvm_apic_map *new,
 		if (apic_x2apic_mode(apic) || x2apic_id > 0xff)
 			new->phys_map[x2apic_id] = apic;
 
-		if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
+		if (!apic_x2apic_mode(apic) &&
+		    (!new->phys_map[xapic_id] ||
+		     x2apic_id < new->phys_map[xapic_id]->vcpu->vcpu_id))
 			new->phys_map[xapic_id] = apic;
 	} else {
 		/*
-- 
2.43.7

[PATCH 2/2] KVM: selftests: Cover aliased xAPIC physical IDs

[Partha Sarathi Satapathy]

From: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>

The optimized physical APIC map must select the lowest vCPU ID
when multiple xAPIC-mode vCPUs share an effective physical APIC ID.

Extend recalc_apic_map_test with a regression test that enables
KVM_X2APIC_API_USE_32BIT_IDS and creates vCPU 285 before vCPU 29.
Keep both LAPICs in xAPIC mode, where their physical APIC IDs alias
to 0x1d.

Inject an MSI to destination 0x1d and verify that its vector is
pending only in vCPU 29's IRR. This checks lowest-ID selection when
the higher-ID alias is created first.

Signed-off-by: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>
Tested-by: Partha Sarathi Satapathy <partha.satapathy@oracle.com>
Signed-off-by: Partha Sarathi Satapathy <partha.satapathy@oracle.com>
---
 .../selftests/kvm/x86/recalc_apic_map_test.c  | 98 +++++++++++++++++++
 1 file changed, 98 insertions(+)

diff --git a/tools/testing/selftests/kvm/x86/recalc_apic_map_test.c b/tools/testing/selftests/kvm/x86/recalc_apic_map_test.c
index cbc92a862ea9..88959adde14d 100644
--- a/tools/testing/selftests/kvm/x86/recalc_apic_map_test.c
+++ b/tools/testing/selftests/kvm/x86/recalc_apic_map_test.c
@@ -18,6 +18,100 @@
 #define LAPIC_X2APIC	(MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)
 #define MAX_XAPIC_ID	0xff
 
+#define ALIAS_LOW_VCPU_ID	29
+#define ALIAS_HIGH_VCPU_ID	(ALIAS_LOW_VCPU_ID | 0x100)
+#define ALIAS_VECTOR		0xe1
+#define X86_MSI_BASE_ADDRESS	0xfee00000
+
+static void assert_lapic_xapic_mode(struct kvm_vcpu *vcpu)
+{
+	u64 apicbase = vcpu_get_msr(vcpu, MSR_IA32_APICBASE);
+
+	TEST_ASSERT(apicbase & MSR_IA32_APICBASE_ENABLE,
+		    "vCPU %u local APIC is not enabled, APICBASE=%#lx",
+		    vcpu->id, (unsigned long)apicbase);
+	TEST_ASSERT(!(apicbase & X2APIC_ENABLE),
+		    "vCPU %u unexpectedly entered x2APIC mode, APICBASE=%#lx",
+		    vcpu->id, (unsigned long)apicbase);
+}
+
+static void lapic_sw_enable(struct kvm_vcpu *vcpu)
+{
+	struct kvm_lapic_state lapic;
+
+	vcpu_ioctl(vcpu, KVM_GET_LAPIC, &lapic);
+	*((u32 *)&lapic.regs[APIC_SPIV]) |= APIC_SPIV_APIC_ENABLED;
+	vcpu_ioctl(vcpu, KVM_SET_LAPIC, &lapic);
+}
+
+static bool lapic_irr_has_vector(struct kvm_vcpu *vcpu, u32 vector)
+{
+	struct kvm_lapic_state lapic;
+	u32 irr;
+
+	vcpu_ioctl(vcpu, KVM_GET_LAPIC, &lapic);
+	irr = *((u32 *)&lapic.regs[APIC_IRR + APIC_VECTOR_TO_REG_OFFSET(vector)]);
+
+	return irr & (1u << APIC_VECTOR_TO_BIT_NUMBER(vector));
+}
+
+static void signal_msi(struct kvm_vm *vm, u32 dest_id, u8 vector)
+{
+	struct kvm_msi msi = {
+		.address_lo	= X86_MSI_BASE_ADDRESS | (dest_id << 12),
+		.address_hi	= 0,
+		.data		= vector,
+	};
+	int r;
+
+	r = __vm_ioctl(vm, KVM_SIGNAL_MSI, &msi);
+	TEST_ASSERT(r == 1, "KVM_SIGNAL_MSI should deliver 1 IRQ, got %d", r);
+}
+
+static void test_xapic_alias_lowest_vcpu_id(void)
+{
+	struct kvm_vcpu *high, *low;
+	struct kvm_vm *vm;
+	bool high_has_irq;
+	bool low_has_irq;
+
+	TEST_REQUIRE(kvm_check_cap(KVM_CAP_X2APIC_API) &
+		     KVM_X2APIC_API_USE_32BIT_IDS);
+	TEST_REQUIRE(kvm_has_cap(KVM_CAP_SIGNAL_MSI));
+	TEST_REQUIRE(kvm_check_cap(KVM_CAP_MAX_VCPU_ID) > ALIAS_HIGH_VCPU_ID);
+
+	vm = vm_create(2);
+	vm_enable_cap(vm, KVM_CAP_X2APIC_API, KVM_X2APIC_API_USE_32BIT_IDS);
+
+	/*
+	 * Reproduce hotplug ordering where the vCPU with x2APIC ID 0x11d is
+	 * created before the vCPU with x2APIC ID 0x1d.  In xAPIC mode they
+	 * alias to physical APIC ID 0x1d.  With KVM_X2APIC_API_USE_32BIT_IDS,
+	 * KVM's ABI says aliased physical APIC events are delivered only to
+	 * the lowest vCPU ID.
+	 */
+	high = vm_vcpu_add(vm, ALIAS_HIGH_VCPU_ID, NULL);
+	low = vm_vcpu_add(vm, ALIAS_LOW_VCPU_ID, NULL);
+
+	assert_lapic_xapic_mode(high);
+	assert_lapic_xapic_mode(low);
+
+	lapic_sw_enable(high);
+	lapic_sw_enable(low);
+
+	signal_msi(vm, ALIAS_LOW_VCPU_ID, ALIAS_VECTOR);
+
+	high_has_irq = lapic_irr_has_vector(high, ALIAS_VECTOR);
+	low_has_irq = lapic_irr_has_vector(low, ALIAS_VECTOR);
+
+	TEST_ASSERT(!high_has_irq && low_has_irq,
+		    "Aliased xAPIC IRQ delivered to wrong vCPU: high vCPU %u IRR=%u, low vCPU %u IRR=%u",
+		    ALIAS_HIGH_VCPU_ID, high_has_irq,
+		    ALIAS_LOW_VCPU_ID, low_has_irq);
+
+	kvm_vm_free(vm);
+}
+
 static void *race(void *arg)
 {
 	struct kvm_lapic_state lapic = {};
@@ -42,6 +136,10 @@ int main(void)
 	int i;
 
 	kvm_static_assert(KVM_MAX_VCPUS > MAX_XAPIC_ID);
+	kvm_static_assert(ALIAS_HIGH_VCPU_ID > MAX_XAPIC_ID);
+	kvm_static_assert((ALIAS_HIGH_VCPU_ID & MAX_XAPIC_ID) == ALIAS_LOW_VCPU_ID);
+
+	test_xapic_alias_lowest_vcpu_id();
 
 	/*
 	 * Create the max number of vCPUs supported by selftests so that KVM
-- 
2.43.7