[RFC PATCH 3/9] KVM: SVM: Add Alternate Injection support

[Melody Wang <huibo.wang@amd.com>]

To avoid breaking the other guests without Alternate Injection, add
new paths for Alternate Injection, which will isolate Alternate
Injection from other interrupt injection path.

Add new functions which supports Alternate Injection.

No functionality changes yet

Signed-off-by: Melody Wang <huibo.wang@amd.com>
---
 arch/x86/kvm/irq.c     |   1 +
 arch/x86/kvm/svm/sev.c | 279 ++++++++++++++++++++++++++++++++++++++++-
 arch/x86/kvm/svm/svm.c |  34 +++++
 arch/x86/kvm/svm/svm.h |  27 ++++
 4 files changed, 339 insertions(+), 2 deletions(-)

diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c
index abce6cf24cb5..5f7e5f3e5528 100644
--- a/arch/x86/kvm/irq.c
+++ b/arch/x86/kvm/irq.c
@@ -165,6 +165,7 @@ int kvm_cpu_get_interrupt(struct kvm_vcpu *v)
 
 	return vector;
 }
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_cpu_get_interrupt);
 
 void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu)
 {
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index 924c1b968fdc..78b8487627ed 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -5741,6 +5741,120 @@ static struct hvdb *map_hvdb(struct kvm_vcpu *vcpu, struct kvm_host_map *map)
 	return map->hva;
 }
 
+bool sev_snp_doorbell_has_injectable_vector(struct kvm_vcpu *vcpu)
+{
+	struct kvm_vcpu *vcpu_plane0;
+	struct kvm_host_map hvdb_map;
+	struct hvdb *hvdb;
+	bool ret = false;
+
+	vcpu_plane0 = vcpu->common->vcpus[0];
+	hvdb = map_hvdb(vcpu_plane0, &hvdb_map);
+	if (!hvdb)
+		return ret;
+
+	if (hvdb->events.vmpl2_irq)
+		ret = true;
+
+	unmap_hvdb(vcpu_plane0, &hvdb_map);
+
+	return ret;
+}
+
+bool sev_snp_ai_update_doorbell(enum inject_type type, struct kvm_vcpu *vcpu,
+				int vector, int trig_mode)
+{
+	struct kvm_vcpu *vcpu_plane0;
+	struct kvm_host_map hvdb_map;
+	struct hvdb *hvdb;
+
+	if (vector < FIRST_EXTERNAL_VECTOR && (type == INJECT_IRQ))
+		return false;
+
+	vcpu_plane0 = vcpu->common->vcpus[0];
+	hvdb = map_hvdb(vcpu_plane0, &hvdb_map);
+	if (!hvdb)
+		return false;
+
+	if (type == INJECT_NMI) {
+		hvdb->events.irqs[1].nmi = 1;
+	} else if (type == INJECT_MCE) {
+		hvdb->events.irqs[1].mce = 1;
+	} else {
+		struct ext_irq_desc *vmpl2 = &hvdb->events.irqs[1];
+		unsigned long *bit_vec = (unsigned long *)vmpl2->interrupt_bit_vector;
+
+		if (!hvdb->events.vmpl2_irq) {
+			/* Only one interrupt is pending, queue it. */
+			vmpl2->si_vector = vector;
+			vmpl2->multi_interrupts = 0;
+			vmpl2->trigger_mode = trig_mode;
+		} else if (trig_mode && vmpl2->trigger_mode) {
+			/*
+			 * A higher prio level-triggered overwrites a previous,
+			 * lower prio, level-triggered one.
+			 */
+			if (vmpl2->si_vector < vector)
+				vmpl2->si_vector = vector;
+		} else if (trig_mode) {
+			/*
+			 * Level-triggered takes the single vector slot from the
+			 * pending edge-triggered one, which - if there still is
+			 * a single one - moves to the bit vector.
+			 */
+			if (vmpl2->si_vector)
+				set_bit(vmpl2->si_vector - FIRST_EXTERNAL_VECTOR, bit_vec);
+
+			vmpl2->si_vector = vector;
+			vmpl2->multi_interrupts = 1;
+			vmpl2->trigger_mode = trig_mode;
+		} else if (vector != vmpl2->si_vector) {
+			/*
+			 * Edge-triggered: set mult only when the new vector
+			 * that's being queued is not the currently pending one.
+			 * The already pending vector, if any, gets a bit too.
+			 */
+			set_bit(vector - FIRST_EXTERNAL_VECTOR, bit_vec);
+
+			if (vmpl2->si_vector)
+				set_bit(vmpl2->si_vector - FIRST_EXTERNAL_VECTOR, bit_vec);
+
+			vmpl2->si_vector = 0;
+			vmpl2->multi_interrupts = 1;
+		}
+		hvdb->events.vmpl2_irq = 1;
+	}
+
+	unmap_hvdb(vcpu_plane0, &hvdb_map);
+
+	return true;
+}
+
+bool sev_snp_ai_inject(struct kvm_vcpu *vcpu)
+{
+	struct kvm_vcpu *vcpu_plane0;
+	struct kvm_host_map hvdb_map;
+	struct hvdb *hvdb;
+	struct vcpu_svm *svm0;
+
+	vcpu_plane0 = vcpu->common->vcpus[0];
+	svm0 = to_svm(vcpu_plane0);
+
+	hvdb = map_hvdb(vcpu_plane0, &hvdb_map);
+	if (!hvdb)
+		return false;
+
+	if (hvdb->events.no_further_signal)
+		goto unmap;
+
+	prepare_hv_injection(svm0, hvdb);
+
+unmap:
+	unmap_hvdb(vcpu_plane0, &hvdb_map);
+
+	return true;
+}
+
 static void __sev_snp_inject(enum inject_type type, struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
@@ -5769,8 +5883,10 @@ bool sev_snp_queue_exception(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
-	if (!sev_snp_is_rinj_active(vcpu))
-		return false;
+	if (!vcpu->plane_level) {
+		if (!sev_snp_is_rinj_active(vcpu))
+			return false;
+	}
 
 	if (vcpu->arch.exception.vector == MC_VECTOR) {
 		__sev_snp_inject(INJECT_MCE, vcpu);
@@ -5799,6 +5915,70 @@ bool sev_snp_queue_exception(struct kvm_vcpu *vcpu)
 	return true;
 }
 
+bool sev_snp_ai_queue_exception(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_svm *svm_current = to_svm(vcpu);
+	struct kvm_vcpu *vcpu_plane0;
+	struct vcpu_svm *svm;
+
+	vcpu_plane0 = vcpu->common->vcpus[0];
+	svm = to_svm(vcpu_plane0);
+
+	if (!sev_snp_is_ai_active(vcpu))
+		return false;
+
+	if (vcpu->arch.exception.vector == MC_VECTOR) {
+		if (svm->sev_es.cancel_flag) {
+			svm_current->sev_es.cancel_flag = false;
+			return sev_snp_ai_inject(vcpu);
+		}
+		sev_snp_ai_update_doorbell(INJECT_MCE, vcpu, 0, 0);
+		return sev_snp_ai_inject(vcpu);
+	}
+
+	/*
+	 * Restricted injection is enabled, only #HV is supported.
+	 * If the vector is not HV_VECTOR, do not inject the exception,
+	 * then return true to skip the original injection path.
+	 */
+	if (WARN_ONCE(vcpu->arch.exception.vector != HV_VECTOR,
+		      "restricted injection enabled, exception %u injection not supported\n",
+		      vcpu->arch.exception.vector))
+		return true;
+
+	/*
+	 * An intercept likely occurred during #HV delivery, so re-inject it
+	 * using the current HVDB pending event values.
+	 */
+	svm->vmcb->control.event_inj = HV_VECTOR |
+				       SVM_EVTINJ_TYPE_EXEPT |
+				       SVM_EVTINJ_VALID;
+	svm->vmcb->control.event_inj_err = 0;
+
+	return true;
+}
+
+static void __maybe_unused sev_snp_ai_inject_irq(struct kvm_vcpu *vcpu)
+{
+	struct kvm_lapic *apic;
+	int irq, trig_mode;
+
+	apic = vcpu->arch.apic;
+
+	irq = vcpu->arch.interrupt.nr;
+	trig_mode = !!(apic_test_vector(irq, apic->regs + APIC_TMR));
+	sev_snp_ai_update_doorbell(INJECT_IRQ, vcpu, irq, trig_mode);
+
+	do {
+		irq = kvm_cpu_get_interrupt(vcpu);
+		if (irq < 0)
+			break;
+
+		trig_mode = !!(apic_test_vector(irq, apic->regs + APIC_TMR));
+		sev_snp_ai_update_doorbell(INJECT_IRQ, vcpu, irq, trig_mode);
+	} while (true);
+}
+
 bool sev_snp_inject(enum inject_type type, struct kvm_vcpu *vcpu)
 {
 	if (!sev_snp_is_rinj_active(vcpu))
@@ -5863,6 +6043,70 @@ void sev_snp_cancel_injection(struct kvm_vcpu *vcpu)
 	unmap_hvdb(vcpu, &hvdb_map);
 }
 
+void sev_snp_cancel_ai_injection(struct kvm_vcpu *vcpu)
+{
+	struct kvm_vcpu *vcpu_plane0;
+	struct vcpu_svm *svm0;
+	struct kvm_host_map hvdb_map;
+	struct hvdb *hvdb;
+	struct vcpu_svm *svm = to_svm(vcpu);
+
+	if (!sev_snp_is_ai_active(vcpu))
+		return;
+
+	/*
+	 * AI is enabled for vmpl2 so supposed that the interrupts are
+	 * only for vmpl2, so cancel only for vmpl2
+	 */
+	vcpu_plane0 = vcpu->common->vcpus[0];
+	svm0 = to_svm(vcpu_plane0);
+	if (!svm0->vmcb->control.event_inj)
+		return;
+
+	if (WARN_ONCE((svm->vmcb->control.event_inj & SVM_EVTINJ_VEC_MASK) != HV_VECTOR,
+			"Alternate Injection enabled,  %u vector not supported\n",
+			svm->vmcb->control.event_inj & SVM_EVTINJ_VEC_MASK))
+		return;
+
+	/*
+	 * Copy the information in the doorbell page into the event injection
+	 * fields to complete the cancellation flow.
+	 */
+	hvdb = map_hvdb(vcpu_plane0, &hvdb_map);
+	if (!hvdb)
+		return;
+
+	if (!hvdb->events.irqs[1].ext_word0) {
+		/* No pending events, then event_inj field should be 0 */
+		WARN_ON_ONCE(svm0->vmcb->control.event_inj);
+		goto out;
+	}
+
+	/*
+	 * When the injection is cancelled, since the #HV is set in Plane0's
+	 * control.event_inj, clear this field, and copy the information back to current
+	 * Plane so it will be reinjected, but keep the data in the doorbell.
+	 */
+	svm0->vmcb->control.event_inj = 0;
+
+	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID;
+
+	if (hvdb->events.irqs[1].multi_interrupts || hvdb->events.irqs[1].si_vector)
+		svm->vmcb->control.event_inj |= SVM_EVTINJ_TYPE_INTR;
+
+	if (hvdb->events.irqs[1].nmi)
+		svm->vmcb->control.event_inj |= SVM_EVTINJ_TYPE_NMI;
+
+	if (hvdb->events.irqs[1].mce)
+		svm->vmcb->control.event_inj |= MC_VECTOR | SVM_EVTINJ_TYPE_EXEPT;
+
+	hvdb->events.no_further_signal = 0;
+	svm->sev_es.cancel_flag = true;
+
+out:
+	unmap_hvdb(vcpu_plane0, &hvdb_map);
+}
+
 /*
  * sev_snp_blocked() is for each vector - interrupt, NMI and MCE.  It is
  * checking if there is an interrupt handled by the guest when
@@ -5895,6 +6139,37 @@ bool sev_snp_blocked(enum inject_type type, struct kvm_vcpu *vcpu)
 	return blocked;
 }
 
+bool sev_snp_ai_blocked(enum inject_type type, struct kvm_vcpu *vcpu)
+{
+	struct kvm_host_map hvdb_map;
+	struct hvdb *hvdb;
+	bool blocked;
+	struct kvm_vcpu *vcpu_plane0;
+
+	vcpu_plane0 = vcpu->common->vcpus[0];
+
+	/* Indicate interrupts are blocked if doorbell page can't be mapped */
+	hvdb = map_hvdb(vcpu_plane0, &hvdb_map);
+	if (!hvdb)
+		return true;
+
+	/* Indicate NMIs, MCEs and interrupts blocked based on guest acknowledgment */
+	if (type == INJECT_NMI)
+		blocked = hvdb->events.irqs[1].nmi;
+	else if (type == INJECT_MCE)
+		blocked = hvdb->events.irqs[1].mce;
+	else
+		/* With Alternate Injection, there is a bitmap to save all of interrupts,
+		 * so no need to block any interrupt
+		 */
+		blocked = false;
+
+
+	unmap_hvdb(vcpu_plane0, &hvdb_map);
+
+	return blocked;
+}
+
 int sev_snp_max_planes(struct kvm *kvm)
 {
 	return vmpl_levels;
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 705063c7f0f0..96d31fabb292 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -4065,6 +4065,40 @@ static int svm_mce_allowed(struct kvm_vcpu *vcpu)
 	return 1;
 }
 
+static int __maybe_unused svm_has_interrupt(struct kvm_vcpu *vcpu)
+{
+	int ret = 0;
+	unsigned int state;
+
+	if (!sev_snp_is_ai_active(vcpu))
+		return ret;
+	/*
+	 * Mapping the #HV doorbell page via kvm_vcpu_map() can sleep (e.g.,
+	 * on page faults). In the HLT fast-path, local IRQs are disabled, so
+	 * invoking kvm_vcpu_map() there can trigger a panic. To avoid sleeping
+	 * in atomic context, assume an event is pending during the fast-path
+	 * and defer the actual check until re-entering outside the fast-path.
+	 *
+	 * A related issue occurs during normal vCPU block polling:
+	 * kvm_vcpu_block() sets the task state to TASK_INTERRUPTIBLE before
+	 * calling kvm_vcpu_check_block(). If kvm_vcpu_map() sleeps during that
+	 * check, waking up resets the task state to TASK_RUNNING, causing the
+	 * subsequent schedule() to return immediately instead of blocking.
+	 *
+	 * Therefore, avoid calling kvm_vcpu_map() from
+	 * svm_hv_doorbell_has_events() unless the task state is TASK_RUNNING,
+	 * preventing both sleeping with IRQs disabled and livelocks during
+	 * vCPU blocking.
+	 */
+	state = get_current_state();
+	if (irqs_disabled() || state != TASK_RUNNING)
+		return true;
+
+	ret = sev_snp_doorbell_has_injectable_vector(vcpu);
+
+	return ret;
+}
+
 static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index d2e397542a84..1ec0a045b1ea 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -288,6 +288,7 @@ struct vcpu_sev_es_state {
 
 	gpa_t hvdb_gpa;
 	u8 hvdb_ai_vector;
+	bool cancel_flag;
 };
 
 struct vcpu_svm {
@@ -1002,6 +1003,13 @@ bool sev_snp_queue_exception(struct kvm_vcpu *vcpu);
 bool sev_snp_inject(enum inject_type type, struct kvm_vcpu *vcpu);
 void sev_snp_cancel_injection(struct kvm_vcpu *vcpu);
 bool sev_snp_blocked(enum inject_type type, struct kvm_vcpu *vcpu);
+bool sev_snp_ai_queue_exception(struct kvm_vcpu *vcpu);
+bool sev_snp_doorbell_has_injectable_vector(struct kvm_vcpu *vcpu);
+bool sev_snp_ai_update_doorbell(enum inject_type type, struct kvm_vcpu *vcpu, int vector, int trig_mode);
+void sev_snp_cancel_ai_injection(struct kvm_vcpu *vcpu);
+bool sev_snp_ai_blocked(enum inject_type type, struct kvm_vcpu *vcpu);
+bool sev_snp_ai_inject(struct kvm_vcpu *vcpu);
+
 static inline bool sev_snp_is_rinj_active(struct kvm_vcpu *vcpu)
 {
 	struct kvm_sev_info_plane *sev = &to_kvm_svm_plane(vcpu->plane)->sev_info_plane;
@@ -1009,6 +1017,18 @@ static inline bool sev_snp_is_rinj_active(struct kvm_vcpu *vcpu)
 	return is_sev_snp_guest(vcpu) &&
 		(sev->vmsa_features & SVM_SEV_FEAT_RESTRICTED_INJECTION);
 };
+
+static inline bool sev_snp_is_ai_active(struct kvm_vcpu *vcpu)
+{
+	struct kvm_vcpu *vcpu_plane0 = vcpu->common->vcpus[0];
+	struct kvm_sev_info_plane *sev0 = &to_kvm_svm_plane(vcpu_plane0->plane)->sev_info_plane;
+	struct kvm_sev_info_plane *sev  = &to_kvm_svm_plane(vcpu->plane)->sev_info_plane;
+
+	return is_sev_snp_guest(vcpu) &&
+		(sev0->vmsa_features & SVM_SEV_FEAT_RESTRICTED_INJECTION) &&
+		(sev->vmsa_features & SVM_SEV_FEAT_ALTERNATE_INJECTION);
+}
+
 int sev_nr_vcpu_planes(struct kvm *kvm);
 int sev_snp_max_planes(struct kvm *kvm);
 #else
@@ -1055,6 +1075,13 @@ static inline void sev_snp_cancel_injection(struct kvm_vcpu *vcpu) {}
 static inline bool sev_snp_blocked(enum inject_type type, struct kvm_vcpu *vcpu) { return false; }
 static inline bool sev_snp_is_rinj_active(struct kvm_vcpu *vcpu) { return false; }
 static inline unsigned sev_snp_max_planes(struct kvm *kvm) { return 1; }
+static inline bool sev_snp_ai_queue_exception(struct kvm_vcpu *vcpu) { return false; }
+static inline bool sev_snp_doorbell_has_injectable_vector(struct kvm_vcpu *vcpu) { return false; }
+static inline bool sev_snp_ai_update_doorbell(enum inject_type type, struct kvm_vcpu *vcpu, int vector, int trig_mode) { return false; }
+static inline void sev_snp_cancel_ai_injection(struct kvm_vcpu *vcpu) {}
+static inline bool sev_snp_ai_blocked(enum inject_type type, struct kvm_vcpu *vcpu) { return false; }
+static inline bool sev_snp_ai_inject(struct kvm_vcpu *vcpu) { return false; }
+static inline bool sev_snp_is_ai_active(struct kvm_vcpu *vcpu) { return false; }
 #endif
 
 /* vmenter.S */
-- 
2.43.0