From: Mark Brown <broonie@kernel.org>
To: Marc Zyngier <maz@kernel.org>,
Oliver Upton <oliver.upton@linux.dev>,
Joey Gouly <joey.gouly@arm.com>,
Catalin Marinas <catalin.marinas@arm.com>,
Suzuki K Poulose <suzuki.poulose@arm.com>,
Will Deacon <will@kernel.org>,
Paolo Bonzini <pbonzini@redhat.com>,
Jonathan Corbet <corbet@lwn.net>, Shuah Khan <shuah@kernel.org>
Cc: Dave Martin <Dave.Martin@arm.com>, Fuad Tabba <tabba@google.com>,
Mark Rutland <mark.rutland@arm.com>,
linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev,
linux-kernel@vger.kernel.org, kvm@vger.kernel.org,
linux-doc@vger.kernel.org, linux-kselftest@vger.kernel.org,
Mark Brown <broonie@kernel.org>
Subject: [PATCH v4 22/27] KVM: arm64: Context switch SME state for normal guests
Date: Fri, 14 Feb 2025 01:58:05 +0000 [thread overview]
Message-ID: <20250214-kvm-arm64-sme-v4-22-d64a681adcc2@kernel.org> (raw)
In-Reply-To: <20250214-kvm-arm64-sme-v4-0-d64a681adcc2@kernel.org>
If the guest has SME state we need to context switch that state, provide
support for that for normal guests.
SME has three sets of registers, ZA, ZT (only present for SME2) and also
streaming SVE which replaces the standard floating point registers when
active. The first two are fairly straightforward, they are accessible only
when PSTATE.ZA is set and we can reuse the assembly from the host to save
and load them from a single contiguous buffer. When PSTATE.ZA is not set
then these registers are inaccessible, when the guest enables PSTATE.ZA
all bits will be set to 0 by that and nothing is required on restore.
Streaming mode is slightly more complicated, when enabled via PSTATE.SM it
provides a version of the SVE registers using the SME vector length and may
optionally omit the FFR register. SME may also be present without SVE. The
register state is stored in sve_state as for non-streaming SVE mode, we
make an initial selection of registers to update based on the guest SVE
support and then override this when loading SVCR if streaming mode is
enabled.
Since in order to avoid duplication with SME we now restore the register
state outside of the SVE specific restore function we need to move the
restore of the effective VL for nested guests to a separate restore
function run after loading the floating point register state, along with
the similar handling required for SME.
The selection of which vector length to use is handled by vcpu_sve_pffr().
Signed-off-by: Mark Brown <broonie@kernel.org>
---
arch/arm64/include/asm/fpsimd.h | 9 +++
arch/arm64/include/asm/kvm_emulate.h | 12 +--
arch/arm64/include/asm/kvm_host.h | 3 +
arch/arm64/kvm/fpsimd.c | 16 ++--
arch/arm64/kvm/hyp/include/hyp/switch.h | 128 +++++++++++++++++++++++++++++---
5 files changed, 147 insertions(+), 21 deletions(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 144cc805bfea112341b89c9c6028cf4b2a201c6c..f517b371e0132271a9bd693349a828e2b824ff07 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -442,6 +442,15 @@ static inline size_t sme_state_size(struct task_struct const *task)
write_sysreg_s(__new, (reg)); \
} while (0)
+#define sme_cond_update_smcr_vq(val, reg) \
+ do { \
+ u64 __smcr = read_sysreg_s((reg)); \
+ u64 __new = __smcr & ~SMCR_ELx_LEN_MASK; \
+ __new |= (val) & SMCR_ELx_LEN_MASK; \
+ if (__smcr != __new) \
+ write_sysreg_s(__new, (reg)); \
+ } while (0)
+
/* For use by EFI runtime services calls only */
extern void __efi_fpsimd_begin(void);
extern void __efi_fpsimd_end(void);
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 78ec1ef2cfe82ac31b19cc26ece0d94cea19b46c..19a0faae5ca02a3c8a21ffd2f8da57d56eec0c65 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -590,12 +590,6 @@ static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)
do { \
BUILD_BUG_ON((set) & CPTR_VHE_EL2_RES0); \
BUILD_BUG_ON((clr) & CPACR_EL1_E0POE); \
- __build_check_all_or_none((clr), CPACR_EL1_FPEN); \
- __build_check_all_or_none((set), CPACR_EL1_FPEN); \
- __build_check_all_or_none((clr), CPACR_EL1_ZEN); \
- __build_check_all_or_none((set), CPACR_EL1_ZEN); \
- __build_check_all_or_none((clr), CPACR_EL1_SMEN); \
- __build_check_all_or_none((set), CPACR_EL1_SMEN); \
\
if (has_vhe() || has_hvhe()) \
sysreg_clear_set(cpacr_el1, clr, set); \
@@ -649,4 +643,10 @@ static inline bool guest_hyp_sve_traps_enabled(const struct kvm_vcpu *vcpu)
{
return __guest_hyp_cptr_xen_trap_enabled(vcpu, ZEN);
}
+
+static inline bool guest_hyp_sme_traps_enabled(const struct kvm_vcpu *vcpu)
+{
+ return __guest_hyp_cptr_xen_trap_enabled(vcpu, SMEN);
+}
+
#endif /* __ARM64_KVM_EMULATE_H__ */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 0b00b1e42b668367b89d7ffee2839fc53c14e753..bbab22688a465dc0b5a4463cfaf061612b550fbd 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -1006,6 +1006,9 @@ struct kvm_vcpu_arch {
#define vcpu_sve_zcr_elx(vcpu) \
(unlikely(is_hyp_ctxt(vcpu)) ? ZCR_EL2 : ZCR_EL1)
+#define vcpu_sme_smcr_elx(vcpu) \
+ (unlikely(is_hyp_ctxt(vcpu)) ? SMCR_EL2 : SMCR_EL1)
+
#define vcpu_sve_state_size(vcpu) ({ \
size_t __size_ret; \
unsigned int __vcpu_vq; \
diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c
index 656e02312a5991419bfc3c22193b6b9b4354be64..ed3a17becc489c06c9e2daa3122ec26f1e240e7c 100644
--- a/arch/arm64/kvm/fpsimd.c
+++ b/arch/arm64/kvm/fpsimd.c
@@ -100,19 +100,25 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
WARN_ON_ONCE(!irqs_disabled());
if (guest_owns_fp_regs()) {
- /*
- * Currently we do not support SME guests so SVCR is
- * always 0 and we just need a variable to point to.
- */
fp_state.st = &vcpu->arch.ctxt.fp_regs;
fp_state.sve_state = vcpu->arch.sve_state;
fp_state.sve_vl = vcpu->arch.max_vl[ARM64_VEC_SVE];
- fp_state.sme_state = NULL;
+ fp_state.sme_vl = vcpu->arch.max_vl[ARM64_VEC_SME];
+ fp_state.sme_state = vcpu->arch.sme_state;
fp_state.svcr = &__vcpu_sys_reg(vcpu, SVCR);
fp_state.fpmr = &__vcpu_sys_reg(vcpu, FPMR);
fp_state.fp_type = &vcpu->arch.fp_type;
+
fp_state.sme_features = 0;
+ if (kvm_has_fa64(vcpu->kvm))
+ fp_state.sme_features |= SMCR_ELx_FA64;
+ if (kvm_has_sme2(vcpu->kvm))
+ fp_state.sme_features |= SMCR_ELx_EZT0;
+ /*
+ * For SME only hosts fpsimd_save() will override the
+ * state selection if we are in streaming mode.
+ */
if (vcpu_has_sve(vcpu))
fp_state.to_save = FP_STATE_SVE;
else
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 2c2e22d86a2353531efa49187fe80acb01d31d20..48f6188040fb6a59441bcb22937875de69d7ae34 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -343,6 +343,29 @@ static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code)
return true;
}
+static inline void __hyp_sme_restore_guest(struct kvm_vcpu *vcpu,
+ bool *restore_sve,
+ bool *restore_ffr)
+{
+ bool has_fa64 = vcpu_has_fa64(vcpu);
+ bool has_sme2 = vcpu_has_sme2(vcpu);
+
+ sme_cond_update_smcr(vcpu_sme_max_vq(vcpu) - 1, has_fa64, has_sme2,
+ SYS_SMCR_EL2);
+
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, SMCR_EL1), SYS_SMCR);
+
+ write_sysreg_s(__vcpu_sys_reg(vcpu, SVCR), SYS_SVCR);
+
+ if (vcpu_in_streaming_mode(vcpu)) {
+ *restore_sve = true;
+ *restore_ffr = has_fa64;
+ }
+
+ if (vcpu_za_enabled(vcpu))
+ __sme_restore_state(vcpu_sme_state(vcpu), has_sme2);
+}
+
static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
{
/*
@@ -350,19 +373,25 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
* vCPU. Start off with the max VL so we can load the SVE state.
*/
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
- __sve_restore_state(vcpu_sve_pffr(vcpu),
- &vcpu->arch.ctxt.fp_regs.fpsr,
- true);
+ write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR);
+}
+
+static inline void __hyp_nv_restore_guest_vls(struct kvm_vcpu *vcpu)
+{
/*
* The effective VL for a VM could differ from the max VL when running a
* nested guest, as the guest hypervisor could select a smaller VL. Slap
* that into hardware before wrapping up.
*/
- if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))
+ if (!(vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)))
+ return;
+
+ if (vcpu_has_sve(vcpu))
sve_cond_update_zcr_vq(__vcpu_sys_reg(vcpu, ZCR_EL2), SYS_ZCR_EL2);
- write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR);
+ if (vcpu_has_sme(vcpu))
+ sme_cond_update_smcr_vq(__vcpu_sys_reg(vcpu, SMCR_EL2), SYS_SMCR_EL2);
}
static inline void __hyp_sve_save_host(void)
@@ -378,6 +407,7 @@ static inline void __hyp_sve_save_host(void)
static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)
{
+ u64 smcr_el1, smcr_el2;
u64 zcr_el1, zcr_el2;
if (!guest_owns_fp_regs())
@@ -395,10 +425,29 @@ static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)
zcr_el1 = __vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu));
write_sysreg_el1(zcr_el1, SYS_ZCR);
}
+
+ if (vcpu_has_sme(vcpu)) {
+ /* A guest hypervisor may restrict the effective max VL. */
+ if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))
+ smcr_el2 = __vcpu_sys_reg(vcpu, SMCR_EL2);
+ else
+ smcr_el2 = vcpu_sme_max_vq(vcpu) - 1;
+
+ if (vcpu_has_fa64(vcpu))
+ smcr_el2 |= SMCR_ELx_FA64;
+ if (vcpu_has_sme2(vcpu))
+ smcr_el2 |= SMCR_ELx_EZT0;
+
+ write_sysreg_el2(smcr_el2, SYS_SMCR);
+
+ smcr_el1 = __vcpu_sys_reg(vcpu, vcpu_sme_smcr_elx(vcpu));
+ write_sysreg_el1(smcr_el1, SYS_SMCR);
+ }
}
static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu)
{
+ u64 smcr_el1, smcr_el2;
u64 zcr_el1, zcr_el2;
if (!guest_owns_fp_regs())
@@ -433,6 +482,36 @@ static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu)
write_sysreg_el1(zcr_el1, SYS_ZCR);
}
}
+
+ if (vcpu_has_sme(vcpu)) {
+ smcr_el1 = read_sysreg_el1(SYS_SMCR);
+ __vcpu_sys_reg(vcpu, vcpu_sme_smcr_elx(vcpu)) = smcr_el1;
+
+ /*
+ * The guest's state is always saved using the guest's max VL.
+ * Ensure that the host has the guest's max VL active such that
+ * the host can save the guest's state lazily, but don't
+ * artificially restrict the host to the guest's max VL.
+ */
+ if (has_vhe()) {
+ smcr_el2 = vcpu_sme_max_vq(vcpu) - 1;
+ if (vcpu_has_fa64(vcpu))
+ smcr_el2 |= SMCR_ELx_FA64;
+ if (vcpu_has_sme2(vcpu))
+ smcr_el2 |= SMCR_ELx_EZT0;
+ write_sysreg_el2(smcr_el2, SYS_SMCR);
+ } else {
+ smcr_el2 = sve_vq_from_vl(kvm_host_max_vl[ARM64_VEC_SME]) - 1;
+ if (system_supports_fa64())
+ smcr_el2 |= SMCR_ELx_FA64;
+ if (system_supports_sme2())
+ smcr_el2 |= SMCR_ELx_EZT0;
+ write_sysreg_el2(smcr_el2, SYS_SMCR);
+
+ smcr_el1 = vcpu_sve_max_vq(vcpu) - 1;
+ write_sysreg_el1(smcr_el1, SYS_SMCR);
+ }
+ }
}
static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu)
@@ -466,14 +545,18 @@ static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu)
*/
static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
{
- bool sve_guest;
- u8 esr_ec;
+ u64 cpacr;
+ bool restore_sve, restore_ffr;
+ bool sve_guest, sme_guest;
+ u8 esr_ec, esr_iss;
if (!system_supports_fpsimd())
return false;
sve_guest = vcpu_has_sve(vcpu);
+ sme_guest = vcpu_has_sme(vcpu);
esr_ec = kvm_vcpu_trap_get_class(vcpu);
+ esr_iss = ESR_ELx_ISS(kvm_vcpu_get_esr(vcpu));
/* Only handle traps the vCPU can support here: */
switch (esr_ec) {
@@ -492,6 +575,15 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
if (guest_hyp_sve_traps_enabled(vcpu))
return false;
break;
+ case ESR_ELx_EC_SME:
+ if (!sme_guest)
+ return false;
+ if (guest_hyp_sme_traps_enabled(vcpu))
+ return false;
+ if (!kvm_has_sme2(vcpu->kvm) &&
+ (esr_iss == ESR_ELx_SME_ISS_ZT_DISABLED))
+ return false;
+ break;
default:
return false;
}
@@ -499,10 +591,12 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
/* Valid trap. Switch the context: */
/* First disable enough traps to allow us to update the registers */
+ cpacr = CPACR_EL1_FPEN;
if (sve_guest || (is_protected_kvm_enabled() && system_supports_sve()))
- cpacr_clear_set(0, CPACR_EL1_FPEN | CPACR_EL1_ZEN);
- else
- cpacr_clear_set(0, CPACR_EL1_FPEN);
+ cpacr |= CPACR_EL1_ZEN;
+ if (sme_guest)
+ cpacr |= CPACR_EL1_SMEN;
+ cpacr_clear_set(0, cpacr);
isb();
/* Write out the host state if it's in the registers */
@@ -510,8 +604,20 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
kvm_hyp_save_fpsimd_host(vcpu);
/* Restore the guest state */
+
+ /* These may be overridden for a SME guest */
+ restore_sve = sve_guest;
+ restore_ffr = sve_guest;
+
if (sve_guest)
__hyp_sve_restore_guest(vcpu);
+ if (sme_guest)
+ __hyp_sme_restore_guest(vcpu, &restore_sve, &restore_ffr);
+
+ if (restore_sve)
+ __sve_restore_state(vcpu_sve_pffr(vcpu),
+ &vcpu->arch.ctxt.fp_regs.fpsr,
+ restore_ffr);
else
__fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs);
@@ -522,6 +628,8 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
if (!(read_sysreg(hcr_el2) & HCR_RW))
write_sysreg(__vcpu_sys_reg(vcpu, FPEXC32_EL2), fpexc32_el2);
+ __hyp_nv_restore_guest_vls(vcpu);
+
*host_data_ptr(fp_owner) = FP_STATE_GUEST_OWNED;
return true;
--
2.39.5
next prev parent reply other threads:[~2025-02-14 2:02 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-02-14 1:57 [PATCH v4 00/27] KVM: arm64: Implement support for SME in non-protected guests Mark Brown
2025-02-14 1:57 ` [PATCH v4 01/27] arm64/fpsimd: Update FA64 and ZT0 enables when loading SME state Mark Brown
2025-02-14 1:57 ` [PATCH v4 02/27] arm64/fpsimd: Decide to save ZT0 and streaming mode FFR at bind time Mark Brown
2025-02-14 1:57 ` [PATCH v4 03/27] arm64/fpsimd: Check enable bit for FA64 when saving EFI state Mark Brown
2025-02-14 1:57 ` [PATCH v4 04/27] arm64/fpsimd: Determine maximum virtualisable SME vector length Mark Brown
2025-02-14 1:57 ` [PATCH v4 05/27] KVM: arm64: Introduce non-UNDEF FGT control Mark Brown
2025-02-14 1:57 ` [PATCH v4 06/27] KVM: arm64: Pay attention to FFR parameter in SVE save and load Mark Brown
2025-02-14 1:57 ` [PATCH v4 07/27] KVM: arm64: Pull ctxt_has_ helpers to start of sysreg-sr.h Mark Brown
2025-02-14 1:57 ` [PATCH v4 08/27] KVM: arm64: Move SVE state access macros after feature test macros Mark Brown
2025-02-14 1:57 ` [PATCH v4 09/27] KVM: arm64: Rename SVE finalization constants to be more general Mark Brown
2025-02-14 1:57 ` [PATCH v4 10/27] KVM: arm64: Document the KVM ABI for SME Mark Brown
2025-02-14 1:57 ` [PATCH v4 11/27] KVM: arm64: Define internal features " Mark Brown
2025-02-14 1:57 ` [PATCH v4 12/27] KVM: arm64: Rename sve_state_reg_region Mark Brown
2025-02-14 1:57 ` [PATCH v4 13/27] KVM: arm64: Store vector lengths in an array Mark Brown
2025-02-14 1:57 ` [PATCH v4 14/27] KVM: arm64: Implement SME vector length configuration Mark Brown
2025-02-14 1:57 ` [PATCH v4 15/27] KVM: arm64: Support SME control registers Mark Brown
2025-02-14 1:57 ` [PATCH v4 16/27] KVM: arm64: Support TPIDR2_EL0 Mark Brown
2025-02-14 1:58 ` [PATCH v4 17/27] KVM: arm64: Support SME identification registers for guests Mark Brown
2025-02-14 1:58 ` [PATCH v4 18/27] KVM: arm64: Support SME priority registers Mark Brown
2025-02-14 1:58 ` [PATCH v4 19/27] KVM: arm64: Provide assembly for SME state restore Mark Brown
2025-02-14 1:58 ` [PATCH v4 20/27] KVM: arm64: Support userspace access to streaming mode Z and P registers Mark Brown
2025-02-14 1:58 ` [PATCH v4 21/27] KVM: arm64: Expose SME specific state to userspace Mark Brown
2025-02-14 1:58 ` Mark Brown [this message]
2025-02-14 1:58 ` [PATCH v4 23/27] KVM: arm64: Handle SME exceptions Mark Brown
2025-02-14 1:58 ` [PATCH v4 24/27] KVM: arm64: Expose SME to nested guests Mark Brown
2025-02-14 1:58 ` [PATCH v4 25/27] KVM: arm64: Provide interface for configuring and enabling SME for guests Mark Brown
2025-02-14 1:58 ` [PATCH v4 26/27] KVM: arm64: selftests: Add SME system registers to get-reg-list Mark Brown
2025-02-14 1:58 ` [PATCH v4 27/27] KVM: arm64: selftests: Add SME to set_id_regs test Mark Brown
2025-02-14 9:24 ` [PATCH v4 00/27] KVM: arm64: Implement support for SME in non-protected guests Marc Zyngier
2025-02-14 15:13 ` Mark Brown
2025-02-17 9:37 ` Marc Zyngier
2025-02-18 22:54 ` Mark Brown
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20250214-kvm-arm64-sme-v4-22-d64a681adcc2@kernel.org \
--to=broonie@kernel.org \
--cc=Dave.Martin@arm.com \
--cc=catalin.marinas@arm.com \
--cc=corbet@lwn.net \
--cc=joey.gouly@arm.com \
--cc=kvm@vger.kernel.org \
--cc=kvmarm@lists.linux.dev \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=maz@kernel.org \
--cc=oliver.upton@linux.dev \
--cc=pbonzini@redhat.com \
--cc=shuah@kernel.org \
--cc=suzuki.poulose@arm.com \
--cc=tabba@google.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®