Re: [PATCH 02/11] KVM: arm64: CCA: Introduce Realms

[Suzuki K Poulose <suzuki.poulose@arm.com>]

On 11/11/2026 10:11, Steven Price wrote:
> On 10/11/2026 11:42, Kohei Enju wrote:
>> Hi Steve,
> 
> Hi Kohei,
> 
>> On 11/02 14:12, Steven Price wrote:
>>> From: Suzuki K Poulose <suzuki.poulose@arm.com>
>>>
>>> Add foundational work for supporting Realms.
>>>   - Add a new VM flavor.
>>>   - At KVM init, check if the KVM can support Realms (though not functional
>>>     yet) and will be advertised by static key kvm_rmi_is_available. This
>>>     will be turned on in a later patch, once we have all the bits and
>>>     pieces ready. For now check if we are blessed with KVM_MODE_RMM.
>>>   - Add realm specific tracking in kvm_arch. Since Realm and protected pKVM
>>>     states are mutually exclusive, move them into a union.
>>>   - Also adjust the memslot preparation checks to reflect the changes.
>>>
>>> Please note that we cannot create Realm VMs yet. This requires further
>>> changes to the UABI and core RMI driver support, which will come later.
>>>
>>> Signed-off-by: Suzuki K Poulose <suzuki.poulose@arm.com>
>>> Signed-off-by: Steven Price <steven.price@arm.com>
>>> ---
>>>   arch/arm64/include/asm/kvm_emulate.h   | 16 +++++++
>>>   arch/arm64/include/asm/kvm_host.h      | 57 +++++++++++++++++-------
>>>   arch/arm64/include/asm/kvm_rmi.h       | 61 ++++++++++++++++++++++++++
>>>   arch/arm64/include/asm/virt.h          |  1 +
>>>   arch/arm64/kvm/Makefile                |  2 +-
>>>   arch/arm64/kvm/arm.c                   | 24 ++++++++--
>>>   arch/arm64/kvm/hyp/include/nvhe/pkvm.h |  2 +-
>>>   arch/arm64/kvm/hyp/nvhe/switch.c       |  4 +-
>>>   arch/arm64/kvm/hyp/nvhe/timer-sr.c     |  2 +-
>>>   arch/arm64/kvm/mmu.c                   | 13 +++---
>>>   arch/arm64/kvm/rmi.c                   | 18 ++++++++
>>>   11 files changed, 169 insertions(+), 31 deletions(-)
>>>   create mode 100644 arch/arm64/include/asm/kvm_rmi.h
>>>   create mode 100644 arch/arm64/kvm/rmi.c
>>>
>>> diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
>>> index 334211b8b2ee..32021bdf2610 100644
>>> --- a/arch/arm64/include/asm/kvm_emulate.h
>>> +++ b/arch/arm64/include/asm/kvm_emulate.h
>>> @@ -831,4 +831,20 @@ static inline void kvm_reset_vcpu_psci(struct kvm_vcpu *vcpu,
>>>   	vcpu_set_reg(vcpu, 0, reset_state->r0);
>>>   }
>>>   
>>> +static inline enum realm_state kvm_realm_state(struct kvm *kvm)
>>> +{
>>> +	return READ_ONCE(kvm->arch.realm.state);
>>> +}
>>> +
>>> +static inline void kvm_set_realm_state(struct kvm *kvm,
>>> +				       enum realm_state new_state)
>>> +{
>>> +	WRITE_ONCE(kvm->arch.realm.state, new_state);
>>> +}
>>> +
>>> +static inline bool kvm_realm_is_created(struct kvm *kvm)
>>> +{
>>> +	return kvm_vm_is_realm(kvm) && kvm_realm_state(kvm) != REALM_STATE_NONE;
>>> +}
>>> +
>>>   #endif /* __ARM64_KVM_EMULATE_H__ */
>>> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
>>> index ccd2dc8f73c3..7d0222047ca1 100644
>>> --- a/arch/arm64/include/asm/kvm_host.h
>>> +++ b/arch/arm64/include/asm/kvm_host.h
>>> @@ -29,6 +29,7 @@
>>>   #include <asm/kvm.h>
>>>   #include <asm/kvm_asm.h>
>>>   #include <asm/kvm_hcall.h>
>>> +#include <asm/kvm_rmi.h>
>>>   #include <asm/vncr_mapping.h>
>>>   
>>>   #define __KVM_HAVE_ARCH_INTC_INITIALIZED
>>> @@ -48,6 +49,13 @@
>>>   #define KVM_VCPU_MAX_FEATURES 10
>>>   #define KVM_VCPU_VALID_FEATURES	(BIT(KVM_VCPU_MAX_FEATURES) - 1)
>>>   
>>> +#define KVM_REALM_VCPU_VALID_FEATURES			\
>>> +	(BIT(KVM_ARM_VCPU_POWER_OFF)		|	\
>>> +	 BIT(KVM_ARM_VCPU_PSCI_0_2)		|	\
>>> +	 BIT(KVM_ARM_VCPU_PTRAUTH_ADDRESS)	|	\
>>> +	 BIT(KVM_ARM_VCPU_SVE)			|	\
>>> +	 BIT(KVM_ARM_VCPU_PTRAUTH_GENERIC))
>>> +
>>>   #define KVM_REQ_SLEEP \
>>>   	KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
>>>   #define KVM_REQ_IRQ_PENDING		KVM_ARCH_REQ(1)
>>> @@ -409,6 +417,7 @@ enum kvm_arm_vm_flavor {
>>>   	VM_PKVM,		/* Normal guests on pKVM */
>>>   	MARKER(__VM_PROTECTED),
>>>   	VM_PROTECTED_PKVM,	/* Protected VM */
>>> +	VM_REALM,		/* CCA */
>>>   	VM_FLAVOR_MAX
>>>   };
>>>   
>>> @@ -502,11 +511,14 @@ struct kvm_arch {
>>>   	/* Count the number of VNCR_EL2 TLBs */
>>>   	atomic_t vncr_tlb_count;
>>>   
>>> -	/*
>>> -	 * For an untrusted host VM, 'pkvm.handle' is used to lookup
>>> -	 * the associated pKVM instance in the hypervisor.
>>> -	 */
>>> -	struct kvm_protected_vm pkvm;
>>> +	union {
>>> +		/*
>>> +		 * For an untrusted host VM, 'pkvm.handle' is used to lookup
>>> +		 * the associated pKVM instance in the hypervisor.
>>> +		 */
>>> +		struct kvm_protected_vm pkvm;
>>> +		struct realm realm;
>>> +	};
>>>   
>>>   #ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS
>>>   	/* Nested virtualization info */
>>> @@ -1572,30 +1584,43 @@ struct kvm *kvm_arch_alloc_vm(void);
>>>   
>>>   #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE
>>>   
>>> -#ifdef __KVM_NVHE_HYPERVISOR__
>>> +#define kvm_vm_is_protected(kvm)	((kvm)->arch.vm_flavor >= __VM_PROTECTED)
>>>   
>>> -#define kvm_vm_is_protected(kvm)			\
>>> -	(is_protected_kvm_enabled() && ((kvm)->arch.vm_flavor == VM_PROTECTED_PKVM))
>>> +#ifdef __KVM_NVHE_HYPERVISOR__
>>>   /*
>>>    * Accessing vcpu->kvm from nVHE hyp stub is tricky, as we need to convert the
>>>    * pointer to the hyp VA. With pKVM, the nVHE code runs with the hyp_vcpu,
>>> - * which is populated correctly and is gated on is_protected_kvm_enabled().
>>> + * which is populated correctly. Always vcpu_is_protected_pkvm(), which is
>>> + * gated on is_protected_kvm_enabled().
>>> + */
>>> +#define vcpu_is_protected(vcpu)		BUILD_BUG_ON(1)
>>> +#else
>>> +#define vcpu_is_protected(vcpu)		kvm_vm_is_protected((vcpu)->kvm)
>>> +#endif
>>> +
>>> +#define kvm_vm_is_protected_pkvm(kvm)					\
>>> +	(is_protected_kvm_enabled() && ((kvm)->arch.vm_flavor == VM_PROTECTED_PKVM))
>>> +
>>> +/*
>>> + * Rely on is_protected_kvm_enabled() check in kvm_vm_is_protected_pkvm() to
>>> + * make sure the vcpu->kvm is always valid VA in the context
>>>    */
>>> -#define vcpu_is_protected(vcpu)						\
>>> +#define vcpu_is_protected_pkvm(vcpu)					\
>>>   	({								\
>>>   		struct kvm *__kvm = READ_ONCE((vcpu)->kvm);		\
>>>   									\
>>> -		(__kvm && kvm_vm_is_protected(__kvm));			\
>>> +		(__kvm && kvm_vm_is_protected_pkvm(__kvm));		\
>>>   	})
>>
>> I've tested the integration branch and found that the secondary CPU
>> bring-up fails:
>>      [    1.426502] smp: Bringing up secondary CPUs ...
>>      [    9.194442] CPU1: failed to come online
>>      [    9.194558] CPU1: failed in unknown state : 0x0
>>      [    9.197714] smp: Brought up 1 node, 1 CPU
>>      [    9.197775] SMP: Total of 1 processors activated.
>>
>> It seems that this patch missed replacing a use of
>> vcpu_is_protected() introduced by the following commit:
>>
>>      25d726dddc23 ("KVM: arm64: Reject host power-on of a vCPU that EL2 holds powered off")
>>
>> The following diff fixes the issue for me. Also, though not included in
>> the diff, I think vcpu_is_protected() in __pkvm_create_hyp_vcpu() should
>> be replaced with vcpu_is_protected_pkvm() as well.
> 
> Thanks for the patch - I'll include that in v2.

Please could we make it something like this :

diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 4edcaf1090f9..2108c1bf5481 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -989,7 +989,8 @@ static bool kvm_arm_vcpu_suspended(struct kvm_vcpu 
*vcpu)
  /* Only the guest's CPU_ON may start a vCPU EL2 holds powered off. */
  bool kvm_pkvm_vcpu_is_powered_off(struct kvm_vcpu *vcpu)
  {
-       return READ_ONCE(vcpu->arch.mp_state.mp_state) == 
KVM_MP_STATE_UNINITIALIZED;
+       return vcpu_is_protected_pkvm(vcpu) &&
+               READ_ONCE(vcpu->arch.mp_state.mp_state) == 
KVM_MP_STATE_UNINITIALIZED;
  }

  int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c
index b01f8d44033f..b7606c3542f8 100644
--- a/arch/arm64/kvm/psci.c
+++ b/arch/arm64/kvm/psci.c
@@ -68,7 +68,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu 
*source_vcpu)
          * STOPPED before that is the VMM's pause.
          */
         if (!kvm_arm_vcpu_stopped(vcpu) ||
-           (vcpu_is_protected(vcpu) && 
!kvm_pkvm_vcpu_is_powered_off(vcpu))) {
+           !kvm_pkvm_vcpu_is_powered_off(vcpu)) {
                 if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1)
                         ret = PSCI_RET_ALREADY_ON;
                 else


i.e, check for pkvm vcpu as it should have been in the first case, 
rather than spilling the check for protected vcpu and an explicit
check to see if that is powered off ?


Suzuki>
> Thanks,
> Steve
> 
>> Thanks,
>> Kohei
>>
>> ---8<---
>> diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
>> index 522a837637fe..be676c13e219 100644
>> --- a/arch/arm64/kvm/arm.c
>> +++ b/arch/arm64/kvm/arm.c
>> @@ -960,7 +960,8 @@ static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu, u32 mp_state)
>>   /* The guest's own CPU_OFF: EL2 has already powered the vCPU off. */
>>   void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
>>   {
>> -       u32 mp_state = vcpu_is_protected(vcpu) ? KVM_MP_STATE_UNINITIALIZED : KVM_MP_STATE_STOPPED;
>> +       u32 mp_state = vcpu_is_protected_pkvm(vcpu) ?
>> +                      KVM_MP_STATE_UNINITIALIZED : KVM_MP_STATE_STOPPED;
>>
>>          spin_lock(&vcpu->arch.mp_state_lock);
>>          __kvm_arm_vcpu_power_off(vcpu, mp_state);
>> diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c
>> index b01f8d44033f..6674e8d6ff4c 100644
>> --- a/arch/arm64/kvm/psci.c
>> +++ b/arch/arm64/kvm/psci.c
>> @@ -68,7 +68,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
>>           * STOPPED before that is the VMM's pause.
>>           */
>>          if (!kvm_arm_vcpu_stopped(vcpu) ||
>> -           (vcpu_is_protected(vcpu) && !kvm_pkvm_vcpu_is_powered_off(vcpu))) {
>> +           (vcpu_is_protected_pkvm(vcpu) && !kvm_pkvm_vcpu_is_powered_off(vcpu))) {
>>                  if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1)
>>                          ret = PSCI_RET_ALREADY_ON;
>>                  else
>>
>