Re: [PATCH v9 16/22] KVM: arm64: Apply dynamic guest counter reservations

[James Clark <james.clark@linaro.org>]

On 24/09/2026 18:29, Colton Lewis wrote:
> Reserve and release guest PMU counters dynamically during vCPU load and
> put rather than statically at VM creation.
> 
> Add kvm_pmu_set_guest_counters() in arch/arm64/kvm/pmu-direct.c, called
> from kvm_pmu_load() and kvm_pmu_put(). When the requested guest counter
> mask collides with active host events in cpuc->used_mask (or when
> releasing counters after squeezing a host event), invoke
> perf_pmu_resched_update() with kvm_pmu_update_mask() to update the
> per-CPU cpuc->cntr_mask between scheduling host events out and back in;
> otherwise update cpuc->cntr_mask directly with interrupts disabled.
> 
> Signed-off-by: Colton Lewis <coltonlewis@google.com>
> ---
>   arch/arm64/kvm/pmu-direct.c  | 77 ++++++++++++++++++++++++++++++++++++
>   include/linux/perf/arm_pmu.h |  1 +
>   2 files changed, 78 insertions(+)
> 
> diff --git a/arch/arm64/kvm/pmu-direct.c b/arch/arm64/kvm/pmu-direct.c
> index a22c9258c2452..31d5afc44f36e 100644
> --- a/arch/arm64/kvm/pmu-direct.c
> +++ b/arch/arm64/kvm/pmu-direct.c
> @@ -115,6 +115,77 @@ u64 kvm_pmu_direct_pmcr_read(struct kvm_vcpu *vcpu)
>   		ARMV8_PMU_PMCR_N);
>   }
>   
> +/* Callback to update counter mask between perf scheduling */
> +static void kvm_pmu_update_mask(struct pmu *pmu, void *data)
> +{
> +	struct arm_pmu *arm_pmu = to_arm_pmu(pmu);
> +	struct pmu_hw_events *cpuc = this_cpu_ptr(arm_pmu->hw_events);
> +	unsigned long *new_mask = data;
> +
> +	bitmap_copy(cpuc->cntr_mask, new_mask, ARMPMU_MAX_HWEVENTS);
> +}
> +
> +/**
> + * kvm_pmu_set_guest_counters() - Handle dynamic counter reservations
> + * @cpu_pmu: struct arm_pmu to potentially modify
> + * @guest_mask: new guest mask for the pmu
> + *
> + * Check if guest counters will interfere with current host events and
> + * call into perf_pmu_resched_update if a reschedule is required.
> + */
> +static void kvm_pmu_set_guest_counters(struct arm_pmu *cpu_pmu, u64 guest_mask)
> +{
> +	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
> +	DECLARE_BITMAP(guest_bitmap, ARMPMU_MAX_HWEVENTS);
> +	DECLARE_BITMAP(new_mask, ARMPMU_MAX_HWEVENTS);
> +	unsigned long flags;
> +	bool need_resched = false;
> +
> +	bitmap_from_arr64(guest_bitmap, &guest_mask, ARMPMU_MAX_HWEVENTS);
> +	bitmap_copy(new_mask, cpu_pmu->cntr_mask, ARMPMU_MAX_HWEVENTS);
> +
> +	local_irq_save(flags);
> +	if (guest_mask) {
> +		/* Subtract guest counters from available host mask */
> +		bitmap_andnot(new_mask, new_mask, guest_bitmap, ARMPMU_MAX_HWEVENTS);
> +
> +		/* Did we collide with an active host event? */
> +		if (bitmap_intersects(cpuc->used_mask, guest_bitmap, ARMPMU_MAX_HWEVENTS)) {
> +			int idx;
> +
> +			need_resched = true;
> +			cpuc->host_squeezed = true;
> +
> +			/* Look for pinned events that are about to be preempted */
> +			for_each_set_bit(idx, guest_bitmap, ARMPMU_MAX_HWEVENTS) {
> +				if (test_bit(idx, cpuc->used_mask) && cpuc->events[idx] &&
> +				    cpuc->events[idx]->attr.pinned) {
> +					pr_warn_once("perf: Pinned host event squeezed out by KVM guest PMU partition\n");

If you enable pseudo-NMIs, watchdog_hardlockup_enable() installs the 
watchdog using a pinned PMU event. If host userspace also has a pinned 
event on the mandatory 1 PMU counter assigned to the host, then a guest 
could potentially squeeze out the watchdog.

I'm wondering if we need to prioritise kernel owned events? Or we just 
treat them the same as any other event, and with PMU partitioning assume 
they can't be guaranteed to be running? I feel like you would expect a 
watchdog to be a bit more than best effort though, especially if there 
was always a guaranteed counter available to put it on.

I didn't follow it through completely, but it also looks like if the 
event gets squeezed it would enter an error state and then never be 
re-enabled, even after the guest stops running.

Note, that I think the current ordering means that the watchdog won't 
actually get squeezed out because it's created first. But I don't think 
we can rely on the ordering as a strong guarantee, and it might get 
broken by refactoring in the future.


> +					break;
> +				}
> +			}
> +		}
> +	} else {
> +		/*
> +		 * Restoring to full mask.
> +		 * Only resched if we previously squeezed an event.
> +		 */
> +		if (cpuc->host_squeezed) {
> +			need_resched = true;
> +			cpuc->host_squeezed = false;
> +		}
> +	}
> +	if (!need_resched)
> +		/* Host was never using guest counters anyway */
> +		bitmap_copy(cpuc->cntr_mask, new_mask, ARMPMU_MAX_HWEVENTS);
> +	local_irq_restore(flags);
> +
> +	if (need_resched) {
> +		/* Collision: run full perf reschedule */
> +		perf_pmu_resched_update(&cpu_pmu->pmu, kvm_pmu_update_mask, new_mask);
> +	}
> +}
> +
>   /**
>    * kvm_pmu_host_counter_mask() - Compute bitmask of host-reserved counters
>    *
> @@ -255,6 +326,7 @@ static void kvm_pmu_apply_event_filter(struct kvm_vcpu *vcpu)
>    */
>   void kvm_pmu_load(struct kvm_vcpu *vcpu)
>   {
> +	struct arm_pmu *pmu;
>   	unsigned long guest_counters;
>   	u64 mask;
>   	u8 i;
> @@ -269,7 +341,9 @@ void kvm_pmu_load(struct kvm_vcpu *vcpu)
>   
>   	preempt_disable();
>   
> +	pmu = vcpu->kvm->arch.arm_pmu;
>   	guest_counters = kvm_vcpu_pmu_guest_counter_mask(vcpu);
> +	kvm_pmu_set_guest_counters(pmu, guest_counters);
>   	kvm_pmu_apply_event_filter(vcpu);
>   
>   	for_each_set_bit(i, &guest_counters, ARMPMU_MAX_HWEVENTS) {
> @@ -329,6 +403,7 @@ void kvm_pmu_load(struct kvm_vcpu *vcpu)
>    */
>   void kvm_pmu_put(struct kvm_vcpu *vcpu)
>   {
> +	struct arm_pmu *pmu;
>   	unsigned long guest_counters;
>   	unsigned long flags;
>   	u64 mask;
> @@ -345,6 +420,7 @@ void kvm_pmu_put(struct kvm_vcpu *vcpu)
>   
>   	preempt_disable();
>   
> +	pmu = vcpu->kvm->arch.arm_pmu;
>   	guest_counters = kvm_vcpu_pmu_guest_counter_mask(vcpu);
>   	mask = guest_counters;
>   
> @@ -395,5 +471,6 @@ void kvm_pmu_put(struct kvm_vcpu *vcpu)
>   	write_sysreg(val & mask, pmovsclr_el0);
>   	local_irq_restore(flags);
>   
> +	kvm_pmu_set_guest_counters(pmu, 0);
>   	preempt_enable();
>   }
> diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h
> index be1e345e99a77..45658273ffa86 100644
> --- a/include/linux/perf/arm_pmu.h
> +++ b/include/linux/perf/arm_pmu.h
> @@ -76,6 +76,7 @@ struct pmu_hw_events {
>   
>   	/* Active events requesting branch records */
>   	unsigned int		branch_users;
> +	bool host_squeezed;
>   };
>   
>   enum armpmu_attr_groups {