* [PATCH 1/2] sched/psi: Preserve pressure accounting across RT trigger registration
2026-10-19 10:46 [PATCH 0/2] sched/psi: Preserve RT poll notifications Hui Su
@ 2026-10-19 10:46 ` Hui Su
2026-10-19 10:46 ` [PATCH 2/2] sched/psi: Deliver pending RT poll events after polling expires Hui Su
1 sibling, 0 replies; 3+ messages in thread
From: Hui Su @ 2026-10-19 10:46 UTC (permalink / raw)
To: hannes, surenb, peterz, mingo, juri.lelli, vincent.guittot
Cc: dietmar.eggemann, rostedt, bsegall, mgorman, vschneid,
kprateek.nayak, linux-kernel, sh_def
Creating a privileged trigger samples per-cpu pressure before an existing
rtpoll worker can consume it. Preserve state changes for existing triggers,
and initialize the new trigger window after the registration sample so it
does not account pressure that predates registration.
Evaluate triggers whenever their update is due, even if the current sample
has no changed states. Advance rtpoll_total after each evaluation so a
consumed sample is not counted again.
Signed-off-by: Hui Su <sh_def@163.com>
---
include/linux/psi_types.h | 1 +
kernel/sched/psi.c | 34 ++++++++++++++++++++++------------
2 files changed, 23 insertions(+), 12 deletions(-)
diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h
index dd10c22299ab8..5507a31477a43 100644
--- a/include/linux/psi_types.h
+++ b/include/linux/psi_types.h
@@ -197,6 +197,7 @@ struct psi_group {
struct list_head rtpoll_triggers;
u32 rtpoll_nr_triggers[NR_PSI_STATES - 1];
u32 rtpoll_states;
+ u32 rtpoll_pending_states;
u64 rtpoll_min_period;
/* Total stall times at the start of RT polling monitor activation */
diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c
index 7c53814233345..733ecacc2adb4 100644
--- a/kernel/sched/psi.c
+++ b/kernel/sched/psi.c
@@ -219,6 +219,7 @@ static void group_init(struct psi_group *group)
mutex_init(&group->rtpoll_trigger_lock);
INIT_LIST_HEAD(&group->rtpoll_triggers);
group->rtpoll_min_period = U32_MAX;
+ group->rtpoll_pending_states = 0;
group->rtpoll_next_update = ULLONG_MAX;
init_waitqueue_head(&group->rtpoll_wait);
timer_setup(&group->rtpoll_timer, poll_timer_fn, 0);
@@ -648,7 +649,7 @@ static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long dela
static void psi_rtpoll_work(struct psi_group *group)
{
bool force_reschedule = false;
- u32 changed_states;
+ u32 changed_states, sampled_states;
u64 now;
mutex_lock(&group->rtpoll_trigger_lock);
@@ -691,8 +692,11 @@ static void psi_rtpoll_work(struct psi_group *group)
force_reschedule = true;
}
-
- collect_percpu_times(group, PSI_POLL, &changed_states);
+ /* Trigger registration may have consumed an activity delta. */
+ changed_states = group->rtpoll_pending_states;
+ group->rtpoll_pending_states = 0;
+ collect_percpu_times(group, PSI_POLL, &sampled_states);
+ changed_states |= sampled_states;
if (changed_states & group->rtpoll_states) {
/* Initialize trigger windows when entering rtpolling mode */
@@ -714,11 +718,8 @@ static void psi_rtpoll_work(struct psi_group *group)
}
if (now >= group->rtpoll_next_update) {
- if (changed_states & group->rtpoll_states) {
- update_triggers(group, now, PSI_POLL);
- memcpy(group->rtpoll_total, group->total[PSI_POLL],
- sizeof(group->rtpoll_total));
- }
+ update_triggers(group, now, PSI_POLL);
+ memcpy(group->rtpoll_total, group->total[PSI_POLL], sizeof(group->rtpoll_total));
group->rtpoll_next_update = now + group->rtpoll_min_period;
}
@@ -1334,7 +1335,7 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
{
struct psi_trigger *t;
enum psi_states state;
- u32 threshold_us;
+ u32 threshold_us, changed_states, existing_states;
bool privileged;
u32 window_us;
@@ -1386,8 +1387,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
t->state = state;
t->threshold = threshold_us * NSEC_PER_USEC;
t->win.size = window_us * NSEC_PER_USEC;
- window_reset(&t->win, sched_clock(),
- group->total[PSI_POLL][t->state], 0);
+ if (!privileged)
+ window_reset(&t->win, sched_clock(), group->total[PSI_POLL][t->state], 0);
t->event = 0;
t->last_event_time = 0;
@@ -1400,13 +1401,19 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
if (privileged) {
mutex_lock(&group->rtpoll_trigger_lock);
+ existing_states = group->rtpoll_states;
list_add(&t->node, &group->rtpoll_triggers);
group->rtpoll_min_period = min(group->rtpoll_min_period,
div_u64(t->win.size, UPDATES_PER_WINDOW));
group->rtpoll_nr_triggers[t->state]++;
group->rtpoll_states |= (1 << t->state);
+ collect_percpu_times(group, PSI_POLL, &changed_states);
+ group->rtpoll_pending_states |= changed_states & existing_states;
+ window_reset(&t->win, sched_clock(), group->total[PSI_POLL][t->state], 0);
*need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task);
+ if (!*need_rtpoll_worker)
+ psi_schedule_rtpoll_work(group, 1, true);
mutex_unlock(&group->rtpoll_trigger_lock);
} else {
@@ -1458,8 +1465,10 @@ void psi_trigger_destroy(struct psi_trigger *t)
list_del(&t->node);
group->rtpoll_nr_triggers[t->state]--;
- if (!group->rtpoll_nr_triggers[t->state])
+ if (!group->rtpoll_nr_triggers[t->state]) {
group->rtpoll_states &= ~(1 << t->state);
+ group->rtpoll_pending_states &= ~(1 << t->state);
+ }
/*
* Reset min update period for the remaining triggers
* iff the destroying trigger had the min window size.
@@ -1473,6 +1482,7 @@ void psi_trigger_destroy(struct psi_trigger *t)
/* Destroy rtpoll_task when the last trigger is destroyed */
if (group->rtpoll_states == 0) {
group->rtpoll_until = 0;
+ group->rtpoll_pending_states = 0;
task_to_destroy = rcu_dereference_protected(
group->rtpoll_task,
lockdep_is_held(&group->rtpoll_trigger_lock));
--
2.55.0
^ permalink raw reply [flat|nested] 3+ messages in thread* [PATCH 2/2] sched/psi: Deliver pending RT poll events after polling expires
2026-10-19 10:46 [PATCH 0/2] sched/psi: Preserve RT poll notifications Hui Su
2026-10-19 10:46 ` [PATCH 1/2] sched/psi: Preserve pressure accounting across RT trigger registration Hui Su
@ 2026-10-19 10:46 ` Hui Su
1 sibling, 0 replies; 3+ messages in thread
From: Hui Su @ 2026-10-19 10:46 UTC (permalink / raw)
To: hannes, surenb, peterz, mingo, juri.lelli, vincent.guittot
Cc: dietmar.eggemann, rostedt, bsegall, mgorman, vschneid,
kprateek.nayak, linux-kernel, sh_def
A PSI real-time trigger can defer a threshold notification until its
rate-limit window ends. If pressure stops or the polling window expires
before then, the rtpoll worker can become idle without delivering it.
Compute pending deadlines from the trigger list when reinitializing or
scheduling passive polling. This avoids a duplicate group-level cache while
preserving pending trigger windows without reprocessing consumed samples.
Return to idle once the pending notification is delivered and no pressure
remains.
Fixes: e6df4ead85d9 ("psi: fix possible trigger missing in the window")
Signed-off-by: Hui Su <sh_def@163.com>
---
kernel/sched/psi.c | 176 ++++++++++++++++++++++++++++++++++-----------
1 file changed, 136 insertions(+), 40 deletions(-)
diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c
index 733ecacc2adb4..27b5473122f15 100644
--- a/kernel/sched/psi.c
+++ b/kernel/sched/psi.c
@@ -461,13 +461,14 @@ static u64 window_update(struct psi_window *win, u64 now, u64 value)
return growth;
}
-static void update_triggers(struct psi_group *group, u64 now,
- enum psi_aggregators aggregator)
+static u64 update_triggers(struct psi_group *group, u64 now,
+ enum psi_aggregators aggregator)
{
struct psi_trigger *t;
u64 *total = group->total[aggregator];
struct list_head *triggers;
u64 *aggregator_total;
+ u64 pending_deadline = ULLONG_MAX;
if (aggregator == PSI_AVGS) {
triggers = &group->avg_triggers;
@@ -507,8 +508,10 @@ static void update_triggers(struct psi_group *group, u64 now,
}
}
/* Limit event signaling to once per window */
- if (now < t->last_event_time + t->win.size)
+ if (now < t->last_event_time + t->win.size) {
+ pending_deadline = min(pending_deadline, t->last_event_time + t->win.size);
continue;
+ }
/* Generate an event */
if (cmpxchg(&t->event, 0, 1) == 0) {
@@ -521,6 +524,21 @@ static void update_triggers(struct psi_group *group, u64 now,
/* Reset threshold breach flag once event got generated */
t->pending_event = false;
}
+
+ return pending_deadline;
+}
+
+static u64 psi_next_pending_rtpoll_deadline(struct psi_group *group)
+{
+ struct psi_trigger *t;
+ u64 deadline = ULLONG_MAX;
+
+ list_for_each_entry(t, &group->rtpoll_triggers, node) {
+ if (t->pending_event)
+ deadline = min(deadline, t->last_event_time + t->win.size);
+ }
+
+ return deadline;
}
static u64 update_averages(struct psi_group *group, u64 now)
@@ -623,6 +641,28 @@ static void init_rtpoll_triggers(struct psi_group *group, u64 now)
group->rtpoll_next_update = now + group->rtpoll_min_period;
}
+static void psi_rtpoll_reinit_triggers(struct psi_group *group, u64 now)
+{
+ struct psi_trigger *t;
+
+ if (psi_next_pending_rtpoll_deadline(group) == ULLONG_MAX) {
+ init_rtpoll_triggers(group, now);
+ return;
+ }
+
+ /* Rebase expired windows without consuming the current sample. */
+ list_for_each_entry(t, &group->rtpoll_triggers, node) {
+ if (t->pending_event || now - t->win.start_time <= t->win.size)
+ continue;
+
+ /* Preserve the sample without rewinding a newer registration baseline. */
+ window_reset(&t->win, now,
+ max(t->win.start_value,
+ group->rtpoll_total[t->state]), 0);
+ }
+ group->rtpoll_next_update = now;
+}
+
/* Schedule rtpolling if it's not already scheduled or forced. */
static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long delay,
bool force)
@@ -646,46 +686,77 @@ static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long dela
atomic_set(&group->rtpoll_scheduled, 0);
}
+static bool psi_rtpoll_has_active_states(struct psi_group *group)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ struct psi_group_cpu *groupc = per_cpu_ptr(group->pcpu, cpu);
+ unsigned int seq;
+ u32 state_mask;
+
+ do {
+ seq = psi_read_begin(cpu);
+ state_mask = groupc->state_mask;
+ } while (psi_read_retry(cpu, seq));
+
+ if (state_mask & group->rtpoll_states)
+ return true;
+ }
+
+ return false;
+}
+
+static void psi_schedule_rtpoll_passive(struct psi_group *group, u64 deadline,
+ u64 now)
+{
+ u32 changed_states;
+ unsigned long delay;
+
+ group->rtpoll_until = now;
+ group->rtpoll_next_update = deadline;
+ delay = deadline > now ? nsecs_to_jiffies(deadline - now) + 1 : 1;
+ psi_schedule_rtpoll_work(group, delay, true);
+ /* Let a new state change replace this timer with an immediate poll. */
+ atomic_set(&group->rtpoll_scheduled, 0);
+ /* Pair with task-side atomic_xchg() before sampling per-CPU state. */
+ smp_mb();
+
+ /* Pair with state changes that raced the transition to the deadline timer. */
+ collect_percpu_times(group, PSI_POLL, &changed_states);
+ if (!(changed_states & group->rtpoll_states) &&
+ !psi_rtpoll_has_active_states(group))
+ return;
+
+ now = sched_clock();
+ psi_rtpoll_reinit_triggers(group, now);
+ group->rtpoll_until = now + group->rtpoll_min_period * UPDATES_PER_WINDOW;
+ psi_schedule_rtpoll_work(group,
+ nsecs_to_jiffies(group->rtpoll_min_period) + 1,
+ false);
+}
+
static void psi_rtpoll_work(struct psi_group *group)
{
bool force_reschedule = false;
+ bool rtpoll_expired;
u32 changed_states, sampled_states;
- u64 now;
+ u64 now, pending_deadline;
mutex_lock(&group->rtpoll_trigger_lock);
now = sched_clock();
+ rtpoll_expired = now > group->rtpoll_until;
- if (now > group->rtpoll_until) {
+ if (rtpoll_expired) {
/*
- * We are either about to start or might stop rtpolling if no
- * state change was recorded. Resetting rtpoll_scheduled leaves
- * a small window for psi_group_change to sneak in and schedule
- * an immediate rtpoll_work before we get to rescheduling. One
- * potential extra wakeup at the end of the rtpolling window
- * should be negligible and rtpoll_next_update still keeps
- * updates correctly on schedule.
+ * Resetting rtpoll_scheduled leaves a small window for a task
+ * change to race with this worker. The barrier pairs with the
+ * atomic_xchg() in psi_schedule_rtpoll_work(): either this worker
+ * observes the state change, or the task change schedules the worker.
*/
atomic_set(&group->rtpoll_scheduled, 0);
- /*
- * A task change can race with the rtpoll worker that is supposed to
- * report on it. To avoid missing events, ensure ordering between
- * rtpoll_scheduled and the task state accesses, such that if the
- * rtpoll worker misses the state update, the task change is
- * guaranteed to reschedule the rtpoll worker:
- *
- * rtpoll worker:
- * atomic_set(rtpoll_scheduled, 0)
- * smp_mb()
- * LOAD states
- *
- * task change:
- * STORE states
- * if atomic_xchg(rtpoll_scheduled, 1) == 0:
- * schedule rtpoll worker
- *
- * The atomic_xchg() implies a full barrier.
- */
+ /* Pair with task-side atomic_xchg() before sampling per-CPU state. */
smp_mb();
} else {
/* The rtpolling window is not over, keep rescheduling */
@@ -699,9 +770,8 @@ static void psi_rtpoll_work(struct psi_group *group)
changed_states |= sampled_states;
if (changed_states & group->rtpoll_states) {
- /* Initialize trigger windows when entering rtpolling mode */
- if (now > group->rtpoll_until)
- init_rtpoll_triggers(group, now);
+ if (rtpoll_expired)
+ psi_rtpoll_reinit_triggers(group, now);
/*
* Keep the monitor active for at least the duration of the
@@ -712,17 +782,42 @@ static void psi_rtpoll_work(struct psi_group *group)
group->rtpoll_min_period * UPDATES_PER_WINDOW;
}
- if (now > group->rtpoll_until) {
- group->rtpoll_next_update = ULLONG_MAX;
- goto out;
- }
-
if (now >= group->rtpoll_next_update) {
- update_triggers(group, now, PSI_POLL);
+ pending_deadline = update_triggers(group, now, PSI_POLL);
memcpy(group->rtpoll_total, group->total[PSI_POLL], sizeof(group->rtpoll_total));
+
+ if (pending_deadline != ULLONG_MAX &&
+ !(changed_states & group->rtpoll_states)) {
+ psi_schedule_rtpoll_passive(group, pending_deadline, now);
+ goto out;
+ }
+
group->rtpoll_next_update = now + group->rtpoll_min_period;
}
+ if (now > group->rtpoll_until) {
+ pending_deadline = psi_next_pending_rtpoll_deadline(group);
+ if (pending_deadline != ULLONG_MAX) {
+ psi_schedule_rtpoll_passive(group,
+ pending_deadline, now);
+ goto out;
+ }
+
+ if (psi_rtpoll_has_active_states(group)) {
+ now = sched_clock();
+ init_rtpoll_triggers(group, now);
+ group->rtpoll_until = now +
+ group->rtpoll_min_period * UPDATES_PER_WINDOW;
+ psi_schedule_rtpoll_work(group,
+ nsecs_to_jiffies(group->rtpoll_min_period) + 1,
+ false);
+ goto out;
+ }
+
+ group->rtpoll_next_update = ULLONG_MAX;
+ goto out;
+ }
+
psi_schedule_rtpoll_work(group,
nsecs_to_jiffies(group->rtpoll_next_update - now) + 1,
force_reschedule);
@@ -1482,6 +1577,7 @@ void psi_trigger_destroy(struct psi_trigger *t)
/* Destroy rtpoll_task when the last trigger is destroyed */
if (group->rtpoll_states == 0) {
group->rtpoll_until = 0;
+ group->rtpoll_next_update = ULLONG_MAX;
group->rtpoll_pending_states = 0;
task_to_destroy = rcu_dereference_protected(
group->rtpoll_task,
--
2.55.0
^ permalink raw reply [flat|nested] 3+ messages in thread