mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: K Prateek Nayak <kprateek.nayak@amd.com>
To: Ingo Molnar <mingo@redhat.com>,
	Peter Zijlstra <peterz@infradead.org>,
	Juri Lelli <juri.lelli@redhat.com>,
	Vincent Guittot <vincent.guittot@linaro.org>,
	Anna-Maria Behnsen <anna-maria@linutronix.de>,
	Frederic Weisbecker <frederic@kernel.org>,
	Thomas Gleixner <tglx@linutronix.de>
Cc: <linux-kernel@vger.kernel.org>,
	Dietmar Eggemann <dietmar.eggemann@arm.com>,
	Steven Rostedt <rostedt@goodmis.org>,
	Ben Segall <bsegall@google.com>, Mel Gorman <mgorman@suse.de>,
	Valentin Schneider <vschneid@redhat.com>,
	K Prateek Nayak <kprateek.nayak@amd.com>,
	"Gautham R. Shenoy" <gautham.shenoy@amd.com>,
	Swapnil Sapkal <swapnil.sapkal@amd.com>,
	Shrikanth Hegde <sshegde@linux.ibm.com>,
	Chen Yu <yu.c.chen@intel.com>
Subject: [RESEND RFC PATCH v2 24/29] sched/fair: Optimize global "nohz.nr_cpus" tracking
Date: Mon, 8 Dec 2025 09:27:10 +0000	[thread overview]
Message-ID: <20251208092744.32737-24-kprateek.nayak@amd.com> (raw)
In-Reply-To: <20251208083602.31898-1-kprateek.nayak@amd.com>

Optimize "nohz.nr_cpus" by tracking number of "sd_nohz->shared" with
non-zero "nr_idle_cpus" count via "nohz.nr_doms" and only updating at
the boundary of "sd_nohz->shared->nr_idle_cpus" going from 0 -> 1 and
back from 1 -> 0.

This also introduces a chance of double accounting when a nohz idle
entry or the tick races with hotplug or cpuset as described in
__nohz_exit_idle_tracking().

__nohz_exit_idle_tracking() called when the sched_domain_shared nodes
tracking idle CPUs are freed is used to correct any potential double
accounting which can unnecessarily trigger nohz idle balances even when
all the CPUs have tick enabled.

Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
 kernel/sched/fair.c     | 63 ++++++++++++++++++++++++++++++++++++-----
 kernel/sched/sched.h    |  1 +
 kernel/sched/topology.c |  1 +
 3 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 46cb88e88b31..f622104d54d7 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7143,7 +7143,7 @@ static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask);
 #ifdef CONFIG_NO_HZ_COMMON
 
 static struct {
-	atomic_t nr_cpus;
+	atomic_t nr_doms;
 	int has_blocked;		/* Idle CPUS has blocked load */
 	int needs_update;		/* Newly idle CPUs need their next_balance collated */
 	unsigned long next_balance;     /* in jiffy units */
@@ -12512,7 +12512,7 @@ static void nohz_balancer_kick(struct rq *rq)
 	 * None are in tickless mode and hence no need for NOHZ idle load
 	 * balancing:
 	 */
-	if (likely(!atomic_read(&nohz.nr_cpus)))
+	if (likely(!atomic_read(&nohz.nr_doms)))
 		return;
 
 	if (READ_ONCE(nohz.has_blocked) &&
@@ -12609,7 +12609,8 @@ static void set_cpu_sd_state_busy(int cpu)
 		return;
 
 	cpumask_clear_cpu(cpu, sd->shared->nohz_idle_cpus_mask);
-	atomic_dec(&sd->shared->nr_idle_cpus);
+	if (!atomic_dec_return(&sd->shared->nr_idle_cpus))
+		atomic_dec(&nohz.nr_doms);
 }
 
 void nohz_balance_exit_idle(struct rq *rq)
@@ -12620,7 +12621,6 @@ void nohz_balance_exit_idle(struct rq *rq)
 		return;
 
 	WRITE_ONCE(rq->nohz_tick_stopped, 0);
-	atomic_dec(&nohz.nr_cpus);
 
 	set_cpu_sd_state_busy(rq->cpu);
 }
@@ -12639,7 +12639,58 @@ static void set_cpu_sd_state_idle(int cpu)
 		return;
 
 	cpumask_set_cpu(cpu, sd->shared->nohz_idle_cpus_mask);
-	atomic_inc(&sd->shared->nr_idle_cpus);
+	if (!atomic_fetch_inc(&sd->shared->nr_idle_cpus))
+		atomic_inc(&nohz.nr_doms);
+}
+
+/*
+ * Correct nohz.nr_doms if sd_nohz->shared was found to have non-zero
+ * nr_idle_cpus when freeing. No local references to sds remain at
+ * this point and the only reference possible via "nohz_shared_list"
+ * will be dropped after the grace period.
+ */
+void __nohz_exit_idle_tracking(struct sched_domain_shared *sds)
+{
+
+	/*
+	 * It is possible for a idle entry to race with sched domain rebuild like:
+	 *
+	 *  CPU0 (hotplug)			CPU1 (nohz idle)
+	 *
+	 *  rq->offline(CPU1)
+	 *    set_cpu_sd_state_busy()
+	 *    rq->sd = sdd;			# Processes IPI, re-enters nohz idle
+	 *    ...				# For old sd_nohz
+	 *    ...				atomic_fetch_inc(&sd_nohz->shared->nr_idle_cpus);
+	 *    ...				atomic_inc(&nohz.nr_doms); # XXX: Accounted once
+	 *    update_top_cache_domains()
+	 *  rq->online(CPU1)
+	 *  # rq->nohz_tick_stopped is true
+	 *  set_cpu_sd_state_idle()
+	 *    # For new sd_nohz
+	 *    atomic_fetch_inc(&sd_nohz->shared->nr_idle_cpus);
+	 *    atomic_inc(&nohz.nr_doms); # XXX: Accounted twice
+	 *  ...
+	 *
+	 * "nohz.nr_doms" is used as an entry criteria in nohz_balancer_kick()
+	 * and this double accounting can lead to wasted idle balancing
+	 * triggers. Use this path to correct the accounting:
+	 *
+	 *  # In sds_delayed_free()
+	 *  __nohz_exit_idle_tracking(sds)
+	 *    # sd->shared->nr_idle_cpus is != 0
+	 *    atomic_dec(&nohz.nr_doms); # XXX: Fixes nohz.nr_doms
+	 */
+	if (atomic_read(&sds->nr_idle_cpus)) {
+		/*
+		 * Reset the "nr_idle_cpus" indicator to prevent
+		 * existing readers from traversing the idle mask
+		 * to reduce chances of traversing the same CPU
+		 * twice.
+		 */
+		atomic_set(&sds->nr_idle_cpus, 0);
+		atomic_dec(&nohz.nr_doms);
+	}
 }
 
 static void cpu_sd_exit_nohz_balance(struct rq *rq)
@@ -12691,8 +12742,6 @@ void nohz_balance_enter_idle(int cpu)
 
 	WRITE_ONCE(rq->nohz_tick_stopped, 1);
 
-	atomic_inc(&nohz.nr_cpus);
-
 	set_cpu_sd_state_idle(cpu);
 
 	/*
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 159c24981ead..3433de20a249 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3266,6 +3266,7 @@ extern void cfs_bandwidth_usage_dec(void);
 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_nohz);
 extern struct list_head nohz_shared_list;
 
+extern void __nohz_exit_idle_tracking(struct sched_domain_shared *sds);
 extern void nohz_balance_exit_idle(struct rq *rq);
 #else /* !CONFIG_NO_HZ_COMMON: */
 static inline void nohz_balance_exit_idle(struct rq *rq) { }
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index af4ce9451cd0..ec549fb7d7fc 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -615,6 +615,7 @@ static int sds_delayed_free(struct sched_domain_shared *sds)
 	scoped_guard(raw_spinlock_irqsave, &nohz_shared_list_lock)
 		list_del_rcu(&sds->nohz_list_node);
 
+	__nohz_exit_idle_tracking(sds);
 	call_rcu(&sds->rcu, destroy_sched_domain_shared_rcu);
 	return 1;
 }
-- 
2.43.0


  parent reply	other threads:[~2025-12-08  9:34 UTC|newest]

Thread overview: 35+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2025-12-08  9:26 [RESEND RFC PATCH v2 00/29] sched/fair: Push-based load balancing K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 01/29] sched/fair: Simplify set_cpu_sd_state_*() with guards K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 02/29] sched/fair: Use rq->nohz_tick_stopped in update_nohz_stats() K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 03/29] sched/topology: Optimize sd->shared allocation and assignment K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 04/29] sched/fair: Simplify the entry condition for update_idle_cpu_scan() K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 05/29] sched/fair: Simplity SIS_UTIL handling in select_idle_cpu() K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 06/29] cpumask: Introduce for_each_cpu_and_wrap() and bitfield helpers K Prateek Nayak
2025-12-12 21:03   ` Yury Norov
2025-12-08  9:26 ` [RESEND RFC PATCH v2 07/29] sched/fair: Use for_each_cpu_and_wrap() in select_idle_capacity() K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 08/29] sched/fair: Use for_each_cpu_and_wrap() in select_idle_cpu() K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 09/29] sched/fair: Rotate the CPU resposible for busy load balancing K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 10/29] sched/fair: Use xchg() to set sd->nohz_idle state K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 11/29] sched/topology: Attach new hierarchy in rq_attach_root() K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 12/29] sched/fair: Fixup sd->nohz_idle state during hotplug / cpuset K Prateek Nayak
2025-12-08  9:26 ` [RESEND RFC PATCH v2 13/29] sched/fair: Account idle cpus instead of busy cpus in sd->shared K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 14/29] sched/topology: Introduce fallback sd->shared assignment K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 15/29] sched/topology: Introduce percpu sd_nohz for nohz state tracking K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 16/29] sched/topology: Introduce "nohz_idle_cpus_mask" in sd->shared K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 17/29] sched/topology: Introduce "nohz_shared_list" to keep track of sd->shared K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 18/29] sched/fair: Reorder the barrier in nohz_balance_enter_idle() K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 19/29] sched/fair: Extract the main _nohz_idle_balance() loop into a helper K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 20/29] sched/fair: Convert find_new_ilb() to use nohz_shared_list K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 21/29] sched/fair: Introduce sched_asym_prefer_idle() for ILB kick K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 22/29] sched/fair: Convert sched_balance_nohz_idle() to use nohz_shared_list K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 23/29] sched/fair: Remove "nohz.idle_cpus_mask" K Prateek Nayak
2025-12-08  9:27 ` K Prateek Nayak [this message]
2025-12-08  9:27 ` [RESEND RFC PATCH v2 25/29] sched/topology: Add basic debug information for "nohz_shared_list" K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 26/29] [EXPERIMENTAL] sched/fair: Add push task framework K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 27/29] [EXPERIMENTAL] sched/fair: Proactive idle balance using push mechanism K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 28/29] [EXPERIMENTAL] sched/fair: Add a local counter to rate limit task push K Prateek Nayak
2025-12-08 12:33   ` Christian Loehle
2025-12-08 17:35     ` K Prateek Nayak
2025-12-08  9:27 ` [RESEND RFC PATCH v2 29/29] [EXPERIMENTAL] sched/fair: Faster alternate for intra-NUMA newidle balance K Prateek Nayak
2025-12-08 14:04 ` [RESEND RFC PATCH v2 00/29] sched/fair: Push-based load balancing Shrikanth Hegde
2025-12-08 17:36   ` K Prateek Nayak

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20251208092744.32737-24-kprateek.nayak@amd.com \
    --to=kprateek.nayak@amd.com \
    --cc=anna-maria@linutronix.de \
    --cc=bsegall@google.com \
    --cc=dietmar.eggemann@arm.com \
    --cc=frederic@kernel.org \
    --cc=gautham.shenoy@amd.com \
    --cc=juri.lelli@redhat.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=mgorman@suse.de \
    --cc=mingo@redhat.com \
    --cc=peterz@infradead.org \
    --cc=rostedt@goodmis.org \
    --cc=sshegde@linux.ibm.com \
    --cc=swapnil.sapkal@amd.com \
    --cc=tglx@linutronix.de \
    --cc=vincent.guittot@linaro.org \
    --cc=vschneid@redhat.com \
    --cc=yu.c.chen@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®