From: neeraj.upadhyay@kernel.org
To: rcu@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, paulmck@kernel.org,
joel@joelfernandes.org, frederic@kernel.org,
boqun.feng@gmail.com, urezki@gmail.com, rostedt@goodmis.org,
mathieu.desnoyers@efficios.com, jiangshanlai@gmail.com,
qiang.zhang1211@gmail.com, peterz@infradead.org,
neeraj.upadhyay@amd.com,
Neeraj Upadhyay <neeraj.upadhyay@kernel.org>
Subject: [PATCH v2 07/10] rcu/tasks: Check RCU watching state for holdout idle injection tasks
Date: Wed, 9 Oct 2024 18:21:24 +0530 [thread overview]
Message-ID: <20241009125127.18902-8-neeraj.upadhyay@kernel.org> (raw)
In-Reply-To: <20241009125127.18902-1-neeraj.upadhyay@kernel.org>
From: Neeraj Upadhyay <neeraj.upadhyay@kernel.org>
Use RCU watching state of a CPU to check whether RCU-tasks GP
need to wait for idle injection task on that CPU. Idle injection
tasks which are in deep-idle states where RCU is not watching or
which have transitioned to/from deep-idle state do not block
RCU-tasks grace period.
Signed-off-by: Neeraj Upadhyay <neeraj.upadhyay@kernel.org>
---
kernel/rcu/tasks.h | 63 +++++++++++++++++++++++++++++++++++-----------
1 file changed, 48 insertions(+), 15 deletions(-)
diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index d8506d2e6f54..1947f9b6346d 100644
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -38,6 +38,8 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp);
* @rtpp: Pointer to the rcu_tasks structure.
* @rcu_watching_snap: Per-GP RCU-watching snapshot for idle tasks.
* @rcu_watching_snap_rec: RCU-watching snapshot recorded for idle task.
+ * @rcu_watching_idle_inj_snap: Per-GP RCU-watching snapshot for idle inject task.
+ * @rcu_watching_idle_inj_rec: RCU-watching snapshot recorded for idle inject task.
*/
struct rcu_tasks_percpu {
struct rcu_segcblist cblist;
@@ -56,6 +58,8 @@ struct rcu_tasks_percpu {
struct rcu_tasks *rtpp;
int rcu_watching_snap;
bool rcu_watching_snap_rec;
+ int rcu_watching_idle_inj_snap;
+ bool rcu_watching_idle_inj_rec;
};
/**
@@ -989,10 +993,34 @@ static void rcu_tasks_pregp_step(struct list_head *hop)
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, cpu);
rtpcp->rcu_watching_snap_rec = false;
+ rtpcp->rcu_watching_idle_inj_rec = false;
}
}
#ifdef CONFIG_SMP
+static bool rcu_idle_check_rcu_watching(int *rcu_watching_snap, bool *rcu_watching_rec, int cpu)
+{
+ if (!*rcu_watching_rec) {
+ /*
+ * Do plain access. Ordering between remote CPU's pre idle accesses
+ * and post rcu-tasks grace period is provided by synchronize_rcu()
+ * in rcu_tasks_postgp().
+ */
+ *rcu_watching_snap = ct_rcu_watching_cpu(cpu);
+ *rcu_watching_rec = true;
+ if (rcu_watching_snap_in_eqs(*rcu_watching_snap))
+ /*
+ * RCU-idle contexts are RCU-tasks quiescent state for idle
+ * (and idle injection) tasks.
+ */
+ return false;
+ } else if (rcu_watching_snap_stopped_since(cpu, *rcu_watching_snap)) {
+ return false;
+ }
+
+ return true;
+}
+
static bool rcu_idle_task_is_holdout(struct task_struct *t, int cpu)
{
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, cpu);
@@ -1010,22 +1038,16 @@ static bool rcu_idle_task_is_holdout(struct task_struct *t, int cpu)
if (!t->on_cpu)
return false;
- if (!rtpcp->rcu_watching_snap_rec) {
- /*
- * Do plain access. Ordering between remote CPU's pre idle accesses
- * and post rcu-tasks grace period is provided by synchronize_rcu()
- * in rcu_tasks_postgp().
- */
- rtpcp->rcu_watching_snap = ct_rcu_watching_cpu(cpu);
- rtpcp->rcu_watching_snap_rec = true;
- /* RCU-idle contexts are RCU-tasks quiescent state for idle tasks. */
- if (rcu_watching_snap_in_eqs(rtpcp->rcu_watching_snap))
- return false;
- } else if (rcu_watching_snap_stopped_since(cpu, rtpcp->rcu_watching_snap)) {
- return false;
- }
+ return rcu_idle_check_rcu_watching(&rtpcp->rcu_watching_snap,
+ &rtpcp->rcu_watching_snap_rec, cpu);
+}
- return true;
+static bool rcu_idle_inj_is_holdout(struct task_struct *t, int cpu)
+{
+ struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, cpu);
+
+ return rcu_idle_check_rcu_watching(&rtpcp->rcu_watching_idle_inj_snap,
+ &rtpcp->rcu_watching_idle_inj_rec, cpu);
}
#else /* #ifdef CONFIG_SMP */
static inline bool rcu_idle_task_is_holdout(struct task_struct *t, int cpu)
@@ -1037,6 +1059,15 @@ static inline bool rcu_idle_task_is_holdout(struct task_struct *t, int cpu)
*/
return false;
}
+
+static inline bool rcu_idle_inj_is_holdout(struct task_struct *t, int cpu)
+{
+ /*
+ * Idle injection tasks are PF_IDLE within preempt disabled
+ * region. So, we should not enter this call for !SMP.
+ */
+ return false;
+}
#endif
/* Check for quiescent states since the pregp's synchronize_rcu() */
@@ -1060,6 +1091,8 @@ static bool rcu_tasks_is_holdout(struct task_struct *t)
if (t == idle_task(cpu))
return rcu_idle_task_is_holdout(t, cpu);
+ else if (is_idle_task(t))
+ return rcu_idle_inj_is_holdout(t, cpu);
return true;
}
--
2.40.1
next prev parent reply other threads:[~2024-10-09 12:52 UTC|newest]
Thread overview: 12+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-10-09 12:51 [PATCH v2 00/10] Make RCU Tasks scan idle tasks neeraj.upadhyay
2024-10-09 12:51 ` [PATCH v2 01/10] rcu: Change rdp arg to cpu number for rcu_watching_snap_stopped_since() neeraj.upadhyay
2024-10-09 12:51 ` [PATCH v2 02/10] rcu: Make some rcu_watching_* functions global neeraj.upadhyay
2024-10-09 12:51 ` [PATCH v2 03/10] rcu/tasks: Move holdout checks for idle task to a separate function neeraj.upadhyay
2024-10-09 12:51 ` [PATCH v2 04/10] rcu/tasks: Create rcu_idle_task_is_holdout() definition for !SMP neeraj.upadhyay
2024-10-09 12:51 ` [PATCH v2 05/10] rcu/tasks: Consider idle tasks not running on CPU as non-holdouts neeraj.upadhyay
2024-10-09 12:51 ` [PATCH v2 06/10] rcu/tasks: Check RCU watching state for holdout idle tasks neeraj.upadhyay
2024-10-09 12:51 ` neeraj.upadhyay [this message]
2024-10-09 14:37 ` [PATCH v2 07/10] rcu/tasks: Check RCU watching state for holdout idle injection tasks Frederic Weisbecker
2024-10-09 12:51 ` [PATCH v2 08/10] rcu/tasks: Make RCU-tasks pay attention to idle tasks neeraj.upadhyay
2024-10-09 12:51 ` [PATCH v2 09/10] context_tracking: Invoke RCU-tasks enter/exit for NMI context neeraj.upadhyay
2024-10-09 12:51 ` [PATCH v2 10/10] rcu: Allow short-circuiting of synchronize_rcu_tasks_rude() neeraj.upadhyay
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20241009125127.18902-8-neeraj.upadhyay@kernel.org \
--to=neeraj.upadhyay@kernel.org \
--cc=boqun.feng@gmail.com \
--cc=frederic@kernel.org \
--cc=jiangshanlai@gmail.com \
--cc=joel@joelfernandes.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mathieu.desnoyers@efficios.com \
--cc=neeraj.upadhyay@amd.com \
--cc=paulmck@kernel.org \
--cc=peterz@infradead.org \
--cc=qiang.zhang1211@gmail.com \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=urezki@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®