mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Bo Li <libo.gcs85@bytedance.com>
To: tglx@linutronix.de, mingo@redhat.com, bp@alien8.de,
	dave.hansen@linux.intel.com, x86@kernel.org, luto@kernel.org,
	kees@kernel.org, akpm@linux-foundation.org, david@redhat.com,
	juri.lelli@redhat.com, vincent.guittot@linaro.org,
	peterz@infradead.org
Cc: dietmar.eggemann@arm.com, hpa@zytor.com, acme@kernel.org,
	namhyung@kernel.org, mark.rutland@arm.com,
	alexander.shishkin@linux.intel.com, jolsa@kernel.org,
	irogers@google.com, adrian.hunter@intel.com,
	kan.liang@linux.intel.com, viro@zeniv.linux.org.uk,
	brauner@kernel.org, jack@suse.cz, lorenzo.stoakes@oracle.com,
	Liam.Howlett@oracle.com, vbabka@suse.cz, rppt@kernel.org,
	surenb@google.com, mhocko@suse.com, rostedt@goodmis.org,
	bsegall@google.com, mgorman@suse.de, vschneid@redhat.com,
	jannh@google.com, pfalcato@suse.de, riel@surriel.com,
	harry.yoo@oracle.com, linux-kernel@vger.kernel.org,
	linux-perf-users@vger.kernel.org, linux-fsdevel@vger.kernel.org,
	linux-mm@kvack.org, duanxiongchun@bytedance.com,
	yinhongbo@bytedance.com, dengliang.1214@bytedance.com,
	xieyongji@bytedance.com, chaiwen.cc@bytedance.com,
	songmuchun@bytedance.com, yuanzhu@bytedance.com,
	chengguozhu@bytedance.com, sunjiadong.lff@bytedance.com,
	Bo Li <libo.gcs85@bytedance.com>
Subject: [RFC v2 31/35] RPAL: add receiver waker
Date: Fri, 30 May 2025 17:27:59 +0800	[thread overview]
Message-ID: <198278a03d91ab7e0e17d782c657da85cff741bb.1748594841.git.libo.gcs85@bytedance.com> (raw)
In-Reply-To: <cover.1748594840.git.libo.gcs85@bytedance.com>

In an RPAL call, the receiver thread is in the TASK_INTERRUPTIBLE state
and cannot be awakened, which may lead to missed wakeups. For example, if
no kernel event occurs during the entire RPAL call, the receiver thread
will remain in the TASK_INTERRUPTIBLE state after the RPAL call completes.

To address this issue, RPAL adds a flag to the receiver whenever it
encounters an unawakened state and introduces a "waker" work. The waker
work runs automatically on every tick to check for receiver threads that
have missed wakeups. If any are found, it wakes them up. For epoll, the
waker also checks for pending user mode events and wakes the receiver
thread if such events exist.

Signed-off-by: Bo Li <libo.gcs85@bytedance.com>
---
 arch/x86/rpal/internal.h |  4 ++
 arch/x86/rpal/service.c  | 98 ++++++++++++++++++++++++++++++++++++++++
 arch/x86/rpal/thread.c   |  3 ++
 include/linux/rpal.h     | 11 +++++
 kernel/sched/core.c      |  3 ++
 5 files changed, 119 insertions(+)

diff --git a/arch/x86/rpal/internal.h b/arch/x86/rpal/internal.h
index e03f8a90619d..117357dabdec 100644
--- a/arch/x86/rpal/internal.h
+++ b/arch/x86/rpal/internal.h
@@ -22,6 +22,10 @@ int rpal_enable_service(unsigned long arg);
 int rpal_disable_service(void);
 int rpal_request_service(unsigned long arg);
 int rpal_release_service(u64 key);
+void rpal_insert_wake_list(struct rpal_service *rs,
+			   struct rpal_receiver_data *rrd);
+void rpal_remove_wake_list(struct rpal_service *rs,
+			   struct rpal_receiver_data *rrd);
 
 /* mm.c */
 static inline struct rpal_shared_page *
diff --git a/arch/x86/rpal/service.c b/arch/x86/rpal/service.c
index 9fd568fa9a29..6fefb7a7729c 100644
--- a/arch/x86/rpal/service.c
+++ b/arch/x86/rpal/service.c
@@ -143,6 +143,99 @@ static void delete_service(struct rpal_service *rs)
 	spin_unlock_irqrestore(&hash_table_lock, flags);
 }
 
+void rpal_insert_wake_list(struct rpal_service *rs,
+			   struct rpal_receiver_data *rrd)
+{
+	unsigned long flags;
+	struct rpal_waker_struct *waker = &rs->waker;
+
+	spin_lock_irqsave(&waker->lock, flags);
+	list_add_tail(&rrd->wake_list, &waker->wake_head);
+	spin_unlock_irqrestore(&waker->lock, flags);
+	pr_debug("rpal debug: [%d] insert wake list\n", current->pid);
+}
+
+void rpal_remove_wake_list(struct rpal_service *rs,
+			   struct rpal_receiver_data *rrd)
+{
+	unsigned long flags;
+	struct rpal_waker_struct *waker = &rs->waker;
+
+	spin_lock_irqsave(&waker->lock, flags);
+	list_del(&rrd->wake_list);
+	spin_unlock_irqrestore(&waker->lock, flags);
+	pr_debug("rpal debug: [%d] remove wake list\n", current->pid);
+}
+
+/* waker->lock must be hold */
+static inline void rpal_wake_all(struct rpal_waker_struct *waker)
+{
+	struct task_struct *wake_list[RPAL_MAX_RECEIVER_NUM];
+	struct list_head *list;
+	unsigned long flags;
+	int i, cnt = 0;
+
+	spin_lock_irqsave(&waker->lock, flags);
+	list_for_each(list, &waker->wake_head) {
+		struct task_struct *task;
+		struct rpal_receiver_call_context *rcc;
+		struct rpal_receiver_data *rrd;
+		int pending;
+
+		rrd = list_entry(list, struct rpal_receiver_data, wake_list);
+		task = rrd->rcd.bp_task;
+		rcc = rrd->rcc;
+
+		pending = atomic_read(&rcc->ep_pending) & RPAL_USER_PENDING;
+
+		if (rpal_test_task_thread_flag(task, RPAL_WAKE_BIT) ||
+		    (pending && atomic_cmpxchg(&rcc->receiver_state,
+					       RPAL_RECEIVER_STATE_WAIT,
+					       RPAL_RECEIVER_STATE_RUNNING) ==
+					RPAL_RECEIVER_STATE_WAIT)) {
+			wake_list[cnt] = task;
+			cnt++;
+		}
+	}
+	spin_unlock_irqrestore(&waker->lock, flags);
+
+	for (i = 0; i < cnt; i++)
+		wake_up_process(wake_list[i]);
+}
+
+static void rpal_wake_callback(struct work_struct *work)
+{
+	struct rpal_waker_struct *waker =
+		container_of(work, struct rpal_waker_struct, waker_work.work);
+
+	rpal_wake_all(waker);
+	/* We check it every ticks */
+	schedule_delayed_work(&waker->waker_work, 1);
+}
+
+static void rpal_enable_waker(struct rpal_waker_struct *waker)
+{
+	INIT_DELAYED_WORK(&waker->waker_work, rpal_wake_callback);
+	schedule_delayed_work(&waker->waker_work, 1);
+	pr_debug("rpal debug: [%d] enable waker\n", current->pid);
+}
+
+static void rpal_disable_waker(struct rpal_waker_struct *waker)
+{
+	unsigned long flags;
+	struct list_head *p, *n;
+
+	cancel_delayed_work_sync(&waker->waker_work);
+	rpal_wake_all(waker);
+	spin_lock_irqsave(&waker->lock, flags);
+	list_for_each_safe(p, n, &waker->wake_head) {
+		list_del_init(p);
+	}
+	INIT_LIST_HEAD(&waker->wake_head);
+	spin_unlock_irqrestore(&waker->lock, flags);
+	pr_debug("rpal debug: [%d] disable waker\n", current->pid);
+}
+
 static inline unsigned long calculate_base_address(int id)
 {
 	return RPAL_ADDRESS_SPACE_LOW + RPAL_ADDR_SPACE_SIZE * id;
@@ -213,6 +306,10 @@ struct rpal_service *rpal_register_service(void)
 	rs->pku_on = PKU_ON_FALSE;
 	rpal_service_pku_init();
 #endif
+	spin_lock_init(&rs->waker.lock);
+	INIT_LIST_HEAD(&rs->waker.wake_head);
+	/* receiver may miss wake up if in lazy switch, try to wake it later */
+	rpal_enable_waker(&rs->waker);
 
 	rs->bad_service = false;
 	rs->base = calculate_base_address(rs->id);
@@ -257,6 +354,7 @@ void rpal_unregister_service(struct rpal_service *rs)
 		schedule();
 
 	delete_service(rs);
+	rpal_disable_waker(&rs->waker);
 
 	pr_debug("rpal: unregister service, id: %d, tgid: %d\n", rs->id,
 		 rs->group_leader->tgid);
diff --git a/arch/x86/rpal/thread.c b/arch/x86/rpal/thread.c
index fcc592baaac0..51c9eec639cb 100644
--- a/arch/x86/rpal/thread.c
+++ b/arch/x86/rpal/thread.c
@@ -186,6 +186,8 @@ int rpal_register_receiver(unsigned long addr)
 	current->rpal_rd = rrd;
 	rpal_set_current_thread_flag(RPAL_RECEIVER_BIT);
 
+	rpal_insert_wake_list(cur, rrd);
+
 	atomic_inc(&cur->thread_cnt);
 
 	return 0;
@@ -214,6 +216,7 @@ int rpal_unregister_receiver(void)
 	clear_fs_tsk_map();
 
 	rpal_put_shared_page(rrd->rsp);
+	rpal_remove_wake_list(cur, rrd);
 	rpal_clear_current_thread_flag(RPAL_RECEIVER_BIT);
 	rpal_free_thread_pending(&rrd->rcd);
 	kfree(rrd);
diff --git a/include/linux/rpal.h b/include/linux/rpal.h
index 16a3c80383f7..1d8c1bdc90f2 100644
--- a/include/linux/rpal.h
+++ b/include/linux/rpal.h
@@ -116,6 +116,7 @@ enum rpal_task_flag_bits {
 	RPAL_RECEIVER_BIT,
 	RPAL_CPU_LOCKED_BIT,
 	RPAL_LAZY_SWITCHED_BIT,
+	RPAL_WAKE_BIT,
 };
 
 enum rpal_receiver_state {
@@ -189,6 +190,12 @@ struct rpal_fsbase_tsk_map {
 	struct task_struct *tsk;
 };
 
+struct rpal_waker_struct {
+	spinlock_t lock;
+	struct list_head wake_head;
+	struct delayed_work waker_work;
+};
+
 /*
  * Each RPAL process (a.k.a RPAL service) should have a pointer to
  * struct rpal_service in all its tasks' task_struct.
@@ -255,6 +262,9 @@ struct rpal_service {
 	int pkey;
 #endif
 
+	/* receiver thread waker */
+	struct rpal_waker_struct waker;
+
 	/* delayed service put work */
 	struct delayed_work delayed_put_work;
 
@@ -347,6 +357,7 @@ struct rpal_receiver_data {
 	struct fd f;
 	struct hrtimer_sleeper ep_sleeper;
 	wait_queue_entry_t ep_wait;
+	struct list_head wake_list;
 };
 
 struct rpal_sender_data {
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 486d59bdd3fc..c219ada29d34 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3943,6 +3943,7 @@ static bool rpal_check_state(struct task_struct *p)
 		struct rpal_receiver_call_context *rcc = p->rpal_rd->rcc;
 		int state;
 
+		rpal_clear_task_thread_flag(p, RPAL_WAKE_BIT);
 retry:
 		state = atomic_read(&rcc->receiver_state) & RPAL_RECEIVER_STATE_MASK;
 		switch (state) {
@@ -3957,6 +3958,7 @@ static bool rpal_check_state(struct task_struct *p)
 		case RPAL_RECEIVER_STATE_RUNNING:
 			break;
 		case RPAL_RECEIVER_STATE_CALL:
+			rpal_set_task_thread_flag(p, RPAL_WAKE_BIT);
 			ret = false;
 			break;
 		default:
@@ -4522,6 +4524,7 @@ int rpal_try_to_wake_up(struct task_struct *p)
 
 	BUG_ON(READ_ONCE(p->__state) == TASK_RUNNING);
 
+	rpal_clear_task_thread_flag(p, RPAL_WAKE_BIT);
 	scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {
 		smp_mb__after_spinlock();
 		if (!ttwu_state_match(p, TASK_NORMAL, &success))
-- 
2.20.1


  parent reply	other threads:[~2025-05-30  9:36 UTC|newest]

Thread overview: 46+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2025-05-30  9:27 [RFC v2 00/35] optimize cost of inter-process communication Bo Li
2025-05-30  9:27 ` [RFC v2 01/35] Kbuild: rpal support Bo Li
2025-05-30  9:27 ` [RFC v2 02/35] RPAL: add struct rpal_service Bo Li
2025-05-30  9:27 ` [RFC v2 03/35] RPAL: add service registration interface Bo Li
2025-05-30  9:27 ` [RFC v2 04/35] RPAL: add member to task_struct and mm_struct Bo Li
2025-05-30  9:27 ` [RFC v2 05/35] RPAL: enable virtual address space partitions Bo Li
2025-05-30  9:27 ` [RFC v2 06/35] RPAL: add user interface Bo Li
2025-05-30  9:27 ` [RFC v2 07/35] RPAL: enable shared page mmap Bo Li
2025-05-30  9:27 ` [RFC v2 08/35] RPAL: enable sender/receiver registration Bo Li
2025-05-30  9:27 ` [RFC v2 09/35] RPAL: enable address space sharing Bo Li
2025-05-30  9:27 ` [RFC v2 10/35] RPAL: allow service enable/disable Bo Li
2025-05-30  9:27 ` [RFC v2 11/35] RPAL: add service request/release Bo Li
2025-05-30  9:27 ` [RFC v2 12/35] RPAL: enable service disable notification Bo Li
2025-05-30  9:27 ` [RFC v2 13/35] RPAL: add tlb flushing support Bo Li
2025-05-30  9:27 ` [RFC v2 14/35] RPAL: enable page fault handling Bo Li
2025-05-30 13:59   ` Dave Hansen
2025-05-30  9:27 ` [RFC v2 15/35] RPAL: add sender/receiver state Bo Li
2025-05-30  9:27 ` [RFC v2 16/35] RPAL: add cpu lock interface Bo Li
2025-05-30  9:27 ` [RFC v2 17/35] RPAL: add a mapping between fsbase and tasks Bo Li
2025-05-30  9:27 ` [RFC v2 18/35] sched: pick a specified task Bo Li
2025-05-30  9:27 ` [RFC v2 19/35] RPAL: add lazy switch main logic Bo Li
2025-05-30  9:27 ` [RFC v2 20/35] RPAL: add rpal_ret_from_lazy_switch Bo Li
2025-05-30  9:27 ` [RFC v2 21/35] RPAL: add kernel entry handling for lazy switch Bo Li
2025-05-30  9:27 ` [RFC v2 22/35] RPAL: rebuild receiver state Bo Li
2025-05-30  9:27 ` [RFC v2 23/35] RPAL: resume cpumask when fork Bo Li
2025-05-30  9:27 ` [RFC v2 24/35] RPAL: critical section optimization Bo Li
2025-05-30  9:27 ` [RFC v2 25/35] RPAL: add MPK initialization and interface Bo Li
2025-05-30  9:27 ` [RFC v2 26/35] RPAL: enable MPK support Bo Li
2025-05-30 17:03   ` Dave Hansen
2025-05-30  9:27 ` [RFC v2 27/35] RPAL: add epoll support Bo Li
2025-05-30  9:27 ` [RFC v2 28/35] RPAL: add rpal_uds_fdmap() support Bo Li
2025-05-30  9:27 ` [RFC v2 29/35] RPAL: fix race condition in pkru update Bo Li
2025-05-30  9:27 ` [RFC v2 30/35] RPAL: fix pkru setup when fork Bo Li
2025-05-30  9:27 ` Bo Li [this message]
2025-05-30  9:28 ` [RFC v2 32/35] RPAL: fix unknown nmi on AMD CPU Bo Li
2025-05-30  9:28 ` [RFC v2 33/35] RPAL: enable time slice correction Bo Li
2025-05-30  9:28 ` [RFC v2 34/35] RPAL: enable fast epoll wait Bo Li
2025-05-30  9:28 ` [RFC v2 35/35] samples/rpal: add RPAL samples Bo Li
2025-05-30  9:33 ` [RFC v2 00/35] optimize cost of inter-process communication Lorenzo Stoakes
2025-06-03  8:22   ` Bo Li
2025-06-03  9:22     ` Lorenzo Stoakes
2025-05-30  9:41 ` Pedro Falcato
2025-05-30  9:56 ` David Hildenbrand
2025-05-30 22:42 ` Andrew Morton
2025-05-31  7:16 ` Ingo Molnar
2025-06-03 17:49 ` H. Peter Anvin

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=198278a03d91ab7e0e17d782c657da85cff741bb.1748594841.git.libo.gcs85@bytedance.com \
    --to=libo.gcs85@bytedance.com \
    --cc=Liam.Howlett@oracle.com \
    --cc=acme@kernel.org \
    --cc=adrian.hunter@intel.com \
    --cc=akpm@linux-foundation.org \
    --cc=alexander.shishkin@linux.intel.com \
    --cc=bp@alien8.de \
    --cc=brauner@kernel.org \
    --cc=bsegall@google.com \
    --cc=chaiwen.cc@bytedance.com \
    --cc=chengguozhu@bytedance.com \
    --cc=dave.hansen@linux.intel.com \
    --cc=david@redhat.com \
    --cc=dengliang.1214@bytedance.com \
    --cc=dietmar.eggemann@arm.com \
    --cc=duanxiongchun@bytedance.com \
    --cc=harry.yoo@oracle.com \
    --cc=hpa@zytor.com \
    --cc=irogers@google.com \
    --cc=jack@suse.cz \
    --cc=jannh@google.com \
    --cc=jolsa@kernel.org \
    --cc=juri.lelli@redhat.com \
    --cc=kan.liang@linux.intel.com \
    --cc=kees@kernel.org \
    --cc=linux-fsdevel@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-mm@kvack.org \
    --cc=linux-perf-users@vger.kernel.org \
    --cc=lorenzo.stoakes@oracle.com \
    --cc=luto@kernel.org \
    --cc=mark.rutland@arm.com \
    --cc=mgorman@suse.de \
    --cc=mhocko@suse.com \
    --cc=mingo@redhat.com \
    --cc=namhyung@kernel.org \
    --cc=peterz@infradead.org \
    --cc=pfalcato@suse.de \
    --cc=riel@surriel.com \
    --cc=rostedt@goodmis.org \
    --cc=rppt@kernel.org \
    --cc=songmuchun@bytedance.com \
    --cc=sunjiadong.lff@bytedance.com \
    --cc=surenb@google.com \
    --cc=tglx@linutronix.de \
    --cc=vbabka@suse.cz \
    --cc=vincent.guittot@linaro.org \
    --cc=viro@zeniv.linux.org.uk \
    --cc=vschneid@redhat.com \
    --cc=x86@kernel.org \
    --cc=xieyongji@bytedance.com \
    --cc=yinhongbo@bytedance.com \
    --cc=yuanzhu@bytedance.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®