[RFC PATCH 07/12] futex: Make FUTEX_*_PING use Proxy Execution.

[Suleiman Souhlal <suleiman@google.com>]

Set the locker side proxy execution bits, so that a task blocked on
a PING futex can act as a donor.

Signed-off-by: Suleiman Souhlal <suleiman@google.com>
---
 include/linux/futex.h | 11 +++++++++++
 include/linux/sched.h | 12 ++++++++++++
 kernel/futex/ping.c   |  7 +++++++
 kernel/sched/core.c   |  7 +++++++
 4 files changed, 37 insertions(+)

diff --git a/include/linux/futex.h b/include/linux/futex.h
index b1b422b480fb..84eca792e44a 100644
--- a/include/linux/futex.h
+++ b/include/linux/futex.h
@@ -170,4 +170,15 @@ static inline struct task_struct *ping_mutex_owner(struct ping_mutex *ping_mutex
 	return READ_ONCE(ping_mutex->owner);
 }
 
+static inline void ping_mutex_lock_wait_lock(struct ping_mutex *ping_mutex)
+{
+	lockdep_assert_irqs_disabled();
+	raw_spin_lock(&ping_mutex->wait_lock);
+}
+
+static inline void ping_mutex_unlock_wait_lock(struct ping_mutex *ping_mutex)
+{
+	raw_spin_unlock(&ping_mutex->wait_lock);
+}
+
 #endif /* _LINUX_FUTEX_H */
diff --git a/include/linux/sched.h b/include/linux/sched.h
index a7de5c496e3c..200f41c38333 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -835,6 +835,7 @@ struct task_ipi_mask { };
 enum blocked_on_type {
 	BO_T_NONE,
 	BO_T_MUTEX,
+	BO_T_PING_FUTEX,
 };
 
 struct blocked_on_lock {
@@ -2257,6 +2258,13 @@ static inline void __set_task_blocked_on(struct task_struct *p, void *m,
 	p->blocked_on.type = type;
 }
 
+static inline void set_task_blocked_on(struct task_struct *p, void *m,
+				       enum blocked_on_type type)
+{
+	guard(raw_spinlock_irqsave)(&p->blocked_lock);
+	__set_task_blocked_on(p, m, type);
+}
+
 static inline void __clear_task_blocked_on(struct task_struct *p, void *m)
 {
 	/* Currently we serialize blocked_on under the task::blocked_lock */
@@ -2278,6 +2286,10 @@ static inline void clear_task_blocked_on(struct task_struct *p, void *m)
 }
 
 #else
+static inline void set_task_blocked_on(struct task_struct *p, void *m,
+				       enum blocked_on_type type)
+{
+}
 static inline void __clear_task_blocked_on(struct task_struct *p, void *m)
 {
 }
diff --git a/kernel/futex/ping.c b/kernel/futex/ping.c
index ebcd3c4a7793..689f149f7150 100644
--- a/kernel/futex/ping.c
+++ b/kernel/futex/ping.c
@@ -370,6 +370,9 @@ int futex_lock_ping(u32 __user *uaddr, unsigned int flags, ktime_t *time,
 
 		queued = false;
 		while (1) {
+			set_task_blocked_on(current, &q.ping_state->ping_mutex,
+					    BO_T_PING_FUTEX);
+
 			set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
 			if (!queued) {
 				futex_queue(&q, hb, current);
@@ -382,6 +385,9 @@ int futex_lock_ping(u32 __user *uaddr, unsigned int flags, ktime_t *time,
 
 			futex_do_wait(&q, to);
 
+			clear_task_blocked_on(current,
+					      &q.ping_state->ping_mutex);
+
 			futex_q_lockptr_lock(&q);
 			if (to && !to->task) {
 				ret = -ETIMEDOUT;
@@ -511,6 +517,7 @@ int futex_unlock_ping(u32 __user *uaddr, unsigned int flags)
 	/* Leave it queued, it gets unqueued on the lock side */
 	get_task_struct(top_waiter->task);
 	wake_q_add_safe(&wake_q, top_waiter->task);
+	clear_task_blocked_on(top_waiter->task, &ping_state->ping_mutex);
 	spin_unlock(&hb->lock);
 
 	/*
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 2e8fe4b9bb88..1d35b1d90ea2 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -68,6 +68,7 @@
 #include <linux/wait_api.h>
 #include <linux/workqueue_api.h>
 #include <linux/livepatch_sched.h>
+#include <linux/futex.h>
 
 #ifdef CONFIG_PREEMPT_DYNAMIC
 # ifdef CONFIG_GENERIC_IRQ_ENTRY
@@ -158,6 +159,8 @@ static inline struct task_struct *__blocked_on_owner(struct blocked_on_lock *bo)
 		return NULL;
 	case BO_T_MUTEX:
 		return __mutex_owner(bo->lock);
+	case BO_T_PING_FUTEX:
+		return ping_mutex_owner(bo->lock);
 	default:
 		WARN_ON_ONCE(1);
 		return NULL;
@@ -6907,6 +6910,8 @@ lock_blocked_on_lock(struct blocked_on_lock *bo)
 {
 	if (bo->type == BO_T_MUTEX)
 		raw_spin_lock(&((struct mutex *)bo->lock)->wait_lock);
+	else if (bo->type == BO_T_PING_FUTEX)
+		ping_mutex_lock_wait_lock(bo->lock);
 	else
 		WARN_ON_ONCE(1);
 }
@@ -6916,6 +6921,8 @@ unlock_blocked_on_lock(struct blocked_on_lock *bo)
 {
 	if (bo->type == BO_T_MUTEX)
 		raw_spin_unlock(&((struct mutex *)bo->lock)->wait_lock);
+	else if (bo->type == BO_T_PING_FUTEX)
+		ping_mutex_unlock_wait_lock(bo->lock);
 	else
 		WARN_ON_ONCE(1);
 }
-- 
2.55.0.1082.g2b9226bbc0-goog