[RFC PATCH 11/12] futex: Allow userspace stealing for PING futexes.

[Suleiman Souhlal <suleiman@google.com>]

Similarly to how the kernel part of PING locking can steal the futex
from the top waiter, it is also possible for userspace to also take
advantage of this and try to steal without going to the kernel.

When a new (contending) locker notices that the lock has been stolen
from userspace, the ownership of the ping_state and the ping_mutex
are fixed up to the real owner.

TODO: Verify that the case where the ping_state owner exits with the
  futex having been user stolen is handled correctly.
  In other words, when ping_state->owner = task getting killed,
  ping_mutex.owner = NULL and uval = real owner (set by userspace).
  Right now, it seems like we might be doing the wrong thing in such
  cases. exit_ping_state_list() needs to detect such situations
  (by walking the uvals?) and fixup the ownerships.

Signed-off-by: Suleiman Souhlal <suleiman@google.com>
---
 kernel/futex/core.c  |  8 ++++++++
 kernel/futex/futex.h |  2 ++
 kernel/futex/pi.c    | 12 ++++++++++--
 kernel/futex/ping.c  | 38 +++++++++++++++++++++++++++++++++++---
 4 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index 56c7e2d5faac..023531c4b45a 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -1445,6 +1445,14 @@ static void exit_ping_state_list(struct task_struct *curr)
 	while (!list_empty(head)) {
 		next = head->next;
 		ping_state = list_entry(next, struct futex_pi_state, list);
+		/*
+		 * XXX In the case when we are ping_state owner but
+		 * the futex is actually owned by someone who stole it
+		 * from userspace, is setting ping_state.owner = NULL here
+		 * enough? Probably not, otherwise the ping_state could leak
+		 * if they also exit before unlocking or someone else
+		 * fixing up the ownership.
+		 */
 		if (1) {
 			CLASS(hbr, hbr)(&key);
 			auto hb = hbr.hb;
diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h
index 113289779dd1..922d130c5b44 100644
--- a/kernel/futex/futex.h
+++ b/kernel/futex/futex.h
@@ -413,6 +413,8 @@ extern int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,
 			      bool ping);
 extern void get_ping_state(struct futex_pi_state *ping_state);
 extern void put_ping_state(struct futex_pi_state *ping_state);
+extern int fixup_ping_owner_after_user_steal(struct futex_pi_state *ping_state,
+				      u32 __user *uaddr, u32 uval);
 
 /*
  * Express the locking dependencies for lockdep:
diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c
index e7e6e347f97d..dfd2da5188b0 100644
--- a/kernel/futex/pi.c
+++ b/kernel/futex/pi.c
@@ -348,8 +348,16 @@ int attach_to_pi_state(u32 __user *uaddr, u32 uval,
 	 * state exists then the owner TID must be the same as the
 	 * user space TID. [9/10]
 	 */
-	if (pid != task_pid_vnr(pi_state->owner))
-		goto out_einval;
+	if (pid != task_pid_vnr(pi_state->owner)) {
+		if (!ping) {
+			goto out_einval;
+		} else {
+			ret = fixup_ping_owner_after_user_steal(pi_state,
+								uaddr, uval);
+			if (ret)
+				goto out_error;
+		}
+	}
 
 out_attach:
 	if (!ping) {
diff --git a/kernel/futex/ping.c b/kernel/futex/ping.c
index e5765b2d3834..4cef1c46b4c6 100644
--- a/kernel/futex/ping.c
+++ b/kernel/futex/ping.c
@@ -96,6 +96,22 @@ put_ping_state(struct futex_pi_state *ping_state)
 	}
 }
 
+int fixup_ping_owner_after_user_steal(struct futex_pi_state *ping_state,
+				      u32 __user *uaddr, u32 uval)
+{
+	struct task_struct *p;
+
+	p = find_get_task_by_vpid(uval & FUTEX_TID_MASK);
+	if (p == NULL)
+		return pi_handle_exit_race(uaddr, uval);
+	if (unlikely(p->flags & PF_KTHREAD))
+		return -EPERM;
+	ping_state_update_owner(ping_state, p);
+	WRITE_ONCE(ping_state->ping_mutex.owner, p);
+
+	return 0;
+}
+
 static void futex_unqueue_ping(struct futex_q *q)
 {
 	if (!plist_node_empty(&q->list))
@@ -149,6 +165,7 @@ static int futex_trylock_ping_state(u32 __user *uaddr,
 				    bool handoff)
 {
 	struct task_struct *owner;
+	pid_t pid;
 	u32 uval, new, newtid;
 	int ret;
 
@@ -163,8 +180,17 @@ static int futex_trylock_ping_state(u32 __user *uaddr,
 
 		WARN_ON_ONCE(ping_state->handoff || ping_state->pickup);
 
-		if (uval & FUTEX_TID_MASK) {
-			ret = -EAGAIN;
+		/*
+		 * No owner but a userspace TID means that it got stolen
+		 * from userspace.
+		 * Fix up the ownership.
+		 */
+		pid = uval & FUTEX_TID_MASK;
+		if (pid) {
+			ret = fixup_ping_owner_after_user_steal(ping_state,
+								uaddr, uval);
+			if (ret == 0)
+				ret = -EAGAIN;
 			goto err;
 		}
 		new = newtid | FUTEX_WAITERS;
@@ -203,7 +229,7 @@ static int futex_trylock_ping_state(u32 __user *uaddr,
 	case -EINVAL:
 		break;
 	default:
-		WARN_ON(1);
+		break;
 	}
 	return ret;
 }
@@ -619,6 +645,12 @@ int futex_unlock_ping(u32 __user *uaddr, unsigned int flags)
 		goto out_unlock;
 	raw_spin_lock_irq(&ping_state->ping_mutex.wait_lock);
 
+	/*
+	 * If we're unlocking a lock we stole from userspace,
+	 * it's possible that we don't own the ping_state or the
+	 * ping_mutex. But we'll give them to the next task here anyway.
+	 */
+
 	next = ping_current_proxy_donor(ping_state);
 	get_ping_state(ping_state);
 	/* Leave it queued, it gets unqueued on the lock side */
-- 
2.55.0.1082.g2b9226bbc0-goog