[RFC PATCH 12/17] perf: Track pinned events per user

[Alexander Shishkin <alexander.shishkin@linux.intel.com>]

Maintain a per-user cpu-indexed array of shmemfs-backed events, same
way as mlock accounting.

Signed-off-by: Alexander Shishkin <alexander.shishkin@linux.intel.com>
---
 include/linux/sched/user.h  |  6 ++++
 kernel/events/core.c        | 14 ++++-----
 kernel/events/ring_buffer.c | 69 +++++++++++++++++++++++++++++++++++++--------
 kernel/user.c               |  1 +
 4 files changed, 71 insertions(+), 19 deletions(-)

diff --git a/include/linux/sched/user.h b/include/linux/sched/user.h
index 5d5415e129..bf10f95250 100644
--- a/include/linux/sched/user.h
+++ b/include/linux/sched/user.h
@@ -5,6 +5,7 @@
 #include <linux/atomic.h>
 
 struct key;
+struct perf_event;
 
 /*
  * Some day this will be a full-fledged user tracking system..
@@ -39,6 +40,11 @@ struct user_struct {
 #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL)
 	atomic_long_t locked_vm;
 #endif
+#ifdef CONFIG_PERF_EVENTS
+	atomic_long_t nr_pinnable_events;
+	struct mutex pinned_mutex;
+	struct perf_event ** __percpu pinned_events;
+#endif
 };
 
 extern int uids_sysfs_init(void);
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 1fed69d4ba..e00f1f6aaf 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -384,7 +384,6 @@ static atomic_t perf_sched_count;
 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
 static DEFINE_PER_CPU(int, perf_sched_cb_usages);
 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
-static DEFINE_PER_CPU(struct perf_event *, shmem_events);
 
 static atomic_t nr_mmap_events __read_mostly;
 static atomic_t nr_comm_events __read_mostly;
@@ -2086,7 +2085,8 @@ enum pin_event_t {
 
 static enum pin_event_t pin_event_pages(struct perf_event *event)
 {
-	struct perf_event **pinned_event = this_cpu_ptr(&shmem_events);
+	struct user_struct *user = event->rb->mmap_user;
+	struct perf_event **pinned_event = this_cpu_ptr(user->pinned_events);
 	struct perf_event *old_event = *pinned_event;
 
 	if (old_event == event)
@@ -4281,13 +4281,14 @@ static void _free_event(struct perf_event *event)
 	unaccount_event(event);
 
 	if (event->attach_state & PERF_ATTACH_SHMEM) {
+		struct user_struct *user = event->rb->mmap_user;
 		struct perf_event_context *ctx = event->ctx;
 		int cpu;
 
 		atomic_set(&event->xpinned, 0);
 		for_each_possible_cpu(cpu) {
 			struct perf_event **pinned_event =
-				per_cpu_ptr(&shmem_events, cpu);
+				per_cpu_ptr(user->pinned_events, cpu);
 
 			cmpxchg(pinned_event, event, NULL);
 		}
@@ -9530,7 +9531,7 @@ perf_event_detach(struct perf_event *event, struct perf_event *parent_event,
 {
 	struct ring_buffer *parent_rb = parent_event ? parent_event->rb : NULL;
 	char *filename;
-	int err;
+	int err = -ENOMEM;
 
 	filename = kasprintf(GFP_KERNEL, "%s:%x.event",
 			     task ? "task" : "cpu",
@@ -9550,10 +9551,9 @@ perf_event_detach(struct perf_event *event, struct perf_event *parent_event,
 	if (err) {
 		tracefs_remove(event->dent);
 		event->dent = NULL;
-		return err;
 	}
 
-	return 0;
+	return err;
 }
 /*
  * Allocate and initialize a event structure
@@ -10290,7 +10290,7 @@ SYSCALL_DEFINE5(perf_event_open,
 	}
 
 	if (detached) {
-		err = perf_event_detach(event, task, NULL);
+		err = perf_event_detach(event, NULL, task, NULL);
 		if (err)
 			goto err_context;
 
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index 896d441642..8d37e4e591 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -563,6 +563,44 @@ void *perf_get_aux(struct perf_output_handle *handle)
 	return handle->rb->aux_priv;
 }
 
+static struct user_struct *get_users_pinned_events(void)
+{
+	struct user_struct *user = current_user(), *ret = NULL;
+
+	if (atomic_long_inc_not_zero(&user->nr_pinnable_events))
+		return user;
+
+	mutex_lock(&user->pinned_mutex);
+	if (!atomic_long_read(&user->nr_pinnable_events)) {
+		if (WARN_ON_ONCE(!!user->pinned_events))
+			goto unlock;
+
+		user->pinned_events = alloc_percpu(struct perf_event *);
+		if (!user->pinned_events) {
+			goto unlock;
+		} else {
+			atomic_long_inc(&user->nr_pinnable_events);
+			ret = get_current_user();
+		}
+	}
+
+unlock:
+	mutex_unlock(&user->pinned_mutex);
+
+	return ret;
+}
+
+static void put_users_pinned_events(struct user_struct *user)
+{
+	if (!atomic_long_dec_and_test(&user->nr_pinnable_events))
+		return;
+
+	mutex_lock(&user->pinned_mutex);
+	free_percpu(user->pinned_events);
+	user->pinned_events = NULL;
+	mutex_unlock(&user->pinned_mutex);
+}
+
 /*
  * Check if the current user can afford @nr_pages, considering the
  * perf_event_mlock sysctl and their mlock limit. If the former is exceeded,
@@ -574,11 +612,14 @@ static int __ring_buffer_account(struct ring_buffer *rb, struct mm_struct *mm,
                                  unsigned long nr_pages, unsigned long *locked)
 {
 	unsigned long total, limit, pinned;
+	struct user_struct *user;
 
 	if (!mm)
 		mm = rb->mmap_mapping;
 
-	rb->mmap_user = current_user();
+	user = get_users_pinned_events();
+	if (!user)
+		return -ENOMEM;
 
 	limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
 
@@ -587,10 +628,7 @@ static int __ring_buffer_account(struct ring_buffer *rb, struct mm_struct *mm,
 	 */
 	limit *= num_online_cpus();
 
-	total = atomic_long_read(&rb->mmap_user->locked_vm) + nr_pages;
-
-	free_uid(rb->mmap_user);
-	rb->mmap_user = NULL;
+	total = atomic_long_read(&user->locked_vm) + nr_pages;
 
 	pinned = 0;
 	if (total > limit) {
@@ -599,7 +637,7 @@ static int __ring_buffer_account(struct ring_buffer *rb, struct mm_struct *mm,
 		 * limit needs to be accounted to the consumer's mm.
 		 */
 		if (!mm)
-			return -EPERM;
+			goto err_put_user;
 
 		pinned = total - limit;
 
@@ -608,9 +646,8 @@ static int __ring_buffer_account(struct ring_buffer *rb, struct mm_struct *mm,
 		total = mm->pinned_vm + pinned;
 
 		if ((total > limit) && perf_paranoid_tracepoint_raw() &&
-		    !capable(CAP_IPC_LOCK)) {
-			return -EPERM;
-		}
+		    !capable(CAP_IPC_LOCK))
+			goto err_put_user;
 
 		*locked = pinned;
 		mm->pinned_vm += pinned;
@@ -619,10 +656,15 @@ static int __ring_buffer_account(struct ring_buffer *rb, struct mm_struct *mm,
 	if (!rb->mmap_mapping)
 		rb->mmap_mapping = mm;
 
-	rb->mmap_user = get_current_user();
-	atomic_long_add(nr_pages, &rb->mmap_user->locked_vm);
+	rb->mmap_user = user;
+	atomic_long_add(nr_pages, &user->locked_vm);
 
 	return 0;
+
+err_put_user:
+	put_users_pinned_events(user);
+
+	return -EPERM;
 }
 
 static int ring_buffer_account(struct ring_buffer *rb, struct mm_struct *mm,
@@ -657,7 +699,7 @@ void ring_buffer_unaccount(struct ring_buffer *rb, bool aux)
 	if (rb->mmap_mapping)
 		rb->mmap_mapping->pinned_vm -= pinned;
 
-	free_uid(rb->mmap_user);
+	put_users_pinned_events(rb->mmap_user);
 }
 
 #define PERF_AUX_GFP	(GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)
@@ -1124,6 +1166,7 @@ rb_shmem_account(struct ring_buffer *rb, struct ring_buffer *parent_rb)
 
 		rb->acct_refcount = parent_rb->acct_refcount;
 		atomic_inc(rb->acct_refcount);
+		rb->mmap_user = get_uid(parent_rb->mmap_user);
 
 		return 0;
 	}
@@ -1146,6 +1189,8 @@ rb_shmem_account(struct ring_buffer *rb, struct ring_buffer *parent_rb)
 
 static void rb_shmem_unaccount(struct ring_buffer *rb)
 {
+	free_uid(rb->mmap_user);
+
 	if (!atomic_dec_and_test(rb->acct_refcount)) {
 		rb->acct_refcount = NULL;
 		return;
diff --git a/kernel/user.c b/kernel/user.c
index 00281add65..e95a82d31d 100644
--- a/kernel/user.c
+++ b/kernel/user.c
@@ -185,6 +185,7 @@ struct user_struct *alloc_uid(kuid_t uid)
 
 		new->uid = uid;
 		atomic_set(&new->__count, 1);
+		mutex_init(&new->pinned_mutex);
 
 		/*
 		 * Before adding this, check whether we raced
-- 
2.14.1