[PATCH 1/2] audit: show user land backtrace as part of audit context messages

[PATCH 1/2] audit: show user land backtrace as part of audit context messages

[Daniel Walker]

From: Victor Kamensky <kamensky@cisco.com>

To efficiently find out where SELinux AVC denial is comming from
take backtrace of user land process and display it as type=UBACKTRACE
message that comes as audit context for SELinux AVC and other audit
messages.

By default UBACKTRACE context messages are off. Needs to be enabled
through audit AUDIT_FEATURE_UBACKTRACE_CONTEXT feature.

Context UBACKTRACE message example:
type=UBACKTRACE msg=audit(1574205625.557:30): backtrace=libc-2.30.so+0xc99ab:dmesg.util-linux+0x1483:libc-2.30.so+0x1e35:dmesg.util-linux+0x1e5a

I.e because of ASLR instead of absolute user land addresses, name
of executable or library captured and followed by offset from text
segment vma. To decode backtrace entry: find executable or library
symbol file, find its text segment vma and add offset to it; run
'addr2line -f -e symbol_file resulting_address'.

Note feature depends on PERF_EVENTS, from perf subsystem it uses
perf_callchain_user function on architectures where it is implemented.
And it has the same capturing restriction as 'perf -g': user land code
must be compiled with -fno-omit-frame-pointer option, otherwise kernel
is not capable walk user land stack frames.

Cc: xe-linux-external@cisco.com
Signed-off-by: Victor Kamensky <kamensky@cisco.com>
Signed-off-by: Ruslan Bilovol <rbilovol@cisco.com>
Signed-off-by: Daniel Walker <danielwa@cisco.com>
---
 include/uapi/linux/audit.h |  6 ++-
 init/Kconfig               | 13 ++++++
 kernel/audit.c             |  3 +-
 kernel/auditsc.c           | 93 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 112 insertions(+), 3 deletions(-)

diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h
index cd2d8279a5e4..7bea44b1c028 100644
--- a/include/uapi/linux/audit.h
+++ b/include/uapi/linux/audit.h
@@ -118,6 +118,7 @@
 #define AUDIT_TIME_ADJNTPVAL	1333	/* NTP value adjustment */
 #define AUDIT_BPF		1334	/* BPF subsystem */
 #define AUDIT_EVENT_LISTENER	1335	/* Task joined multicast read socket */
+#define AUDIT_UBACKTRACE	1336	/* User land backtrace */
 
 #define AUDIT_AVC		1400	/* SE Linux avc denial or grant */
 #define AUDIT_SELINUX_ERR	1401	/* Internal SE Linux Errors */
@@ -474,7 +475,7 @@ struct audit_status {
 };
 
 struct audit_features {
-#define AUDIT_FEATURE_VERSION	1
+#define AUDIT_FEATURE_VERSION	2
 	__u32	vers;
 	__u32	mask;		/* which bits we are dealing with */
 	__u32	features;	/* which feature to enable/disable */
@@ -483,7 +484,8 @@ struct audit_features {
 
 #define AUDIT_FEATURE_ONLY_UNSET_LOGINUID	0
 #define AUDIT_FEATURE_LOGINUID_IMMUTABLE	1
-#define AUDIT_LAST_FEATURE			AUDIT_FEATURE_LOGINUID_IMMUTABLE
+#define AUDIT_FEATURE_UBACKTRACE_CONTEXT	2
+#define AUDIT_LAST_FEATURE			AUDIT_FEATURE_UBACKTRACE_CONTEXT
 
 #define audit_feature_valid(x)		((x) >= 0 && (x) <= AUDIT_LAST_FEATURE)
 #define AUDIT_FEATURE_TO_MASK(x)	(1 << ((x) & 31)) /* mask for __u32 */
diff --git a/init/Kconfig b/init/Kconfig
index b77c60f8b963..4327a8afb1f9 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -433,6 +433,19 @@ config AUDITSYSCALL
 	depends on AUDIT && HAVE_ARCH_AUDITSYSCALL
 	select FSNOTIFY
 
+config AUDIT_USER_BACKTRACE
+        bool "Enable user land backtrace in audit context messages"
+        def_bool n
+        depends on AUDITSYSCALL && PERF_EVENTS
+        help
+          Enable UBACKTRACE audit context messages, capturing backtrace of
+          user land process causing the message.
+
+config AUDIT_USER_BACKTRACE_SIZE
+        int "Maximum size of user land backtrace entries captured in UBACKTACE"
+        depends on AUDIT_USER_BACKTRACE
+        default 40
+
 source "kernel/irq/Kconfig"
 source "kernel/time/Kconfig"
 source "kernel/Kconfig.preempt"
diff --git a/kernel/audit.c b/kernel/audit.c
index 1ffc2e059027..4608cddb4bb9 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -165,9 +165,10 @@ static struct audit_features af = {.vers = AUDIT_FEATURE_VERSION,
 				   .features = 0,
 				   .lock = 0,};
 
-static char *audit_feature_names[2] = {
+static char *audit_feature_names[3] = {
 	"only_unset_loginuid",
 	"loginuid_immutable",
+	"ubacktrace_context",
 };
 
 /**
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index ce8c9e2279ba..d048b01345b8 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -67,6 +67,7 @@
 #include <linux/highmem.h>
 #include <linux/syscalls.h>
 #include <asm/syscall.h>
+#include <linux/mmap_lock.h>
 #include <linux/capability.h>
 #include <linux/fs_struct.h>
 #include <linux/compat.h>
@@ -74,6 +75,7 @@
 #include <linux/string.h>
 #include <linux/uaccess.h>
 #include <linux/fsnotify_backend.h>
+#include <linux/perf_event.h>
 #include <uapi/linux/limits.h>
 #include <uapi/linux/netfilter/nf_tables.h>
 
@@ -1470,6 +1472,92 @@ static void audit_log_proctitle(void)
 	audit_log_end(ab);
 }
 
+#ifdef CONFIG_AUDIT_USER_BACKTRACE
+static void audit_log_print_backtrace(struct audit_buffer *ab,
+				      struct task_struct *tsk,
+				      struct perf_callchain_entry *entry)
+{
+	struct mm_struct *mm;
+	const struct vm_area_struct *vma;
+	struct file *file;
+	const char *filename;
+	unsigned long vmstart;
+	int i;
+
+	audit_log_format(ab, "backtrace=");
+
+	mm = tsk->mm;
+	mmap_write_lock(mm);
+	for (i = 0; i < entry->nr; i++) {
+		vma = find_vma(mm, entry->ip[i]);
+		if (vma && (vma->vm_flags & VM_EXEC)) {
+			file = vma->vm_file;
+			vmstart = vma->vm_start;
+
+			if (file && file->f_path.dentry) {
+				filename = file->f_path.dentry->d_name.name;
+			} else {
+				filename = "";
+			}
+
+			if (i == 0) {
+				audit_log_format(ab, "%s+0x%llx",
+						 filename,
+						 entry->ip[i] - vmstart);
+			} else {
+				audit_log_format(ab, ":%s+0x%llx",
+						 filename,
+						 entry->ip[i] - vmstart);
+			}
+		} else {
+			/* No corresponding executable vma, assume garbage entry */
+			break;
+		}
+	}
+	mmap_write_unlock(mm);
+}
+
+static void audit_log_ubacktrace(struct task_struct *tsk,
+				 struct audit_context *context)
+{
+	struct audit_buffer *ab;
+	struct perf_callchain_entry_ctx ctx;
+	struct pt_regs *regs = NULL;
+
+	if  (tsk->mm)
+		regs = task_pt_regs(tsk);
+
+	if (regs) {
+		ctx.entry = kmalloc(sizeof(struct perf_callchain_entry) +
+				    CONFIG_AUDIT_USER_BACKTRACE_SIZE * sizeof(__u64),
+				    GFP_KERNEL);
+		if (ctx.entry) {
+			mm_segment_t fs;
+
+			ctx.max_stack = CONFIG_AUDIT_USER_BACKTRACE_SIZE;
+			ctx.nr = ctx.entry->nr = 0;
+			ctx.contexts = 0;
+			ctx.contexts_maxed = false;
+
+			fs = force_uaccess_begin();
+			perf_callchain_user(&ctx, regs);
+			force_uaccess_end(fs);
+
+			if (ctx.entry->nr) {
+				ab = audit_log_start(context, GFP_KERNEL, AUDIT_UBACKTRACE);
+
+				if (ab) {
+					audit_log_print_backtrace(ab, tsk, ctx.entry);
+					audit_log_end(ab);
+				}
+			}
+
+			kfree(ctx.entry);
+		}
+	}
+}
+#endif /* CONFIG_AUDIT_USER_BACKTRACE */
+
 static void audit_log_exit(void)
 {
 	int i, call_panic = 0;
@@ -1594,6 +1682,11 @@ static void audit_log_exit(void)
 
 	audit_log_proctitle();
 
+#ifdef CONFIG_AUDIT_USER_BACKTRACE
+	if (is_audit_feature_set(AUDIT_FEATURE_UBACKTRACE_CONTEXT))
+		audit_log_ubacktrace(current, context);
+#endif /* CONFIG_AUDIT_USER_BACKTRACE */
+
 	/* Send end of event record to help user space know we are finished */
 	ab = audit_log_start(context, GFP_KERNEL, AUDIT_EOE);
 	if (ab)
-- 
2.17.1

Re: [PATCH 1/2] audit: show user land backtrace as part of audit context messages

[Paul Moore]

On Tue, Feb 2, 2021 at 4:29 PM Daniel Walker <danielwa@cisco.com> wrote:
> From: Victor Kamensky <kamensky@cisco.com>
>
> To efficiently find out where SELinux AVC denial is comming from
> take backtrace of user land process and display it as type=UBACKTRACE
> message that comes as audit context for SELinux AVC and other audit
> messages ...

Have you tried the new perf tracepoint for SELinux AVC decisions that
trigger an audit event?  It's a new feature for v5.10 and looks to
accomplish most of what you are looking for with this patch.

* https://www.paul-moore.com/blog/d/2020/12/linux_v510.html

-- 
paul moore
www.paul-moore.com

Re: [PATCH 1/2] audit: show user land backtrace as part of audit context messages

[Daniel Walker (danielwa)]

On Tue, Feb 02, 2021 at 04:35:42PM -0500, Paul Moore wrote:
> On Tue, Feb 2, 2021 at 4:29 PM Daniel Walker <danielwa@cisco.com> wrote:
> > From: Victor Kamensky <kamensky@cisco.com>
> >
> > To efficiently find out where SELinux AVC denial is comming from
> > take backtrace of user land process and display it as type=UBACKTRACE
> > message that comes as audit context for SELinux AVC and other audit
> > messages ...
> 
> Have you tried the new perf tracepoint for SELinux AVC decisions that
> trigger an audit event?  It's a new feature for v5.10 and looks to
> accomplish most of what you are looking for with this patch.
> 
> * https://www.paul-moore.com/blog/d/2020/12/linux_v510.html

We haven't tried it, but I can look into it. We're not using v5.10 extensively
yet.

Daniel

Re: [PATCH 1/2] audit: show user land backtrace as part of audit context messages

[Paul Moore]

On Tue, Feb 2, 2021 at 4:44 PM Daniel Walker (danielwa)
<danielwa@cisco.com> wrote:
> On Tue, Feb 02, 2021 at 04:35:42PM -0500, Paul Moore wrote:
> > On Tue, Feb 2, 2021 at 4:29 PM Daniel Walker <danielwa@cisco.com> wrote:
> > > From: Victor Kamensky <kamensky@cisco.com>
> > >
> > > To efficiently find out where SELinux AVC denial is comming from
> > > take backtrace of user land process and display it as type=UBACKTRACE
> > > message that comes as audit context for SELinux AVC and other audit
> > > messages ...
> >
> > Have you tried the new perf tracepoint for SELinux AVC decisions that
> > trigger an audit event?  It's a new feature for v5.10 and looks to
> > accomplish most of what you are looking for with this patch.
> >
> > * https://www.paul-moore.com/blog/d/2020/12/linux_v510.html
>
> We haven't tried it, but I can look into it. We're not using v5.10 extensively
> yet.

Let us know if that works for you, and if it doesn't, let us know what
might be missing.  I hate seeing the kernel grow multiple features
which do the same thing.

-- 
paul moore
www.paul-moore.com

Re: [PATCH 1/2] audit: show user land backtrace as part of audit context messages

[Ondrej Mosnacek]

On Tue, Feb 2, 2021 at 10:46 PM Paul Moore <paul@paul-moore.com> wrote:
> On Tue, Feb 2, 2021 at 4:44 PM Daniel Walker (danielwa)
> <danielwa@cisco.com> wrote:
> > On Tue, Feb 02, 2021 at 04:35:42PM -0500, Paul Moore wrote:
> > > On Tue, Feb 2, 2021 at 4:29 PM Daniel Walker <danielwa@cisco.com> wrote:
> > > > From: Victor Kamensky <kamensky@cisco.com>
> > > >
> > > > To efficiently find out where SELinux AVC denial is comming from
> > > > take backtrace of user land process and display it as type=UBACKTRACE
> > > > message that comes as audit context for SELinux AVC and other audit
> > > > messages ...
> > >
> > > Have you tried the new perf tracepoint for SELinux AVC decisions that
> > > trigger an audit event?  It's a new feature for v5.10 and looks to
> > > accomplish most of what you are looking for with this patch.
> > >
> > > * https://www.paul-moore.com/blog/d/2020/12/linux_v510.html
> >
> > We haven't tried it, but I can look into it. We're not using v5.10 extensively
> > yet.
>
> Let us know if that works for you, and if it doesn't, let us know what
> might be missing.  I hate seeing the kernel grow multiple features
> which do the same thing.

I agree - I played around with this new tracepoint recently and you
can use it to achieve what you want quite easily:

# collect traces for denials (just interrupt/kill the sleep process
when done) - will create a perf.data file you can analyze later
perf record -a -e avc:selinux_audited -g --call-graph=dwarf sleep infinity
# dump all collected backtraces from the perf.data file
perf script

It's a bit complicated if you want to have it running in the
background permanently as a service (you need to tell perf to dump the
recorded data before you can read it), but perf already has some
command-line options to help with that use case.

--
Ondrej Mosnacek
Software Engineer, Linux Security - SELinux kernel
Red Hat, Inc.