[PATCH] arm64: scrub vector and pointer auth state on task death

[Bradley Morgan <brads@mainlining.org>]

When a task dies its SVE, SME and FPSIMD register state, and its
pointer auth keys, stay in freed slab pages until the allocator hands
them out again. init_on_free=1 users get this wiped for the whole heap
but the option is expensive, so it is off in most builds.

The vector registers are not idle memory: userspace crypto runs
AES-GCM and Argon2 through SVE, so key material ends up in
sve_state, sme_state and thread_struct. On task exit and on the
exec and vector length reallocation paths those buffers are
dropped with a plain kfree(), and on task death the register file
copy embedded in thread_struct is not wiped at all. Anything that
reads the freed slab back (slab bugs, cold boot, a leaked page)
gets the dead task's keys.

Scrub it. kfree_sensitive() already exists for this exact job,
so the freeing sites just switch to it. The register file copy
and the pointer auth keys embedded in thread_struct cannot go
through kfree_sensitive(), so arch_release_task_struct() zeroes
them directly with memzero_explicit().

Scrubbed state:
  - sve_state, sme_state allocations, freed on task death, exec
    flush and vector length change
  - the uw.fpsimd_state register file copy embedded in
    thread_struct
  - pointer auth user and kernel keys embedded in thread_struct

This is a cold path. The cost is one memset of at most a few KB
plus the register file per dying task.

Signed-off-by: Bradley Morgan <brads@mainlining.org>
---
 arch/arm64/kernel/fpsimd.c  | 12 ++++++------
 arch/arm64/kernel/process.c | 16 +++++++++++++++-
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index e7f1682a3059..c149c12ba9f9 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -737,7 +737,7 @@ void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__always_unused p)
 #ifdef CONFIG_ARM64_SVE
 static void sve_free(struct task_struct *task)
 {
-	kfree(task->thread.sve_state);
+	kfree_sensitive(task->thread.sve_state);
 	task->thread.sve_state = NULL;
 }
 
@@ -846,13 +846,13 @@ static int change_live_vector_length(struct task_struct *task,
 	 */
 	fpsimd_sync_from_effective_state(task);
 	task_set_vl(task, type, vl);
-	kfree(task->thread.sve_state);
+	kfree_sensitive(task->thread.sve_state);
 	task->thread.sve_state = sve_state;
 	fpsimd_sync_to_effective_state_zeropad(task);
 
 	if (type == ARM64_VEC_SME) {
 		task->thread.svcr &= ~SVCR_ZA_MASK;
-		kfree(task->thread.sme_state);
+		kfree_sensitive(task->thread.sme_state);
 		task->thread.sme_state = sme_state;
 	}
 
@@ -1195,7 +1195,7 @@ void sme_alloc(struct task_struct *task, bool flush)
 
 static void sme_free(struct task_struct *task)
 {
-	kfree(task->thread.sme_state);
+	kfree_sensitive(task->thread.sme_state);
 	task->thread.sme_state = NULL;
 }
 
@@ -1687,8 +1687,8 @@ void fpsimd_flush_thread(void)
 	current->thread.fp_type = FP_STATE_FPSIMD;
 
 	put_cpu_fpsimd_context();
-	kfree(sve_state);
-	kfree(sme_state);
+	kfree_sensitive(sve_state);
+	kfree_sensitive(sme_state);
 }
 
 /*
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 581f80e9b9b7..c1d5d7c0fd3f 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -344,6 +344,20 @@ void flush_thread(void)
 void arch_release_task_struct(struct task_struct *tsk)
 {
 	fpsimd_release_task(tsk);
+
+	/*
+	 * Wipe the register file copy and the pointer auth keys, they
+	 * can hold key material the dead task used.
+	 */
+	memzero_explicit(&tsk->thread.uw.fpsimd_state,
+			 sizeof(tsk->thread.uw.fpsimd_state));
+#ifdef CONFIG_ARM64_PTR_AUTH
+	memzero_explicit(&tsk->thread.keys_user, sizeof(tsk->thread.keys_user));
+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
+	memzero_explicit(&tsk->thread.keys_kernel,
+			 sizeof(tsk->thread.keys_kernel));
+#endif
+#endif
 }
 
 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
@@ -397,7 +411,7 @@ static int copy_thread_za(struct task_struct *dst, struct task_struct *src)
 					sme_state_size(src),
 					GFP_KERNEL);
 	if (!dst->thread.sme_state) {
-		kfree(dst->thread.sve_state);
+		kfree_sensitive(dst->thread.sve_state);
 		dst->thread.sve_state = NULL;
 		return -ENOMEM;
 	}
-- 
2.53.0