[PATCH 04/24] arm64: Skip TCO on privileged uaccess without kMTE

["Pierre-Clément Tosi" <ptosi@google.com>]

PSTATE.TCO toggling can be expensive but uaccess_*_privileged() (used in
futex()) were doing it unconditionally when built with KASAN_HW_TAGS and
running on MTE-capable CPUs, due to mte_{enable,disable}_tco(). This was
unnecessary when 'kasan=off', adding overhead despite the care taken in
mte_disable_tco_entry() to keep TCO set.

Instead, only toggle TCO if EL1 accesses risk triggering tag faults.

Name the new helpers by following the example of __uaccess_*_hw_pan().
Specifically, this favors a simpler terminology ("re-enabling MTE")
over confusing double-negations ("disabling the override").

Move the comment intended to document this out of a shared KASAN header,
to be next to the code it explains.

Signed-off-by: Pierre-Clément Tosi <ptosi@google.com>
---
 arch/arm64/include/asm/mte-kasan.h | 17 -----------------
 arch/arm64/include/asm/uaccess.h   | 21 +++++++++++++++++++--
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h
index 43510341b965..467d1d41a392 100644
--- a/arch/arm64/include/asm/mte-kasan.h
+++ b/arch/arm64/include/asm/mte-kasan.h
@@ -34,23 +34,6 @@ static inline bool system_uses_mte_async_or_asymm_mode(void)
 
 #ifdef CONFIG_ARM64_MTE
 
-/*
- * The Tag Check Flag (TCF) mode for MTE is per EL, hence TCF0
- * affects EL0 and TCF affects EL1 irrespective of which TTBR is
- * used.
- * The kernel accesses TTBR0 usually with LDTR/STTR instructions
- * when UAO is available, so these would act as EL0 accesses using
- * TCF0.
- * However futex.h code uses exclusives which would be executed as
- * EL1, this can potentially cause a tag check fault even if the
- * user disables TCF0.
- *
- * To address the problem we set the PSTATE.TCO bit in uaccess_enable()
- * and reset it in uaccess_disable().
- *
- * The Tag check override (TCO) bit disables temporarily the tag checking
- * preventing the issue.
- */
 static inline void mte_disable_tco(void)
 {
 	if (!IS_ENABLED(CONFIG_KASAN_HW_TAGS))
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 9f5bd9c69c24..f0924dc0bd64 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -16,6 +16,7 @@
  */
 #include <linux/bitops.h>
 #include <linux/kasan-checks.h>
+#include <linux/kasan-enabled.h>
 #include <linux/string.h>
 
 #include <asm/asm-extable.h>
@@ -132,9 +133,25 @@ static inline void __uaccess_enable_hw_pan(void)
 	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN));
 }
 
+static inline void __uaccess_disable_mte(void)
+{
+	/*
+	 * Unlike get_user(), privileged uaccess (TTBR0) uses TCF (not TCF0)
+	 * such that a bad user address tag could trigger a kernel MTE fault.
+	 */
+	if (kasan_hw_tags_enabled())
+		asm volatile(SET_PSTATE_TCO(1));
+}
+
+static inline void __uaccess_enable_mte(void)
+{
+	if (kasan_hw_tags_enabled())
+		asm volatile(SET_PSTATE_TCO(0));
+}
+
 static inline void uaccess_disable_privileged(void)
 {
-	mte_disable_tco();
+	__uaccess_enable_mte();
 
 	if (uaccess_ttbr0_disable())
 		return;
@@ -144,7 +161,7 @@ static inline void uaccess_disable_privileged(void)
 
 static inline void uaccess_enable_privileged(void)
 {
-	mte_enable_tco();
+	__uaccess_disable_mte();
 
 	if (uaccess_ttbr0_enable())
 		return;

-- 
2.56.0.455.g6c5a503b38-goog