[PATCH bpf-next 02/25] x86: implement per-function metadata storage for x86

[Menglong Dong <menglong8.dong@gmail.com>]

With CONFIG_CALL_PADDING enabled, there will be 16-bytes padding space
before all the kernel functions. And some kernel features can use it,
such as MITIGATION_CALL_DEPTH_TRACKING, CFI_CLANG, FINEIBT, etc.

In my research, MITIGATION_CALL_DEPTH_TRACKING will consume the tail
9-bytes in the function padding, CFI_CLANG will consume the head 5-bytes,
and FINEIBT will consume all the 16 bytes if it is enabled. So there will
be no space for us if MITIGATION_CALL_DEPTH_TRACKING and CFI_CLANG are
both enabled, or FINEIBT is enabled.

In order to implement the padding-based function metadata, we need 5-bytes
to prepend a "mov %eax xxx" insn in x86_64, which can hold a 4-bytes
index. So we have following logic:

1. use the head 5-bytes if CFI_CLANG is not enabled
2. use the tail 5-bytes if MITIGATION_CALL_DEPTH_TRACKING and FINEIBT are
   not enabled
3. try to probe if fineibt or the call thunks is enabled after the kernel
   boot dynamically

On the third case, we implement the function metadata by hash table if
"cfi_mode==CFI_FINEIBT || thunks_initialized". Therefore, we need to make
thunks_initialized global in arch/x86/kernel/callthunks.c

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
 arch/x86/Kconfig                   | 26 +++++++++++++++++
 arch/x86/include/asm/alternative.h |  2 ++
 arch/x86/include/asm/ftrace.h      | 47 ++++++++++++++++++++++++++++++
 arch/x86/kernel/callthunks.c       |  2 +-
 arch/x86/kernel/ftrace.c           | 26 +++++++++++++++++
 5 files changed, 102 insertions(+), 1 deletion(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 4b9f378e05f6..0405288c42c6 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2459,6 +2459,32 @@ config PREFIX_SYMBOLS
 	def_bool y
 	depends on CALL_PADDING && !CFI_CLANG
 
+config FUNCTION_METADATA
+	bool "Per-function metadata storage support"
+	default y
+	depends on CC_HAS_ENTRY_PADDING && OBJTOOL
+	help
+	  Support function padding based per-function metadata storage for
+	  kernel functions, and get the metadata of the function by its
+	  address with almost no overhead.
+
+	  The index of the metadata will be stored in the function padding
+	  and consumes 5-bytes.
+
+	  Hash table based function metadata will be used if this option
+	  is not enabled.
+
+config FUNCTION_METADATA_PADDING
+	bool "function padding is available for metadata"
+	default y
+	depends on FUNCTION_METADATA && !FINEIBT && !(CFI_CLANG && CALL_THUNKS)
+	select CALL_PADDING
+	help
+	  Function padding is available for the function metadata. If this
+	  option is disabled, function metadata will try to probe if there
+	  are usable function padding during the system boot. If not, the
+	  hash table based function metadata will be used instead.
+
 menuconfig CPU_MITIGATIONS
 	bool "Mitigations for CPU vulnerabilities"
 	default y
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 4a37a8bd87fd..951edf1857c3 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -103,6 +103,8 @@ struct callthunk_sites {
 };
 
 #ifdef CONFIG_CALL_THUNKS
+extern bool thunks_initialized;
+
 extern void callthunks_patch_builtin_calls(void);
 extern void callthunks_patch_module_calls(struct callthunk_sites *sites,
 					  struct module *mod);
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index 93156ac4ffe0..ed1fdfce824e 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -4,6 +4,21 @@
 
 #include <asm/ptrace.h>
 
+#ifdef CONFIG_FUNCTION_METADATA_PADDING
+
+#ifdef CONFIG_CFI_CLANG
+/* use the space that CALL_THUNKS suppose to use */
+#define KFUNC_MD_INSN_OFFSET	(5)
+#define KFUNC_MD_DATA_OFFSET	(4)
+#else
+/* use the space that CFI_CLANG suppose to use */
+#define KFUNC_MD_INSN_OFFSET	(CONFIG_FUNCTION_PADDING_BYTES)
+#define KFUNC_MD_DATA_OFFSET	(CONFIG_FUNCTION_PADDING_BYTES - 1)
+#endif
+#endif
+
+#define KFUNC_MD_INSN_SIZE	(5)
+
 #ifdef CONFIG_FUNCTION_TRACER
 #ifndef CC_USING_FENTRY
 # error Compiler does not support fentry?
@@ -154,6 +169,38 @@ static inline bool arch_trace_is_compat_syscall(struct pt_regs *regs)
 }
 #endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_IA32_EMULATION */
 #endif /* !COMPILE_OFFSETS */
+
+#ifdef CONFIG_FUNCTION_METADATA
+#include <asm/text-patching.h>
+
+static inline bool kfunc_md_arch_exist(unsigned long ip, int insn_offset)
+{
+	return *(u8 *)(ip - insn_offset) == 0xB8;
+}
+
+static inline void kfunc_md_arch_pretend(u8 *insn, u32 index)
+{
+	*insn = 0xB8;
+	*(u32 *)(insn + 1) = index;
+}
+
+static inline void kfunc_md_arch_nops(u8 *insn)
+{
+	*(insn++) = BYTES_NOP1;
+	*(insn++) = BYTES_NOP1;
+	*(insn++) = BYTES_NOP1;
+	*(insn++) = BYTES_NOP1;
+	*(insn++) = BYTES_NOP1;
+}
+
+static inline int kfunc_md_arch_poke(void *ip, u8 *insn, int insn_offset)
+{
+	text_poke(ip, insn, insn_offset);
+	text_poke_sync();
+	return 0;
+}
+#endif /* CONFIG_FUNCTION_METADATA */
+
 #endif /* !__ASSEMBLER__ */
 
 #endif /* _ASM_X86_FTRACE_H */
diff --git a/arch/x86/kernel/callthunks.c b/arch/x86/kernel/callthunks.c
index d86d7d6e750c..6ed49904cd61 100644
--- a/arch/x86/kernel/callthunks.c
+++ b/arch/x86/kernel/callthunks.c
@@ -56,7 +56,7 @@ struct core_text {
 	const char	*name;
 };
 
-static bool thunks_initialized __ro_after_init;
+bool thunks_initialized __ro_after_init;
 
 static const struct core_text builtin_coretext = {
 	.base = (unsigned long)_text,
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index cace6e8d7cc7..2504c2556508 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -26,6 +26,7 @@
 #include <linux/vmalloc.h>
 #include <linux/set_memory.h>
 #include <linux/execmem.h>
+#include <linux/kfunc_md.h>
 
 #include <trace/syscall.h>
 
@@ -569,6 +570,31 @@ void arch_ftrace_trampoline_free(struct ftrace_ops *ops)
 	ops->trampoline = 0;
 }
 
+#if defined(CONFIG_FUNCTION_METADATA) && !defined(CONFIG_FUNCTION_METADATA_PADDING)
+bool kfunc_md_arch_support(int *insn, int *data)
+{
+	/* when fineibt is enabled, the 16-bytes padding are all used */
+	if (IS_ENABLED(CONFIG_FINEIBT) && cfi_mode == CFI_FINEIBT)
+		return false;
+
+	if (IS_ENABLED(CONFIG_CALL_THUNKS) && IS_ENABLED(CONFIG_CFI_CLANG)) {
+		/* when call thunks and cfi are both enabled, no enough space
+		 * for us.
+		 */
+		if (thunks_initialized)
+			return false;
+		/* use the tail 5-bytes for function meta data */
+		*insn = 5;
+		*data = 4;
+
+		return true;
+	}
+
+	WARN_ON_ONCE(1);
+	return true;
+}
+#endif
+
 #endif /* CONFIG_X86_64 */
 #endif /* CONFIG_DYNAMIC_FTRACE */
 
-- 
2.39.5