* [PATCH 0/3] Rewrite x86/ftrace to use text_poke()
@ 2019-08-26 12:51 Peter Zijlstra
2019-08-26 12:51 ` [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions Peter Zijlstra
` (3 more replies)
0 siblings, 4 replies; 15+ messages in thread
From: Peter Zijlstra @ 2019-08-26 12:51 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Nadav Amit, Andy Lutomirski, Dave Hansen, Song Liu,
Masami Hiramatsu, Steven Rostedt, Daniel Bristot de Oliveira,
Peter Zijlstra
Ftrace was one of the last W^X violators; these patches move it over to the
generic text_poke() interface and thereby get rid of this oddity.
Very lightly tested...
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions
2019-08-26 12:51 [PATCH 0/3] Rewrite x86/ftrace to use text_poke() Peter Zijlstra
@ 2019-08-26 12:51 ` Peter Zijlstra
2019-08-26 12:51 ` [PATCH 2/3] x86/alternatives: Move tp_vec Peter Zijlstra
` (2 subsequent siblings)
3 siblings, 0 replies; 15+ messages in thread
From: Peter Zijlstra @ 2019-08-26 12:51 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Nadav Amit, Andy Lutomirski, Dave Hansen, Song Liu,
Masami Hiramatsu, Steven Rostedt, Daniel Bristot de Oliveira,
Peter Zijlstra
In preparation for static_call and variable size jump_label support,
teach text_poke_bp() to emulate instructions, namely:
JMP32, JMP8, CALL, NOP2, NOP_ATOMIC5
The current text_poke_bp() takes a @handler argument which is used as
a jump target when the temporary INT3 is hit by a different CPU.
When patching CALL instructions, this doesn't work because we'd miss
the PUSH of the return address. Instead, teach poke_int3_handler() to
emulate an instruction, typically the instruction we're patching in.
This fits almost all text_poke_bp() users, except
arch_unoptimize_kprobe() which restores random text, and for that site
we have to build an explicit emulate instruction.
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Daniel Bristot de Oliveira <bristot@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/include/asm/text-patching.h | 24 ++++++--
arch/x86/kernel/alternative.c | 98 ++++++++++++++++++++++++++---------
arch/x86/kernel/jump_label.c | 9 +--
arch/x86/kernel/kprobes/opt.c | 11 ++-
4 files changed, 103 insertions(+), 39 deletions(-)
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -26,10 +26,11 @@ static inline void apply_paravirt(struct
#define POKE_MAX_OPCODE_SIZE 5
struct text_poke_loc {
- void *detour;
void *addr;
- size_t len;
- const char opcode[POKE_MAX_OPCODE_SIZE];
+ int len;
+ s32 rel32;
+ u8 opcode;
+ const char text[POKE_MAX_OPCODE_SIZE];
};
extern void text_poke_early(void *addr, const void *opcode, size_t len);
@@ -51,8 +52,10 @@ extern void text_poke_early(void *addr,
extern void *text_poke(void *addr, const void *opcode, size_t len);
extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
+extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
extern void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries);
+extern void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
+ const void *opcode, size_t len, const void *emulate);
extern int after_bootmem;
extern __ro_after_init struct mm_struct *poking_mm;
extern __ro_after_init unsigned long poking_addr;
@@ -63,8 +66,17 @@ static inline void int3_emulate_jmp(stru
regs->ip = ip;
}
-#define INT3_INSN_SIZE 1
-#define CALL_INSN_SIZE 5
+#define INT3_INSN_SIZE 1
+#define INT3_INSN_OPCODE 0xCC
+
+#define CALL_INSN_SIZE 5
+#define CALL_INSN_OPCODE 0xE8
+
+#define JMP32_INSN_SIZE 5
+#define JMP32_INSN_OPCODE 0xE9
+
+#define JMP8_INSN_SIZE 2
+#define JMP8_INSN_OPCODE 0xEB
static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
{
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -956,16 +956,15 @@ NOKPROBE_SYMBOL(patch_cmp);
int poke_int3_handler(struct pt_regs *regs)
{
struct text_poke_loc *tp;
- unsigned char int3 = 0xcc;
void *ip;
/*
* Having observed our INT3 instruction, we now must observe
* bp_patching.nr_entries.
*
- * nr_entries != 0 INT3
- * WMB RMB
- * write INT3 if (nr_entries)
+ * nr_entries != 0 INT3
+ * WMB RMB
+ * write INT3 if (nr_entries)
*
* Idem for other elements in bp_patching.
*/
@@ -978,9 +977,9 @@ int poke_int3_handler(struct pt_regs *re
return 0;
/*
- * Discount the sizeof(int3). See text_poke_bp_batch().
+ * Discount the INT3. See text_poke_bp_batch().
*/
- ip = (void *) regs->ip - sizeof(int3);
+ ip = (void *) regs->ip - INT3_INSN_SIZE;
/*
* Skip the binary search if there is a single member in the vector.
@@ -997,8 +996,22 @@ int poke_int3_handler(struct pt_regs *re
return 0;
}
- /* set up the specified breakpoint detour */
- regs->ip = (unsigned long) tp->detour;
+ ip += tp->len;
+
+ switch (tp->opcode) {
+ case CALL_INSN_OPCODE:
+ int3_emulate_call(regs, (long)ip + tp->rel32);
+ break;
+
+ case JMP32_INSN_OPCODE:
+ case JMP8_INSN_OPCODE:
+ int3_emulate_jmp(regs, (long)ip + tp->rel32);
+ break;
+
+ default: /* nop */
+ int3_emulate_jmp(regs, (long)ip);
+ break;
+ }
return 1;
}
@@ -1027,8 +1040,8 @@ NOKPROBE_SYMBOL(poke_int3_handler);
*/
void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries)
{
+ unsigned char int3 = INT3_INSN_OPCODE;
int patched_all_but_first = 0;
- unsigned char int3 = 0xcc;
unsigned int i;
lockdep_assert_held(&text_mutex);
@@ -1056,7 +1069,7 @@ void text_poke_bp_batch(struct text_poke
for (i = 0; i < nr_entries; i++) {
if (tp[i].len - sizeof(int3) > 0) {
text_poke((char *)tp[i].addr + sizeof(int3),
- (const char *)tp[i].opcode + sizeof(int3),
+ (const char *)tp[i].text + sizeof(int3),
tp[i].len - sizeof(int3));
patched_all_but_first++;
}
@@ -1076,7 +1089,7 @@ void text_poke_bp_batch(struct text_poke
* replacing opcode.
*/
for (i = 0; i < nr_entries; i++)
- text_poke(tp[i].addr, tp[i].opcode, sizeof(int3));
+ text_poke(tp[i].addr, tp[i].text, sizeof(int3));
on_each_cpu(do_sync_core, NULL, 1);
/*
@@ -1087,6 +1100,53 @@ void text_poke_bp_batch(struct text_poke
bp_patching.nr_entries = 0;
}
+void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
+ const void *opcode, size_t len, const void *emulate)
+{
+ struct insn insn;
+
+ if (!opcode)
+ opcode = (void *)tp->text;
+ else
+ memcpy((void *)tp->text, opcode, len);
+
+ if (!emulate)
+ emulate = opcode;
+
+ kernel_insn_init(&insn, emulate, MAX_INSN_SIZE);
+ insn_get_length(&insn);
+
+ BUG_ON(!insn_complete(&insn));
+ BUG_ON(len != insn.length);
+
+ tp->addr = addr;
+ tp->len = len;
+ tp->opcode = insn.opcode.bytes[0];
+
+ switch (tp->opcode) {
+ case CALL_INSN_OPCODE:
+ case JMP32_INSN_OPCODE:
+ case JMP8_INSN_OPCODE:
+ tp->rel32 = insn.immediate.value;
+ break;
+
+ default: /* assume NOP */
+ switch (len) {
+ case 2:
+ BUG_ON(memcmp(emulate, ideal_nops[len], len));
+ break;
+
+ case 5:
+ BUG_ON(memcmp(emulate, ideal_nops[NOP_ATOMIC5], len));
+ break;
+
+ default:
+ BUG();
+ }
+ break;
+ }
+}
+
/**
* text_poke_bp() -- update instructions on live kernel on SMP
* @addr: address to patch
@@ -1098,20 +1158,10 @@ void text_poke_bp_batch(struct text_poke
* dynamically allocated memory. This function should be used when it is
* not possible to allocate memory.
*/
-void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
+void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
{
- struct text_poke_loc tp = {
- .detour = handler,
- .addr = addr,
- .len = len,
- };
-
- if (len > POKE_MAX_OPCODE_SIZE) {
- WARN_ONCE(1, "len is larger than %d\n", POKE_MAX_OPCODE_SIZE);
- return;
- }
-
- memcpy((void *)tp.opcode, opcode, len);
+ struct text_poke_loc tp;
+ text_poke_loc_init(&tp, addr, opcode, len, emulate);
text_poke_bp_batch(&tp, 1);
}
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -89,8 +89,7 @@ static void __ref __jump_label_transform
return;
}
- text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE,
- (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
+ text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE, NULL);
}
void arch_jump_label_transform(struct jump_entry *entry,
@@ -147,11 +146,9 @@ bool arch_jump_label_transform_queue(str
}
__jump_label_set_jump_code(entry, type,
- (union jump_code_union *) &tp->opcode, 0);
+ (union jump_code_union *)&tp->text, 0);
- tp->addr = entry_code;
- tp->detour = entry_code + JUMP_LABEL_NOP_SIZE;
- tp->len = JUMP_LABEL_NOP_SIZE;
+ text_poke_loc_init(tp, entry_code, NULL, JUMP_LABEL_NOP_SIZE, NULL);
tp_vec_nr++;
--- a/arch/x86/kernel/kprobes/opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h
insn_buff[0] = RELATIVEJUMP_OPCODE;
*(s32 *)(&insn_buff[1]) = rel;
- text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
- op->optinsn.insn);
+ text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, NULL);
list_del_init(&op->list);
}
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
void arch_unoptimize_kprobe(struct optimized_kprobe *op)
{
u8 insn_buff[RELATIVEJUMP_SIZE];
+ u8 emulate_buff[RELATIVEJUMP_SIZE];
/* Set int3 to first byte for kprobes */
insn_buff[0] = BREAKPOINT_INSTRUCTION;
memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
+
+ emulate_buff[0] = RELATIVEJUMP_OPCODE;
+ *(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
+ ((long)op->kp.addr + RELATIVEJUMP_SIZE));
+
text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
- op->optinsn.insn);
+ emulate_buff);
}
/*
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH 2/3] x86/alternatives: Move tp_vec
2019-08-26 12:51 [PATCH 0/3] Rewrite x86/ftrace to use text_poke() Peter Zijlstra
2019-08-26 12:51 ` [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions Peter Zijlstra
@ 2019-08-26 12:51 ` Peter Zijlstra
2019-08-26 14:36 ` Peter Zijlstra
2019-08-26 12:51 ` [RFC][PATCH 3/3] x86/ftrace: Use text_poke() Peter Zijlstra
2019-08-26 13:04 ` [PATCH 0/3] Rewrite x86/ftrace to use text_poke() Peter Zijlstra
3 siblings, 1 reply; 15+ messages in thread
From: Peter Zijlstra @ 2019-08-26 12:51 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Nadav Amit, Andy Lutomirski, Dave Hansen, Song Liu,
Daniel Bristot de Oliveira, Peter Zijlstra
In order to allow other users to make use of the tp_vec; move it near
the text_poke_bp_batch() code, instead of keeping it in the jump_label
code.
Cc: Daniel Bristot de Oliveira <bristot@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/include/asm/text-patching.h | 4 ++++
arch/x86/kernel/alternative.c | 3 +++
arch/x86/kernel/jump_label.c | 25 ++++++++++++-------------
3 files changed, 19 insertions(+), 13 deletions(-)
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -60,6 +60,10 @@ extern int after_bootmem;
extern __ro_after_init struct mm_struct *poking_mm;
extern __ro_after_init unsigned long poking_addr;
+#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc))
+extern struct text_poke_loc tp_vec[TP_VEC_MAX];
+extern int tp_vec_nr;
+
#ifndef CONFIG_UML_X86
static inline void int3_emulate_jmp(struct pt_regs *regs, unsigned long ip)
{
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -1017,6 +1017,9 @@ int poke_int3_handler(struct pt_regs *re
}
NOKPROBE_SYMBOL(poke_int3_handler);
+struct text_poke_loc tp_vec[TP_VEC_MAX];
+int tp_vec_nr;
+
/**
* text_poke_bp_batch() -- update instructions on live kernel on SMP
* @tp: vector of instructions to patch
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -100,15 +100,12 @@ void arch_jump_label_transform(struct ju
mutex_unlock(&text_mutex);
}
-#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc))
-static struct text_poke_loc tp_vec[TP_VEC_MAX];
-static int tp_vec_nr;
-
bool arch_jump_label_transform_queue(struct jump_entry *entry,
enum jump_label_type type)
{
struct text_poke_loc *tp;
void *entry_code;
+ bool ret = true;
if (system_state == SYSTEM_BOOTING) {
/*
@@ -118,12 +115,15 @@ bool arch_jump_label_transform_queue(str
return true;
}
+ mutex_lock(&text_mutex);
/*
* No more space in the vector, tell upper layer to apply
* the queue before continuing.
*/
- if (tp_vec_nr == TP_VEC_MAX)
- return false;
+ if (tp_vec_nr == TP_VEC_MAX) {
+ ret = false;
+ goto unlock;
+ }
tp = &tp_vec[tp_vec_nr];
@@ -151,20 +151,19 @@ bool arch_jump_label_transform_queue(str
text_poke_loc_init(tp, entry_code, NULL, JUMP_LABEL_NOP_SIZE, NULL);
tp_vec_nr++;
+unlock:
+ mutex_unlock(&text_mutex);
- return true;
+ return ret;
}
void arch_jump_label_transform_apply(void)
{
- if (!tp_vec_nr)
- return;
-
mutex_lock(&text_mutex);
- text_poke_bp_batch(tp_vec, tp_vec_nr);
- mutex_unlock(&text_mutex);
-
+ if (tp_vec_nr)
+ text_poke_bp_batch(tp_vec, tp_vec_nr);
tp_vec_nr = 0;
+ mutex_unlock(&text_mutex);
}
static enum {
^ permalink raw reply [flat|nested] 15+ messages in thread
* [RFC][PATCH 3/3] x86/ftrace: Use text_poke()
2019-08-26 12:51 [PATCH 0/3] Rewrite x86/ftrace to use text_poke() Peter Zijlstra
2019-08-26 12:51 ` [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions Peter Zijlstra
2019-08-26 12:51 ` [PATCH 2/3] x86/alternatives: Move tp_vec Peter Zijlstra
@ 2019-08-26 12:51 ` Peter Zijlstra
2019-08-26 15:13 ` Peter Zijlstra
2019-08-26 13:04 ` [PATCH 0/3] Rewrite x86/ftrace to use text_poke() Peter Zijlstra
3 siblings, 1 reply; 15+ messages in thread
From: Peter Zijlstra @ 2019-08-26 12:51 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Nadav Amit, Andy Lutomirski, Dave Hansen, Song Liu,
Daniel Bristot de Oliveira, Steven Rostedt, Peter Zijlstra
Move ftrace over to using the generic x86 text_poke functions; this
avoids having a second/different copy of that code around.
Cc: Daniel Bristot de Oliveira <bristot@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/include/asm/ftrace.h | 2
arch/x86/kernel/ftrace.c | 571 +++---------------------------------------
arch/x86/kernel/traps.c | 9
3 files changed, 51 insertions(+), 531 deletions(-)
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -35,8 +35,6 @@ struct dyn_arch_ftrace {
/* No extra data needed for x86 */
};
-int ftrace_int3_handler(struct pt_regs *regs);
-
#define FTRACE_GRAPH_TRAMP_ADDR FTRACE_GRAPH_ADDR
#endif /* CONFIG_DYNAMIC_FTRACE */
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -43,16 +43,17 @@ int ftrace_arch_code_modify_prepare(void
* ftrace has it set to "read/write".
*/
mutex_lock(&text_mutex);
- set_kernel_text_rw();
- set_all_modules_text_rw();
+ WARN_ON_ONCE(tp_vec_nr);
return 0;
}
int ftrace_arch_code_modify_post_process(void)
__releases(&text_mutex)
{
- set_all_modules_text_ro();
- set_kernel_text_ro();
+ if (tp_vec_nr) {
+ text_poke_bp_batch(tp_vec, tp_vec_nr);
+ tp_vec_nr = 0;
+ }
mutex_unlock(&text_mutex);
return 0;
}
@@ -60,67 +61,36 @@ int ftrace_arch_code_modify_post_process
union ftrace_code_union {
char code[MCOUNT_INSN_SIZE];
struct {
- unsigned char op;
+ char op;
int offset;
} __attribute__((packed));
};
-static int ftrace_calc_offset(long ip, long addr)
-{
- return (int)(addr - ip);
-}
-
-static unsigned char *
-ftrace_text_replace(unsigned char op, unsigned long ip, unsigned long addr)
+static char *ftrace_text_replace(char op, unsigned long ip, unsigned long addr)
{
static union ftrace_code_union calc;
- calc.op = op;
- calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
+ calc.op = op;
+ calc.offset = (int)(addr - (ip + MCOUNT_INSN_SIZE));
return calc.code;
}
-static unsigned char *
-ftrace_call_replace(unsigned long ip, unsigned long addr)
-{
- return ftrace_text_replace(0xe8, ip, addr);
-}
-
-static inline int
-within(unsigned long addr, unsigned long start, unsigned long end)
+static char *ftrace_nop_replace(void)
{
- return addr >= start && addr < end;
-}
-
-static unsigned long text_ip_addr(unsigned long ip)
-{
- /*
- * On x86_64, kernel text mappings are mapped read-only, so we use
- * the kernel identity mapping instead of the kernel text mapping
- * to modify the kernel text.
- *
- * For 32bit kernels, these mappings are same and we can use
- * kernel identity mapping to modify code.
- */
- if (within(ip, (unsigned long)_text, (unsigned long)_etext))
- ip = (unsigned long)__va(__pa_symbol(ip));
-
- return ip;
+ return ideal_nops[NOP_ATOMIC5];
}
-static const unsigned char *ftrace_nop_replace(void)
+static char *ftrace_call_replace(unsigned long ip, unsigned long addr)
{
- return ideal_nops[NOP_ATOMIC5];
+ return ftrace_text_replace(CALL_INSN_OPCODE, ip, addr);
}
-static int
-ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
- unsigned const char *new_code)
+static int;
+ftrace_modify_code_direct(unsigned long ip, const char *old_code,
+ const char *new_code)
{
- unsigned char replaced[MCOUNT_INSN_SIZE];
-
- ftrace_expected = old_code;
+ char cur_code[MCOUNT_INSN_SIZE];
/*
* Note:
@@ -129,31 +99,23 @@ ftrace_modify_code_direct(unsigned long
* Carefully read and modify the code with probe_kernel_*(), and make
* sure what we read is what we expected it to be before modifying it.
*/
-
/* read the text we want to modify */
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(cur_code, (void *)ip, MCOUNT_INSN_SIZE))
return -EFAULT;
/* Make sure it is what we expect it to be */
- if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
+ if (memcmp(cur_code, old_code, MCOUNT_INSN_SIZE) != 0)
return -EINVAL;
- ip = text_ip_addr(ip);
-
/* replace the text with the new text */
- if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
- return -EPERM;
-
- sync_core();
-
+ text_poke_early((void *)ip, new_code, MCOUNT_INSN_SIZE);
return 0;
}
-int ftrace_make_nop(struct module *mod,
- struct dyn_ftrace *rec, unsigned long addr)
+int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr)
{
- unsigned const char *new, *old;
unsigned long ip = rec->ip;
+ const char *new, *old;
old = ftrace_call_replace(ip, addr);
new = ftrace_nop_replace();
@@ -166,20 +128,21 @@ int ftrace_make_nop(struct module *mod,
* We do not want to use the breakpoint version in this case,
* just modify the code directly.
*/
- if (addr == MCOUNT_ADDR)
- return ftrace_modify_code_direct(rec->ip, old, new);
+ if (addr == MCOUNT)
+ return ftrace_modify_code_direct(ip, old, new);
- ftrace_expected = NULL;
-
- /* Normal cases use add_brk_on_nop */
+ /*
+ * x86 overrides ftrace_replace_code -- this function will never be used
+ * in this case.
+ */
WARN_ONCE(1, "invalid use of ftrace_make_nop");
return -EINVAL;
}
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
- unsigned const char *new, *old;
unsigned long ip = rec->ip;
+ const char *new, *old;
old = ftrace_nop_replace();
new = ftrace_call_replace(ip, addr);
@@ -189,43 +152,6 @@ int ftrace_make_call(struct dyn_ftrace *
}
/*
- * The modifying_ftrace_code is used to tell the breakpoint
- * handler to call ftrace_int3_handler(). If it fails to
- * call this handler for a breakpoint added by ftrace, then
- * the kernel may crash.
- *
- * As atomic_writes on x86 do not need a barrier, we do not
- * need to add smp_mb()s for this to work. It is also considered
- * that we can not read the modifying_ftrace_code before
- * executing the breakpoint. That would be quite remarkable if
- * it could do that. Here's the flow that is required:
- *
- * CPU-0 CPU-1
- *
- * atomic_inc(mfc);
- * write int3s
- * <trap-int3> // implicit (r)mb
- * if (atomic_read(mfc))
- * call ftrace_int3_handler()
- *
- * Then when we are finished:
- *
- * atomic_dec(mfc);
- *
- * If we hit a breakpoint that was not set by ftrace, it does not
- * matter if ftrace_int3_handler() is called or not. It will
- * simply be ignored. But it is crucial that a ftrace nop/caller
- * breakpoint is handled. No other user should ever place a
- * breakpoint on an ftrace nop/caller location. It must only
- * be done by this code.
- */
-atomic_t modifying_ftrace_code __read_mostly;
-
-static int
-ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
- unsigned const char *new_code);
-
-/*
* Should never be called:
* As it is only called by __ftrace_replace_code() which is called by
* ftrace_replace_code() that x86 overrides, and by ftrace_update_code()
@@ -237,452 +163,57 @@ int ftrace_modify_call(struct dyn_ftrace
unsigned long addr)
{
WARN_ON(1);
- ftrace_expected = NULL;
return -EINVAL;
}
-static unsigned long ftrace_update_func;
-static unsigned long ftrace_update_func_call;
-
-static int update_ftrace_func(unsigned long ip, void *new)
-{
- unsigned char old[MCOUNT_INSN_SIZE];
- int ret;
-
- memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
-
- ftrace_update_func = ip;
- /* Make sure the breakpoints see the ftrace_update_func update */
- smp_wmb();
-
- /* See comment above by declaration of modifying_ftrace_code */
- atomic_inc(&modifying_ftrace_code);
-
- ret = ftrace_modify_code(ip, old, new);
-
- atomic_dec(&modifying_ftrace_code);
-
- return ret;
-}
-
int ftrace_update_ftrace_func(ftrace_func_t func)
{
unsigned long ip = (unsigned long)(&ftrace_call);
unsigned char *new;
- int ret;
-
- ftrace_update_func_call = (unsigned long)func;
new = ftrace_call_replace(ip, (unsigned long)func);
- ret = update_ftrace_func(ip, new);
-
- /* Also update the regs callback function */
- if (!ret) {
- ip = (unsigned long)(&ftrace_regs_call);
- new = ftrace_call_replace(ip, (unsigned long)func);
- ret = update_ftrace_func(ip, new);
- }
-
- return ret;
-}
-
-static nokprobe_inline int is_ftrace_caller(unsigned long ip)
-{
- if (ip == ftrace_update_func)
- return 1;
-
- return 0;
-}
-
-/*
- * A breakpoint was added to the code address we are about to
- * modify, and this is the handle that will just skip over it.
- * We are either changing a nop into a trace call, or a trace
- * call to a nop. While the change is taking place, we treat
- * it just like it was a nop.
- */
-int ftrace_int3_handler(struct pt_regs *regs)
-{
- unsigned long ip;
-
- if (WARN_ON_ONCE(!regs))
- return 0;
-
- ip = regs->ip - INT3_INSN_SIZE;
-
- if (ftrace_location(ip)) {
- int3_emulate_call(regs, (unsigned long)ftrace_regs_caller);
- return 1;
- } else if (is_ftrace_caller(ip)) {
- if (!ftrace_update_func_call) {
- int3_emulate_jmp(regs, ip + CALL_INSN_SIZE);
- return 1;
- }
- int3_emulate_call(regs, ftrace_update_func_call);
- return 1;
- }
-
- return 0;
-}
-NOKPROBE_SYMBOL(ftrace_int3_handler);
-
-static int ftrace_write(unsigned long ip, const char *val, int size)
-{
- ip = text_ip_addr(ip);
-
- if (probe_kernel_write((void *)ip, val, size))
- return -EPERM;
-
- return 0;
-}
-
-static int add_break(unsigned long ip, const char *old)
-{
- unsigned char replaced[MCOUNT_INSN_SIZE];
- unsigned char brk = BREAKPOINT_INSTRUCTION;
-
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
- return -EFAULT;
-
- ftrace_expected = old;
-
- /* Make sure it is what we expect it to be */
- if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
- return -EINVAL;
-
- return ftrace_write(ip, &brk, 1);
-}
-
-static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
-{
- unsigned const char *old;
- unsigned long ip = rec->ip;
-
- old = ftrace_call_replace(ip, addr);
-
- return add_break(rec->ip, old);
-}
-
-
-static int add_brk_on_nop(struct dyn_ftrace *rec)
-{
- unsigned const char *old;
-
- old = ftrace_nop_replace();
-
- return add_break(rec->ip, old);
-}
-
-static int add_breakpoints(struct dyn_ftrace *rec, bool enable)
-{
- unsigned long ftrace_addr;
- int ret;
-
- ftrace_addr = ftrace_get_addr_curr(rec);
-
- ret = ftrace_test_record(rec, enable);
-
- switch (ret) {
- case FTRACE_UPDATE_IGNORE:
- return 0;
-
- case FTRACE_UPDATE_MAKE_CALL:
- /* converting nop to call */
- return add_brk_on_nop(rec);
-
- case FTRACE_UPDATE_MODIFY_CALL:
- case FTRACE_UPDATE_MAKE_NOP:
- /* converting a call to a nop */
- return add_brk_on_call(rec, ftrace_addr);
- }
- return 0;
-}
-
-/*
- * On error, we need to remove breakpoints. This needs to
- * be done caefully. If the address does not currently have a
- * breakpoint, we know we are done. Otherwise, we look at the
- * remaining 4 bytes of the instruction. If it matches a nop
- * we replace the breakpoint with the nop. Otherwise we replace
- * it with the call instruction.
- */
-static int remove_breakpoint(struct dyn_ftrace *rec)
-{
- unsigned char ins[MCOUNT_INSN_SIZE];
- unsigned char brk = BREAKPOINT_INSTRUCTION;
- const unsigned char *nop;
- unsigned long ftrace_addr;
- unsigned long ip = rec->ip;
-
- /* If we fail the read, just give up */
- if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
- return -EFAULT;
-
- /* If this does not have a breakpoint, we are done */
- if (ins[0] != brk)
- return 0;
-
- nop = ftrace_nop_replace();
-
- /*
- * If the last 4 bytes of the instruction do not match
- * a nop, then we assume that this is a call to ftrace_addr.
- */
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
- /*
- * For extra paranoidism, we check if the breakpoint is on
- * a call that would actually jump to the ftrace_addr.
- * If not, don't touch the breakpoint, we make just create
- * a disaster.
- */
- ftrace_addr = ftrace_get_addr_new(rec);
- nop = ftrace_call_replace(ip, ftrace_addr);
-
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
- goto update;
-
- /* Check both ftrace_addr and ftrace_old_addr */
- ftrace_addr = ftrace_get_addr_curr(rec);
- nop = ftrace_call_replace(ip, ftrace_addr);
-
- ftrace_expected = nop;
-
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
- return -EINVAL;
- }
-
- update:
- return ftrace_write(ip, nop, 1);
-}
-
-static int add_update_code(unsigned long ip, unsigned const char *new)
-{
- /* skip breakpoint */
- ip++;
- new++;
- return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
-}
-
-static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
-{
- unsigned long ip = rec->ip;
- unsigned const char *new;
-
- new = ftrace_call_replace(ip, addr);
- return add_update_code(ip, new);
-}
-
-static int add_update_nop(struct dyn_ftrace *rec)
-{
- unsigned long ip = rec->ip;
- unsigned const char *new;
-
- new = ftrace_nop_replace();
- return add_update_code(ip, new);
-}
-
-static int add_update(struct dyn_ftrace *rec, bool enable)
-{
- unsigned long ftrace_addr;
- int ret;
-
- ret = ftrace_test_record(rec, enable);
-
- ftrace_addr = ftrace_get_addr_new(rec);
-
- switch (ret) {
- case FTRACE_UPDATE_IGNORE:
- return 0;
-
- case FTRACE_UPDATE_MODIFY_CALL:
- case FTRACE_UPDATE_MAKE_CALL:
- /* converting nop to call */
- return add_update_call(rec, ftrace_addr);
-
- case FTRACE_UPDATE_MAKE_NOP:
- /* converting a call to a nop */
- return add_update_nop(rec);
- }
-
- return 0;
-}
-
-static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
-{
- unsigned long ip = rec->ip;
- unsigned const char *new;
-
- new = ftrace_call_replace(ip, addr);
-
- return ftrace_write(ip, new, 1);
-}
-
-static int finish_update_nop(struct dyn_ftrace *rec)
-{
- unsigned long ip = rec->ip;
- unsigned const char *new;
-
- new = ftrace_nop_replace();
-
- return ftrace_write(ip, new, 1);
-}
-
-static int finish_update(struct dyn_ftrace *rec, bool enable)
-{
- unsigned long ftrace_addr;
- int ret;
-
- ret = ftrace_update_record(rec, enable);
-
- ftrace_addr = ftrace_get_addr_new(rec);
-
- switch (ret) {
- case FTRACE_UPDATE_IGNORE:
- return 0;
-
- case FTRACE_UPDATE_MODIFY_CALL:
- case FTRACE_UPDATE_MAKE_CALL:
- /* converting nop to call */
- return finish_update_call(rec, ftrace_addr);
-
- case FTRACE_UPDATE_MAKE_NOP:
- /* converting a call to a nop */
- return finish_update_nop(rec);
- }
+ text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL);
return 0;
}
-static void do_sync_core(void *data)
-{
- sync_core();
-}
-
-static void run_sync(void)
-{
- int enable_irqs;
-
- /* No need to sync if there's only one CPU */
- if (num_online_cpus() == 1)
- return;
-
- enable_irqs = irqs_disabled();
-
- /* We may be called with interrupts disabled (on bootup). */
- if (enable_irqs)
- local_irq_enable();
- on_each_cpu(do_sync_core, NULL, 1);
- if (enable_irqs)
- local_irq_disable();
-}
-
void ftrace_replace_code(int enable)
{
struct ftrace_rec_iter *iter;
+ struct text_poke_loc *tp;
struct dyn_ftrace *rec;
- const char *report = "adding breakpoints";
- int count = 0;
- int ret;
+ const char *new;
for_ftrace_rec_iter(iter) {
rec = ftrace_rec_iter_record(iter);
- ret = add_breakpoints(rec, enable);
- if (ret)
- goto remove_breakpoints;
- count++;
- }
-
- run_sync();
-
- report = "updating code";
- count = 0;
-
- for_ftrace_rec_iter(iter) {
- rec = ftrace_rec_iter_record(iter);
-
- ret = add_update(rec, enable);
- if (ret)
- goto remove_breakpoints;
- count++;
- }
-
- run_sync();
-
- report = "removing breakpoints";
- count = 0;
-
- for_ftrace_rec_iter(iter) {
- rec = ftrace_rec_iter_record(iter);
-
- ret = finish_update(rec, enable);
- if (ret)
- goto remove_breakpoints;
- count++;
- }
-
- run_sync();
+ if (tp_vec_nr == TP_VEC_MAX) {
+ text_poke_bp_batch(tp_vec, tp_vec_nr);
+ tp_vec_nr = 0;
+ }
- return;
+ switch (ftrace_test_record(rec, enable)) {
+ case FTRACE_UPDATE_IGNORE:
+ continue;
+
+ case FTRACE_UPDATE_MODIFY_CALL:
+ case FTRACE_UPDATE_MAKE_CALL:
+ new = ftrace_call_replace(rec->ip, ftrace_get_addr_new(rec));
+ break;
+
+ case FTRACE_UPDATE_MAKE_NOP:
+ new = ftrace_nop_replace();
+ break;
+ };
- remove_breakpoints:
- pr_warn("Failed on %s (%d):\n", report, count);
- ftrace_bug(ret, rec);
- for_ftrace_rec_iter(iter) {
- rec = ftrace_rec_iter_record(iter);
- /*
- * Breakpoints are handled only when this function is in
- * progress. The system could not work with them.
- */
- if (remove_breakpoint(rec))
- BUG();
+ tp = &tp_vec[tp_vec_nr++];
+ text_poke_loc_init(tp, (void *)rec->ip, new, MCOUNT_INSN_SIZE, NULL);
}
- run_sync();
-}
-
-static int
-ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
- unsigned const char *new_code)
-{
- int ret;
-
- ret = add_break(ip, old_code);
- if (ret)
- goto out;
-
- run_sync();
-
- ret = add_update_code(ip, new_code);
- if (ret)
- goto fail_update;
-
- run_sync();
-
- ret = ftrace_write(ip, new_code, 1);
- /*
- * The breakpoint is handled only when this function is in progress.
- * The system could not work if we could not remove it.
- */
- BUG_ON(ret);
- out:
- run_sync();
- return ret;
-
- fail_update:
- /* Also here the system could not work with the breakpoint */
- if (ftrace_write(ip, old_code, 1))
- BUG();
- goto out;
}
void arch_ftrace_update_code(int command)
{
- /* See comment above by declaration of modifying_ftrace_code */
- atomic_inc(&modifying_ftrace_code);
-
ftrace_modify_all_code(command);
-
- atomic_dec(&modifying_ftrace_code);
}
int __init ftrace_dyn_arch_init(void)
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -568,15 +568,6 @@ NOKPROBE_SYMBOL(do_general_protection);
dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
{
-#ifdef CONFIG_DYNAMIC_FTRACE
- /*
- * ftrace must be first, everything else may cause a recursive crash.
- * See note by declaration of modifying_ftrace_code in ftrace.c
- */
- if (unlikely(atomic_read(&modifying_ftrace_code)) &&
- ftrace_int3_handler(regs))
- return;
-#endif
if (poke_int3_handler(regs))
return;
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 0/3] Rewrite x86/ftrace to use text_poke()
2019-08-26 12:51 [PATCH 0/3] Rewrite x86/ftrace to use text_poke() Peter Zijlstra
` (2 preceding siblings ...)
2019-08-26 12:51 ` [RFC][PATCH 3/3] x86/ftrace: Use text_poke() Peter Zijlstra
@ 2019-08-26 13:04 ` Peter Zijlstra
3 siblings, 0 replies; 15+ messages in thread
From: Peter Zijlstra @ 2019-08-26 13:04 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Nadav Amit, Andy Lutomirski, Dave Hansen, Song Liu,
Masami Hiramatsu, Steven Rostedt, Daniel Bristot de Oliveira
On Mon, Aug 26, 2019 at 02:51:38PM +0200, Peter Zijlstra wrote:
> Ftrace was one of the last W^X violators; these patches move it over to the
> generic text_poke() interface and thereby get rid of this oddity.
>
> Very lightly tested...
I'm thinking there's more cleanup to be had; set_kernel_text_*() can go
away now, and I'm thinking the whole batching thing can be further
simplified/shared between this and jump_label.
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 2/3] x86/alternatives: Move tp_vec
2019-08-26 12:51 ` [PATCH 2/3] x86/alternatives: Move tp_vec Peter Zijlstra
@ 2019-08-26 14:36 ` Peter Zijlstra
0 siblings, 0 replies; 15+ messages in thread
From: Peter Zijlstra @ 2019-08-26 14:36 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Nadav Amit, Andy Lutomirski, Dave Hansen, Song Liu,
Daniel Bristot de Oliveira, Steven Rostedt
On Mon, Aug 26, 2019 at 02:51:40PM +0200, Peter Zijlstra wrote:
> +#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc))
> +extern struct text_poke_loc tp_vec[TP_VEC_MAX];
> +extern int tp_vec_nr;
FWIW, that currently results in a batch size of 128, but I've not
noticed any delay in flipping ftrace on and off on the commandline.
Growing that buffer really shouldn't be a problem, but I'm thinking we'd
want solid performance numbers to justify it.
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [RFC][PATCH 3/3] x86/ftrace: Use text_poke()
2019-08-26 12:51 ` [RFC][PATCH 3/3] x86/ftrace: Use text_poke() Peter Zijlstra
@ 2019-08-26 15:13 ` Peter Zijlstra
0 siblings, 0 replies; 15+ messages in thread
From: Peter Zijlstra @ 2019-08-26 15:13 UTC (permalink / raw)
To: linux-kernel, x86
Cc: Nadav Amit, Andy Lutomirski, Dave Hansen, Song Liu,
Daniel Bristot de Oliveira, Steven Rostedt
On Mon, Aug 26, 2019 at 02:51:41PM +0200, Peter Zijlstra wrote:
> Move ftrace over to using the generic x86 text_poke functions; this
> avoids having a second/different copy of that code around.
>
> Cc: Daniel Bristot de Oliveira <bristot@redhat.com>
> Cc: Steven Rostedt <rostedt@goodmis.org>
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
*sigh*.. one of those days again. I seem to have tested without this
patch applied.
Let me go fix this.
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions
2019-10-04 13:45 ` Masami Hiramatsu
@ 2019-10-07 8:05 ` Peter Zijlstra
0 siblings, 0 replies; 15+ messages in thread
From: Peter Zijlstra @ 2019-10-07 8:05 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: linux-kernel, x86, Nadav Amit, Andy Lutomirski, Dave Hansen,
Song Liu, Steven Rostedt, Daniel Bristot de Oliveira
On Fri, Oct 04, 2019 at 10:45:40PM +0900, Masami Hiramatsu wrote:
> Hi Peter,
>
> On Thu, 3 Oct 2019 13:01:06 +0200
> Peter Zijlstra <peterz@infradead.org> wrote:
> > > I'm halfway through a patch introducing:
> > >
> > > union text_poke_insn {
> > > u8 code[POKE_MAX_OPCODE_SUZE];
> > > struct {
> > > u8 opcode;
> > > s32 disp;
> > > } __attribute__((packed));
> > > };
> > >
> > > to text-patching.h to unify all such custom unions we have all over the
> > > place. I'll mob up the above in that.
>
> I think it is good to unify such unions, but I meant above was, it was
> also important to unify the opcode macro. Since poke_int3_handler()
> clasifies the opcode by your *_INSN_OPCODE macro, it is natual to use
> those opcode for text_poke_bp() interface.
Right, but I think we should do that as another patch, there's a lot of
instances in that file and just changing one or two over is 'weird'.
I can put it on the todo to fix that all up.
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions
2019-10-03 11:01 ` Peter Zijlstra
2019-10-03 12:32 ` Peter Zijlstra
@ 2019-10-04 13:45 ` Masami Hiramatsu
2019-10-07 8:05 ` Peter Zijlstra
1 sibling, 1 reply; 15+ messages in thread
From: Masami Hiramatsu @ 2019-10-04 13:45 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-kernel, x86, Nadav Amit, Andy Lutomirski, Dave Hansen,
Song Liu, Steven Rostedt, Daniel Bristot de Oliveira
Hi Peter,
On Thu, 3 Oct 2019 13:01:06 +0200
Peter Zijlstra <peterz@infradead.org> wrote:
> On Thu, Oct 03, 2019 at 10:27:51AM +0200, Peter Zijlstra wrote:
> > On Thu, Oct 03, 2019 at 02:00:50PM +0900, Masami Hiramatsu wrote:
> >
> > > > This fits almost all text_poke_bp() users, except
> > > > arch_unoptimize_kprobe() which restores random text, and for that site
> > > > we have to build an explicit emulate instruction.
> > >
> > > OK, and in this case, I would like to change RELATIVEJUMP_OPCODE
> > > to JMP32_INSN_OPCODE for readability. (or at least
> > > making RELATIVEJUMP_OPCODE as an alias of JMP32_INSN_OPCODE)
> >
> > > > @@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
> > > > void arch_unoptimize_kprobe(struct optimized_kprobe *op)
> > > > {
> > > > u8 insn_buff[RELATIVEJUMP_SIZE];
> > > > + u8 emulate_buff[RELATIVEJUMP_SIZE];
> > > >
> > > > /* Set int3 to first byte for kprobes */
> > > > insn_buff[0] = BREAKPOINT_INSTRUCTION;
> > > > memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
> > > > +
> > > > + emulate_buff[0] = RELATIVEJUMP_OPCODE;
> > > > + *(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
> > > > + ((long)op->kp.addr + RELATIVEJUMP_SIZE));
> >
> > I'm halfway through a patch introducing:
> >
> > union text_poke_insn {
> > u8 code[POKE_MAX_OPCODE_SUZE];
> > struct {
> > u8 opcode;
> > s32 disp;
> > } __attribute__((packed));
> > };
> >
> > to text-patching.h to unify all such custom unions we have all over the
> > place. I'll mob up the above in that.
I think it is good to unify such unions, but I meant above was, it was
also important to unify the opcode macro. Since poke_int3_handler()
clasifies the opcode by your *_INSN_OPCODE macro, it is natual to use
those opcode for text_poke_bp() interface.
> > > > +
> > > > text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
> > > > - op->optinsn.insn);
> > > > + emulate_buff);
> > > > }
> >
> > As argued in a previous thread, text_poke_bp() is broken when it changes
> > more than a single instruction at a time.
> >
> > Now, ISTR optimized kprobes does something like:
> >
> > poke INT3
>
> Hmm, it does this using text_poke(), but lacks a
> on_each_cpu(do_sync_core, NULL, 1), which I suppose is OK-ish IFF you do
> that synchronize_rcu_tasks() after it, but less so if you don't.
>
> That is, without either, you can't really tell if the kprobe is in
> effect or not.
Yes, it doesn't wait the change by design at this moment.
> Also, I think text_poke_bp(INT3) is broken, although I don't think
> anybody actually does that. Still, let me fix that.
OK.
>
> > synchronize_rcu_tasks() /* waits for all tasks to schedule
> > guarantees instructions after INT3
> > are unused */
> > install optimized probe /* overwrites multiple instrctions with
> > JMP.d32 */
> >
> > And the above then undoes that by:
> >
> > poke INT3 on top of the optimzed probe
> >
> > poke tail instructions back /* guaranteed safe because the
> > above INT3 poke ensures the
> > JMP.d32 instruction is unused */
> >
> > poke head byte back
Yes, anyway, the last poke should recover another INT3... (for kprobe)
> >
> > Is this correct? If so, we should probably put a comment in there
> > explaining how all this is unusual but safe.
OK.
Thank you,
--
Masami Hiramatsu <mhiramat@kernel.org>
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions
2019-10-03 8:27 ` Peter Zijlstra
2019-10-03 11:01 ` Peter Zijlstra
@ 2019-10-03 13:05 ` Peter Zijlstra
1 sibling, 0 replies; 15+ messages in thread
From: Peter Zijlstra @ 2019-10-03 13:05 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: linux-kernel, x86, Nadav Amit, Andy Lutomirski, Dave Hansen,
Song Liu, Steven Rostedt, Daniel Bristot de Oliveira
On Thu, Oct 03, 2019 at 10:27:51AM +0200, Peter Zijlstra wrote:
> On Thu, Oct 03, 2019 at 02:00:50PM +0900, Masami Hiramatsu wrote:
>
> > > This fits almost all text_poke_bp() users, except
> > > arch_unoptimize_kprobe() which restores random text, and for that site
> > > we have to build an explicit emulate instruction.
> >
> > OK, and in this case, I would like to change RELATIVEJUMP_OPCODE
> > to JMP32_INSN_OPCODE for readability. (or at least
> > making RELATIVEJUMP_OPCODE as an alias of JMP32_INSN_OPCODE)
>
> > > @@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
> > > void arch_unoptimize_kprobe(struct optimized_kprobe *op)
> > > {
> > > u8 insn_buff[RELATIVEJUMP_SIZE];
> > > + u8 emulate_buff[RELATIVEJUMP_SIZE];
> > >
> > > /* Set int3 to first byte for kprobes */
> > > insn_buff[0] = BREAKPOINT_INSTRUCTION;
> > > memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
> > > +
> > > + emulate_buff[0] = RELATIVEJUMP_OPCODE;
> > > + *(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
> > > + ((long)op->kp.addr + RELATIVEJUMP_SIZE));
>
> I'm halfway through a patch introducing:
>
> union text_poke_insn {
> u8 code[POKE_MAX_OPCODE_SUZE];
> struct {
> u8 opcode;
> s32 disp;
> } __attribute__((packed));
> };
>
> to text-patching.h to unify all such custom unions we have all over the
> place. I'll mob up the above in that.
How's something like so?
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -49,6 +49,8 @@ extern void text_poke_bp(void *addr, con
extern void text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate);
extern void text_poke_finish(void);
+extern void *text_gen_insn(u8 opcode, unsigned long addr, unsigned long dest);
+
extern int after_bootmem;
extern __ro_after_init struct mm_struct *poking_mm;
extern __ro_after_init unsigned long poking_addr;
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -1237,3 +1237,39 @@ void text_poke_bp(void *addr, const void
text_poke_loc_init(&tp, addr, opcode, len, emulate);
text_poke_bp_batch(&tp, 1);
}
+
+union text_poke_insn {
+ u8 text[POKE_MAX_OPCODE_SIZE];
+ struct {
+ u8 opcode;
+ s32 disp;
+ } __attribute__((packed));
+};
+
+void *text_gen_insn(u8 opcode, unsigned long addr, unsigned long dest)
+{
+ static union text_poke_insn insn; /* text_mutex */
+ int size = 0;
+
+ lockdep_assert_held(&text_mutex);
+
+ insn.opcode = opcode;
+
+#define __CASE(insn) \
+ case insn##_INSN_OPCODE: size = insn##_INSN_SIZE; break
+
+ switch(opcode) {
+ __CASE(INT3);
+ __CASE(CALL);
+ __CASE(JMP32);
+ __CASE(JMP8);
+ }
+
+ if (size > 1) {
+ insn.disp = dest - (addr + size);
+ if (size == 2)
+ BUG_ON((insn.disp >> 31) != (insn.disp >> 7));
+ }
+
+ return &insn.text;
+}
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -16,14 +16,6 @@
#include <asm/alternative.h>
#include <asm/text-patching.h>
-union jump_code_union {
- char code[JUMP_LABEL_NOP_SIZE];
- struct {
- char jump;
- int offset;
- } __attribute__((packed));
-};
-
static void bug_at(unsigned char *ip, int line)
{
/*
@@ -38,33 +30,29 @@ static void bug_at(unsigned char *ip, in
static const void *
__jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type, int init)
{
- static union jump_code_union code; /* relies on text_mutex */
const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
- const void *expect;
+ const void *expect, *code;
int line;
- lockdep_assert_held(&text_mutex);
-
- code.jump = JMP32_INSN_OPCODE;
- code.offset = jump_entry_target(entry) -
- (jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
+ code = text_gen_insn(JMP32_INSN_OPCODE, jump_entry_code(entry),
+ jump_entry_target(entry));
if (init) {
expect = default_nop; line = __LINE__;
} else if (type == JUMP_LABEL_JMP) {
expect = ideal_nop; line = __LINE__;
} else {
- expect = code.code; line = __LINE__;
+ expect = code; line = __LINE__;
}
if (memcmp((void *)jump_entry_code(entry), expect, JUMP_LABEL_NOP_SIZE))
bug_at((void *)jump_entry_code(entry), line);
if (type == JUMP_LABEL_NOP)
- memcpy(&code, ideal_nop, JUMP_LABEL_NOP_SIZE);
+ code = ideal_nop;
- return &code;
+ return code;
}
static void inline __jump_label_transform(struct jump_entry *entry,
--- a/arch/x86/kernel/kprobes/opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -447,18 +447,13 @@ void arch_optimize_kprobes(struct list_h
void arch_unoptimize_kprobe(struct optimized_kprobe *op)
{
u8 insn_buff[RELATIVEJUMP_SIZE];
- u8 emulate_buff[RELATIVEJUMP_SIZE];
/* Set int3 to first byte for kprobes */
insn_buff[0] = BREAKPOINT_INSTRUCTION;
memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
- emulate_buff[0] = RELATIVEJUMP_OPCODE;
- *(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
- ((long)op->kp.addr + RELATIVEJUMP_SIZE));
-
text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
- emulate_buff);
+ text_gen_insn(JMP32_INSN_OPCODE, (long)op->kp.addr, (long)op->optinsn.insn));
}
/*
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions
2019-10-03 11:01 ` Peter Zijlstra
@ 2019-10-03 12:32 ` Peter Zijlstra
2019-10-04 13:45 ` Masami Hiramatsu
1 sibling, 0 replies; 15+ messages in thread
From: Peter Zijlstra @ 2019-10-03 12:32 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: linux-kernel, x86, Nadav Amit, Andy Lutomirski, Dave Hansen,
Song Liu, Steven Rostedt, Daniel Bristot de Oliveira
On Thu, Oct 03, 2019 at 01:01:06PM +0200, Peter Zijlstra wrote:
> Also, I think text_poke_bp(INT3) is broken, although I don't think
> anybody actually does that. Still, let me fix that.
Something like so should allow text_poke_bp(INT3) to work as expected.
---
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -999,6 +999,13 @@ int poke_int3_handler(struct pt_regs *re
ip += tp->len;
switch (tp->opcode) {
+ case INT3_INSN_OPCODE:
+ /*
+ * Someone poked an explicit INT3, they'll want to handle it,
+ * do not consume.
+ */
+ return 0;
+
case CALL_INSN_OPCODE:
int3_emulate_call(regs, (long)ip + tp->rel32);
break;
@@ -1040,8 +1047,8 @@ NOKPROBE_SYMBOL(poke_int3_handler);
void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries)
{
unsigned char int3 = INT3_INSN_OPCODE;
- int patched_all_but_first = 0;
unsigned int i;
+ int do_sync;
lockdep_assert_held(&text_mutex);
@@ -1065,16 +1072,16 @@ void text_poke_bp_batch(struct text_poke
/*
* Second step: update all but the first byte of the patched range.
*/
- for (i = 0; i < nr_entries; i++) {
+ for (do_sync = 0, i = 0; i < nr_entries; i++) {
if (tp[i].len - sizeof(int3) > 0) {
text_poke((char *)tp[i].addr + sizeof(int3),
(const char *)tp[i].text + sizeof(int3),
tp[i].len - sizeof(int3));
- patched_all_but_first++;
+ do_sync++;
}
}
- if (patched_all_but_first) {
+ if (do_sync) {
/*
* According to Intel, this core syncing is very likely
* not necessary and we'd be safe even without it. But
@@ -1087,10 +1094,17 @@ void text_poke_bp_batch(struct text_poke
* Third step: replace the first byte (int3) by the first byte of
* replacing opcode.
*/
- for (i = 0; i < nr_entries; i++)
+ for (do_sync = 0, i = 0; i < nr_entries; i++) {
+ if (tp[i].text[0] == INT3_INSN_OPCODE)
+ continue;
+
text_poke(tp[i].addr, tp[i].text, sizeof(int3));
+ do_sync++;
+ }
+
+ if (do_sync)
+ on_each_cpu(do_sync_core, NULL, 1);
- on_each_cpu(do_sync_core, NULL, 1);
/*
* sync_core() implies an smp_mb() and orders this store against
* the writing of the new instruction.
@@ -1123,6 +1137,9 @@ void text_poke_loc_init(struct text_poke
tp->opcode = insn.opcode.bytes[0];
switch (tp->opcode) {
+ case INT3_INSN_OPCPDE:
+ break;
+
case CALL_INSN_OPCODE:
case JMP32_INSN_OPCODE:
case JMP8_INSN_OPCODE:
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions
2019-10-03 8:27 ` Peter Zijlstra
@ 2019-10-03 11:01 ` Peter Zijlstra
2019-10-03 12:32 ` Peter Zijlstra
2019-10-04 13:45 ` Masami Hiramatsu
2019-10-03 13:05 ` Peter Zijlstra
1 sibling, 2 replies; 15+ messages in thread
From: Peter Zijlstra @ 2019-10-03 11:01 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: linux-kernel, x86, Nadav Amit, Andy Lutomirski, Dave Hansen,
Song Liu, Steven Rostedt, Daniel Bristot de Oliveira
On Thu, Oct 03, 2019 at 10:27:51AM +0200, Peter Zijlstra wrote:
> On Thu, Oct 03, 2019 at 02:00:50PM +0900, Masami Hiramatsu wrote:
>
> > > This fits almost all text_poke_bp() users, except
> > > arch_unoptimize_kprobe() which restores random text, and for that site
> > > we have to build an explicit emulate instruction.
> >
> > OK, and in this case, I would like to change RELATIVEJUMP_OPCODE
> > to JMP32_INSN_OPCODE for readability. (or at least
> > making RELATIVEJUMP_OPCODE as an alias of JMP32_INSN_OPCODE)
>
> > > @@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
> > > void arch_unoptimize_kprobe(struct optimized_kprobe *op)
> > > {
> > > u8 insn_buff[RELATIVEJUMP_SIZE];
> > > + u8 emulate_buff[RELATIVEJUMP_SIZE];
> > >
> > > /* Set int3 to first byte for kprobes */
> > > insn_buff[0] = BREAKPOINT_INSTRUCTION;
> > > memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
> > > +
> > > + emulate_buff[0] = RELATIVEJUMP_OPCODE;
> > > + *(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
> > > + ((long)op->kp.addr + RELATIVEJUMP_SIZE));
>
> I'm halfway through a patch introducing:
>
> union text_poke_insn {
> u8 code[POKE_MAX_OPCODE_SUZE];
> struct {
> u8 opcode;
> s32 disp;
> } __attribute__((packed));
> };
>
> to text-patching.h to unify all such custom unions we have all over the
> place. I'll mob up the above in that.
>
> > > +
> > > text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
> > > - op->optinsn.insn);
> > > + emulate_buff);
> > > }
>
> As argued in a previous thread, text_poke_bp() is broken when it changes
> more than a single instruction at a time.
>
> Now, ISTR optimized kprobes does something like:
>
> poke INT3
Hmm, it does this using text_poke(), but lacks a
on_each_cpu(do_sync_core, NULL, 1), which I suppose is OK-ish IFF you do
that synchronize_rcu_tasks() after it, but less so if you don't.
That is, without either, you can't really tell if the kprobe is in
effect or not.
Also, I think text_poke_bp(INT3) is broken, although I don't think
anybody actually does that. Still, let me fix that.
> synchronize_rcu_tasks() /* waits for all tasks to schedule
> guarantees instructions after INT3
> are unused */
> install optimized probe /* overwrites multiple instrctions with
> JMP.d32 */
>
> And the above then undoes that by:
>
> poke INT3 on top of the optimzed probe
>
> poke tail instructions back /* guaranteed safe because the
> above INT3 poke ensures the
> JMP.d32 instruction is unused */
>
> poke head byte back
>
> Is this correct? If so, we should probably put a comment in there
> explaining how all this is unusual but safe.
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions
2019-10-03 5:00 ` Masami Hiramatsu
@ 2019-10-03 8:27 ` Peter Zijlstra
2019-10-03 11:01 ` Peter Zijlstra
2019-10-03 13:05 ` Peter Zijlstra
0 siblings, 2 replies; 15+ messages in thread
From: Peter Zijlstra @ 2019-10-03 8:27 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: linux-kernel, x86, Nadav Amit, Andy Lutomirski, Dave Hansen,
Song Liu, Steven Rostedt, Daniel Bristot de Oliveira
On Thu, Oct 03, 2019 at 02:00:50PM +0900, Masami Hiramatsu wrote:
> > This fits almost all text_poke_bp() users, except
> > arch_unoptimize_kprobe() which restores random text, and for that site
> > we have to build an explicit emulate instruction.
>
> OK, and in this case, I would like to change RELATIVEJUMP_OPCODE
> to JMP32_INSN_OPCODE for readability. (or at least
> making RELATIVEJUMP_OPCODE as an alias of JMP32_INSN_OPCODE)
> > @@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
> > void arch_unoptimize_kprobe(struct optimized_kprobe *op)
> > {
> > u8 insn_buff[RELATIVEJUMP_SIZE];
> > + u8 emulate_buff[RELATIVEJUMP_SIZE];
> >
> > /* Set int3 to first byte for kprobes */
> > insn_buff[0] = BREAKPOINT_INSTRUCTION;
> > memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
> > +
> > + emulate_buff[0] = RELATIVEJUMP_OPCODE;
> > + *(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
> > + ((long)op->kp.addr + RELATIVEJUMP_SIZE));
I'm halfway through a patch introducing:
union text_poke_insn {
u8 code[POKE_MAX_OPCODE_SUZE];
struct {
u8 opcode;
s32 disp;
} __attribute__((packed));
};
to text-patching.h to unify all such custom unions we have all over the
place. I'll mob up the above in that.
> > +
> > text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
> > - op->optinsn.insn);
> > + emulate_buff);
> > }
As argued in a previous thread, text_poke_bp() is broken when it changes
more than a single instruction at a time.
Now, ISTR optimized kprobes does something like:
poke INT3
synchronize_rcu_tasks() /* waits for all tasks to schedule
guarantees instructions after INT3
are unused */
install optimized probe /* overwrites multiple instrctions with
JMP.d32 */
And the above then undoes that by:
poke INT3 on top of the optimzed probe
poke tail instructions back /* guaranteed safe because the
above INT3 poke ensures the
JMP.d32 instruction is unused */
poke head byte back
Is this correct? If so, we should probably put a comment in there
explaining how all this is unusual but safe.
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions
2019-08-27 18:06 ` [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions Peter Zijlstra
@ 2019-10-03 5:00 ` Masami Hiramatsu
2019-10-03 8:27 ` Peter Zijlstra
0 siblings, 1 reply; 15+ messages in thread
From: Masami Hiramatsu @ 2019-10-03 5:00 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-kernel, x86, Nadav Amit, Andy Lutomirski, Dave Hansen,
Song Liu, Masami Hiramatsu, Steven Rostedt,
Daniel Bristot de Oliveira
Hi Peter,
On Tue, 27 Aug 2019 20:06:23 +0200
Peter Zijlstra <peterz@infradead.org> wrote:
> In preparation for static_call and variable size jump_label support,
> teach text_poke_bp() to emulate instructions, namely:
>
> JMP32, JMP8, CALL, NOP2, NOP_ATOMIC5
>
> The current text_poke_bp() takes a @handler argument which is used as
> a jump target when the temporary INT3 is hit by a different CPU.
>
> When patching CALL instructions, this doesn't work because we'd miss
> the PUSH of the return address. Instead, teach poke_int3_handler() to
> emulate an instruction, typically the instruction we're patching in.
>
> This fits almost all text_poke_bp() users, except
> arch_unoptimize_kprobe() which restores random text, and for that site
> we have to build an explicit emulate instruction.
OK, and in this case, I would like to change RELATIVEJUMP_OPCODE
to JMP32_INSN_OPCODE for readability. (or at least
making RELATIVEJUMP_OPCODE as an alias of JMP32_INSN_OPCODE)
Except for that, this looks good to me.
Reviewed-by: Masami Hiramatsu <mhiramat@kernel.org>
Thank you,
>
> Cc: Masami Hiramatsu <mhiramat@kernel.org>
> Cc: Steven Rostedt <rostedt@goodmis.org>
> Cc: Daniel Bristot de Oliveira <bristot@redhat.com>
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
> ---
> arch/x86/include/asm/text-patching.h | 24 ++++++--
> arch/x86/kernel/alternative.c | 98 ++++++++++++++++++++++++++---------
> arch/x86/kernel/jump_label.c | 9 +--
> arch/x86/kernel/kprobes/opt.c | 11 ++-
> 4 files changed, 103 insertions(+), 39 deletions(-)
>
> --- a/arch/x86/include/asm/text-patching.h
> +++ b/arch/x86/include/asm/text-patching.h
> @@ -26,10 +26,11 @@ static inline void apply_paravirt(struct
> #define POKE_MAX_OPCODE_SIZE 5
>
> struct text_poke_loc {
> - void *detour;
> void *addr;
> - size_t len;
> - const char opcode[POKE_MAX_OPCODE_SIZE];
> + int len;
> + s32 rel32;
> + u8 opcode;
> + const char text[POKE_MAX_OPCODE_SIZE];
> };
>
> extern void text_poke_early(void *addr, const void *opcode, size_t len);
> @@ -51,8 +52,10 @@ extern void text_poke_early(void *addr,
> extern void *text_poke(void *addr, const void *opcode, size_t len);
> extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
> extern int poke_int3_handler(struct pt_regs *regs);
> -extern void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
> +extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
> extern void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries);
> +extern void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
> + const void *opcode, size_t len, const void *emulate);
> extern int after_bootmem;
> extern __ro_after_init struct mm_struct *poking_mm;
> extern __ro_after_init unsigned long poking_addr;
> @@ -63,8 +66,17 @@ static inline void int3_emulate_jmp(stru
> regs->ip = ip;
> }
>
> -#define INT3_INSN_SIZE 1
> -#define CALL_INSN_SIZE 5
> +#define INT3_INSN_SIZE 1
> +#define INT3_INSN_OPCODE 0xCC
> +
> +#define CALL_INSN_SIZE 5
> +#define CALL_INSN_OPCODE 0xE8
> +
> +#define JMP32_INSN_SIZE 5
> +#define JMP32_INSN_OPCODE 0xE9
> +
> +#define JMP8_INSN_SIZE 2
> +#define JMP8_INSN_OPCODE 0xEB
>
> static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
> {
> --- a/arch/x86/kernel/alternative.c
> +++ b/arch/x86/kernel/alternative.c
> @@ -956,16 +956,15 @@ NOKPROBE_SYMBOL(patch_cmp);
> int poke_int3_handler(struct pt_regs *regs)
> {
> struct text_poke_loc *tp;
> - unsigned char int3 = 0xcc;
> void *ip;
>
> /*
> * Having observed our INT3 instruction, we now must observe
> * bp_patching.nr_entries.
> *
> - * nr_entries != 0 INT3
> - * WMB RMB
> - * write INT3 if (nr_entries)
> + * nr_entries != 0 INT3
> + * WMB RMB
> + * write INT3 if (nr_entries)
> *
> * Idem for other elements in bp_patching.
> */
> @@ -978,9 +977,9 @@ int poke_int3_handler(struct pt_regs *re
> return 0;
>
> /*
> - * Discount the sizeof(int3). See text_poke_bp_batch().
> + * Discount the INT3. See text_poke_bp_batch().
> */
> - ip = (void *) regs->ip - sizeof(int3);
> + ip = (void *) regs->ip - INT3_INSN_SIZE;
>
> /*
> * Skip the binary search if there is a single member in the vector.
> @@ -997,8 +996,22 @@ int poke_int3_handler(struct pt_regs *re
> return 0;
> }
>
> - /* set up the specified breakpoint detour */
> - regs->ip = (unsigned long) tp->detour;
> + ip += tp->len;
> +
> + switch (tp->opcode) {
> + case CALL_INSN_OPCODE:
> + int3_emulate_call(regs, (long)ip + tp->rel32);
> + break;
> +
> + case JMP32_INSN_OPCODE:
> + case JMP8_INSN_OPCODE:
> + int3_emulate_jmp(regs, (long)ip + tp->rel32);
> + break;
> +
> + default: /* nop */
> + int3_emulate_jmp(regs, (long)ip);
> + break;
> + }
>
> return 1;
> }
> @@ -1027,8 +1040,8 @@ NOKPROBE_SYMBOL(poke_int3_handler);
> */
> void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries)
> {
> + unsigned char int3 = INT3_INSN_OPCODE;
> int patched_all_but_first = 0;
> - unsigned char int3 = 0xcc;
> unsigned int i;
>
> lockdep_assert_held(&text_mutex);
> @@ -1056,7 +1069,7 @@ void text_poke_bp_batch(struct text_poke
> for (i = 0; i < nr_entries; i++) {
> if (tp[i].len - sizeof(int3) > 0) {
> text_poke((char *)tp[i].addr + sizeof(int3),
> - (const char *)tp[i].opcode + sizeof(int3),
> + (const char *)tp[i].text + sizeof(int3),
> tp[i].len - sizeof(int3));
> patched_all_but_first++;
> }
> @@ -1076,7 +1089,7 @@ void text_poke_bp_batch(struct text_poke
> * replacing opcode.
> */
> for (i = 0; i < nr_entries; i++)
> - text_poke(tp[i].addr, tp[i].opcode, sizeof(int3));
> + text_poke(tp[i].addr, tp[i].text, sizeof(int3));
>
> on_each_cpu(do_sync_core, NULL, 1);
> /*
> @@ -1087,6 +1100,53 @@ void text_poke_bp_batch(struct text_poke
> bp_patching.nr_entries = 0;
> }
>
> +void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
> + const void *opcode, size_t len, const void *emulate)
> +{
> + struct insn insn;
> +
> + if (!opcode)
> + opcode = (void *)tp->text;
> + else
> + memcpy((void *)tp->text, opcode, len);
> +
> + if (!emulate)
> + emulate = opcode;
> +
> + kernel_insn_init(&insn, emulate, MAX_INSN_SIZE);
> + insn_get_length(&insn);
> +
> + BUG_ON(!insn_complete(&insn));
> + BUG_ON(len != insn.length);
> +
> + tp->addr = addr;
> + tp->len = len;
> + tp->opcode = insn.opcode.bytes[0];
> +
> + switch (tp->opcode) {
> + case CALL_INSN_OPCODE:
> + case JMP32_INSN_OPCODE:
> + case JMP8_INSN_OPCODE:
> + tp->rel32 = insn.immediate.value;
> + break;
> +
> + default: /* assume NOP */
> + switch (len) {
> + case 2:
> + BUG_ON(memcmp(emulate, ideal_nops[len], len));
> + break;
> +
> + case 5:
> + BUG_ON(memcmp(emulate, ideal_nops[NOP_ATOMIC5], len));
> + break;
> +
> + default:
> + BUG();
> + }
> + break;
> + }
> +}
> +
> /**
> * text_poke_bp() -- update instructions on live kernel on SMP
> * @addr: address to patch
> @@ -1098,20 +1158,10 @@ void text_poke_bp_batch(struct text_poke
> * dynamically allocated memory. This function should be used when it is
> * not possible to allocate memory.
> */
> -void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
> +void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
> {
> - struct text_poke_loc tp = {
> - .detour = handler,
> - .addr = addr,
> - .len = len,
> - };
> -
> - if (len > POKE_MAX_OPCODE_SIZE) {
> - WARN_ONCE(1, "len is larger than %d\n", POKE_MAX_OPCODE_SIZE);
> - return;
> - }
> -
> - memcpy((void *)tp.opcode, opcode, len);
> + struct text_poke_loc tp;
>
> + text_poke_loc_init(&tp, addr, opcode, len, emulate);
> text_poke_bp_batch(&tp, 1);
> }
> --- a/arch/x86/kernel/jump_label.c
> +++ b/arch/x86/kernel/jump_label.c
> @@ -89,8 +89,7 @@ static void __ref __jump_label_transform
> return;
> }
>
> - text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE,
> - (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
> + text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE, NULL);
> }
>
> void arch_jump_label_transform(struct jump_entry *entry,
> @@ -147,11 +146,9 @@ bool arch_jump_label_transform_queue(str
> }
>
> __jump_label_set_jump_code(entry, type,
> - (union jump_code_union *) &tp->opcode, 0);
> + (union jump_code_union *)&tp->text, 0);
>
> - tp->addr = entry_code;
> - tp->detour = entry_code + JUMP_LABEL_NOP_SIZE;
> - tp->len = JUMP_LABEL_NOP_SIZE;
> + text_poke_loc_init(tp, entry_code, NULL, JUMP_LABEL_NOP_SIZE, NULL);
>
> tp_vec_nr++;
>
> --- a/arch/x86/kernel/kprobes/opt.c
> +++ b/arch/x86/kernel/kprobes/opt.c
> @@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h
> insn_buff[0] = RELATIVEJUMP_OPCODE;
> *(s32 *)(&insn_buff[1]) = rel;
>
> - text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
> - op->optinsn.insn);
> + text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, NULL);
>
> list_del_init(&op->list);
> }
> @@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
> void arch_unoptimize_kprobe(struct optimized_kprobe *op)
> {
> u8 insn_buff[RELATIVEJUMP_SIZE];
> + u8 emulate_buff[RELATIVEJUMP_SIZE];
>
> /* Set int3 to first byte for kprobes */
> insn_buff[0] = BREAKPOINT_INSTRUCTION;
> memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
> +
> + emulate_buff[0] = RELATIVEJUMP_OPCODE;
> + *(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
> + ((long)op->kp.addr + RELATIVEJUMP_SIZE));
> +
> text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
> - op->optinsn.insn);
> + emulate_buff);
> }
>
> /*
>
>
--
Masami Hiramatsu <mhiramat@kernel.org>
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions
2019-08-27 18:06 Peter Zijlstra
@ 2019-08-27 18:06 ` Peter Zijlstra
2019-10-03 5:00 ` Masami Hiramatsu
0 siblings, 1 reply; 15+ messages in thread
From: Peter Zijlstra @ 2019-08-27 18:06 UTC (permalink / raw)
To: linux-kernel, x86, peterz
Cc: Nadav Amit, Andy Lutomirski, Dave Hansen, Song Liu,
Masami Hiramatsu, Steven Rostedt, Daniel Bristot de Oliveira
In preparation for static_call and variable size jump_label support,
teach text_poke_bp() to emulate instructions, namely:
JMP32, JMP8, CALL, NOP2, NOP_ATOMIC5
The current text_poke_bp() takes a @handler argument which is used as
a jump target when the temporary INT3 is hit by a different CPU.
When patching CALL instructions, this doesn't work because we'd miss
the PUSH of the return address. Instead, teach poke_int3_handler() to
emulate an instruction, typically the instruction we're patching in.
This fits almost all text_poke_bp() users, except
arch_unoptimize_kprobe() which restores random text, and for that site
we have to build an explicit emulate instruction.
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Daniel Bristot de Oliveira <bristot@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/include/asm/text-patching.h | 24 ++++++--
arch/x86/kernel/alternative.c | 98 ++++++++++++++++++++++++++---------
arch/x86/kernel/jump_label.c | 9 +--
arch/x86/kernel/kprobes/opt.c | 11 ++-
4 files changed, 103 insertions(+), 39 deletions(-)
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -26,10 +26,11 @@ static inline void apply_paravirt(struct
#define POKE_MAX_OPCODE_SIZE 5
struct text_poke_loc {
- void *detour;
void *addr;
- size_t len;
- const char opcode[POKE_MAX_OPCODE_SIZE];
+ int len;
+ s32 rel32;
+ u8 opcode;
+ const char text[POKE_MAX_OPCODE_SIZE];
};
extern void text_poke_early(void *addr, const void *opcode, size_t len);
@@ -51,8 +52,10 @@ extern void text_poke_early(void *addr,
extern void *text_poke(void *addr, const void *opcode, size_t len);
extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
+extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
extern void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries);
+extern void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
+ const void *opcode, size_t len, const void *emulate);
extern int after_bootmem;
extern __ro_after_init struct mm_struct *poking_mm;
extern __ro_after_init unsigned long poking_addr;
@@ -63,8 +66,17 @@ static inline void int3_emulate_jmp(stru
regs->ip = ip;
}
-#define INT3_INSN_SIZE 1
-#define CALL_INSN_SIZE 5
+#define INT3_INSN_SIZE 1
+#define INT3_INSN_OPCODE 0xCC
+
+#define CALL_INSN_SIZE 5
+#define CALL_INSN_OPCODE 0xE8
+
+#define JMP32_INSN_SIZE 5
+#define JMP32_INSN_OPCODE 0xE9
+
+#define JMP8_INSN_SIZE 2
+#define JMP8_INSN_OPCODE 0xEB
static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
{
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -956,16 +956,15 @@ NOKPROBE_SYMBOL(patch_cmp);
int poke_int3_handler(struct pt_regs *regs)
{
struct text_poke_loc *tp;
- unsigned char int3 = 0xcc;
void *ip;
/*
* Having observed our INT3 instruction, we now must observe
* bp_patching.nr_entries.
*
- * nr_entries != 0 INT3
- * WMB RMB
- * write INT3 if (nr_entries)
+ * nr_entries != 0 INT3
+ * WMB RMB
+ * write INT3 if (nr_entries)
*
* Idem for other elements in bp_patching.
*/
@@ -978,9 +977,9 @@ int poke_int3_handler(struct pt_regs *re
return 0;
/*
- * Discount the sizeof(int3). See text_poke_bp_batch().
+ * Discount the INT3. See text_poke_bp_batch().
*/
- ip = (void *) regs->ip - sizeof(int3);
+ ip = (void *) regs->ip - INT3_INSN_SIZE;
/*
* Skip the binary search if there is a single member in the vector.
@@ -997,8 +996,22 @@ int poke_int3_handler(struct pt_regs *re
return 0;
}
- /* set up the specified breakpoint detour */
- regs->ip = (unsigned long) tp->detour;
+ ip += tp->len;
+
+ switch (tp->opcode) {
+ case CALL_INSN_OPCODE:
+ int3_emulate_call(regs, (long)ip + tp->rel32);
+ break;
+
+ case JMP32_INSN_OPCODE:
+ case JMP8_INSN_OPCODE:
+ int3_emulate_jmp(regs, (long)ip + tp->rel32);
+ break;
+
+ default: /* nop */
+ int3_emulate_jmp(regs, (long)ip);
+ break;
+ }
return 1;
}
@@ -1027,8 +1040,8 @@ NOKPROBE_SYMBOL(poke_int3_handler);
*/
void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries)
{
+ unsigned char int3 = INT3_INSN_OPCODE;
int patched_all_but_first = 0;
- unsigned char int3 = 0xcc;
unsigned int i;
lockdep_assert_held(&text_mutex);
@@ -1056,7 +1069,7 @@ void text_poke_bp_batch(struct text_poke
for (i = 0; i < nr_entries; i++) {
if (tp[i].len - sizeof(int3) > 0) {
text_poke((char *)tp[i].addr + sizeof(int3),
- (const char *)tp[i].opcode + sizeof(int3),
+ (const char *)tp[i].text + sizeof(int3),
tp[i].len - sizeof(int3));
patched_all_but_first++;
}
@@ -1076,7 +1089,7 @@ void text_poke_bp_batch(struct text_poke
* replacing opcode.
*/
for (i = 0; i < nr_entries; i++)
- text_poke(tp[i].addr, tp[i].opcode, sizeof(int3));
+ text_poke(tp[i].addr, tp[i].text, sizeof(int3));
on_each_cpu(do_sync_core, NULL, 1);
/*
@@ -1087,6 +1100,53 @@ void text_poke_bp_batch(struct text_poke
bp_patching.nr_entries = 0;
}
+void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
+ const void *opcode, size_t len, const void *emulate)
+{
+ struct insn insn;
+
+ if (!opcode)
+ opcode = (void *)tp->text;
+ else
+ memcpy((void *)tp->text, opcode, len);
+
+ if (!emulate)
+ emulate = opcode;
+
+ kernel_insn_init(&insn, emulate, MAX_INSN_SIZE);
+ insn_get_length(&insn);
+
+ BUG_ON(!insn_complete(&insn));
+ BUG_ON(len != insn.length);
+
+ tp->addr = addr;
+ tp->len = len;
+ tp->opcode = insn.opcode.bytes[0];
+
+ switch (tp->opcode) {
+ case CALL_INSN_OPCODE:
+ case JMP32_INSN_OPCODE:
+ case JMP8_INSN_OPCODE:
+ tp->rel32 = insn.immediate.value;
+ break;
+
+ default: /* assume NOP */
+ switch (len) {
+ case 2:
+ BUG_ON(memcmp(emulate, ideal_nops[len], len));
+ break;
+
+ case 5:
+ BUG_ON(memcmp(emulate, ideal_nops[NOP_ATOMIC5], len));
+ break;
+
+ default:
+ BUG();
+ }
+ break;
+ }
+}
+
/**
* text_poke_bp() -- update instructions on live kernel on SMP
* @addr: address to patch
@@ -1098,20 +1158,10 @@ void text_poke_bp_batch(struct text_poke
* dynamically allocated memory. This function should be used when it is
* not possible to allocate memory.
*/
-void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
+void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
{
- struct text_poke_loc tp = {
- .detour = handler,
- .addr = addr,
- .len = len,
- };
-
- if (len > POKE_MAX_OPCODE_SIZE) {
- WARN_ONCE(1, "len is larger than %d\n", POKE_MAX_OPCODE_SIZE);
- return;
- }
-
- memcpy((void *)tp.opcode, opcode, len);
+ struct text_poke_loc tp;
+ text_poke_loc_init(&tp, addr, opcode, len, emulate);
text_poke_bp_batch(&tp, 1);
}
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -89,8 +89,7 @@ static void __ref __jump_label_transform
return;
}
- text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE,
- (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
+ text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE, NULL);
}
void arch_jump_label_transform(struct jump_entry *entry,
@@ -147,11 +146,9 @@ bool arch_jump_label_transform_queue(str
}
__jump_label_set_jump_code(entry, type,
- (union jump_code_union *) &tp->opcode, 0);
+ (union jump_code_union *)&tp->text, 0);
- tp->addr = entry_code;
- tp->detour = entry_code + JUMP_LABEL_NOP_SIZE;
- tp->len = JUMP_LABEL_NOP_SIZE;
+ text_poke_loc_init(tp, entry_code, NULL, JUMP_LABEL_NOP_SIZE, NULL);
tp_vec_nr++;
--- a/arch/x86/kernel/kprobes/opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h
insn_buff[0] = RELATIVEJUMP_OPCODE;
*(s32 *)(&insn_buff[1]) = rel;
- text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
- op->optinsn.insn);
+ text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, NULL);
list_del_init(&op->list);
}
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
void arch_unoptimize_kprobe(struct optimized_kprobe *op)
{
u8 insn_buff[RELATIVEJUMP_SIZE];
+ u8 emulate_buff[RELATIVEJUMP_SIZE];
/* Set int3 to first byte for kprobes */
insn_buff[0] = BREAKPOINT_INSTRUCTION;
memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
+
+ emulate_buff[0] = RELATIVEJUMP_OPCODE;
+ *(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
+ ((long)op->kp.addr + RELATIVEJUMP_SIZE));
+
text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
- op->optinsn.insn);
+ emulate_buff);
}
/*
^ permalink raw reply [flat|nested] 15+ messages in thread
end of thread, other threads:[~2019-10-07 8:05 UTC | newest]
Thread overview: 15+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-08-26 12:51 [PATCH 0/3] Rewrite x86/ftrace to use text_poke() Peter Zijlstra
2019-08-26 12:51 ` [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions Peter Zijlstra
2019-08-26 12:51 ` [PATCH 2/3] x86/alternatives: Move tp_vec Peter Zijlstra
2019-08-26 14:36 ` Peter Zijlstra
2019-08-26 12:51 ` [RFC][PATCH 3/3] x86/ftrace: Use text_poke() Peter Zijlstra
2019-08-26 15:13 ` Peter Zijlstra
2019-08-26 13:04 ` [PATCH 0/3] Rewrite x86/ftrace to use text_poke() Peter Zijlstra
2019-08-27 18:06 Peter Zijlstra
2019-08-27 18:06 ` [PATCH 1/3] x86/alternatives: Teach text_poke_bp() to emulate instructions Peter Zijlstra
2019-10-03 5:00 ` Masami Hiramatsu
2019-10-03 8:27 ` Peter Zijlstra
2019-10-03 11:01 ` Peter Zijlstra
2019-10-03 12:32 ` Peter Zijlstra
2019-10-04 13:45 ` Masami Hiramatsu
2019-10-07 8:05 ` Peter Zijlstra
2019-10-03 13:05 ` Peter Zijlstra
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®