Merge branch 'arm64/ftrace-with-regs' of git://git.kernel.org/pub/scm/linux/kernel/git/mark/linux into for-next/core
FTRACE_WITH_REGS support for arm64. * 'arm64/ftrace-with-regs' of git://git.kernel.org/pub/scm/linux/kernel/git/mark/linux: arm64: ftrace: minimize ifdeffery arm64: implement ftrace with regs arm64: asm-offsets: add S_FP arm64: insn: add encoder for MOV (register) arm64: module/ftrace: intialize PLT at load time arm64: module: rework special section handling module/ftrace: handle patchable-function-entry ftrace: add ftrace_init_nop()
This commit is contained in:
commit
c1c9ea6371
|
@ -143,6 +143,8 @@ config ARM64
|
||||||
select HAVE_DEBUG_KMEMLEAK
|
select HAVE_DEBUG_KMEMLEAK
|
||||||
select HAVE_DMA_CONTIGUOUS
|
select HAVE_DMA_CONTIGUOUS
|
||||||
select HAVE_DYNAMIC_FTRACE
|
select HAVE_DYNAMIC_FTRACE
|
||||||
|
select HAVE_DYNAMIC_FTRACE_WITH_REGS \
|
||||||
|
if $(cc-option,-fpatchable-function-entry=2)
|
||||||
select HAVE_EFFICIENT_UNALIGNED_ACCESS
|
select HAVE_EFFICIENT_UNALIGNED_ACCESS
|
||||||
select HAVE_FAST_GUP
|
select HAVE_FAST_GUP
|
||||||
select HAVE_FTRACE_MCOUNT_RECORD
|
select HAVE_FTRACE_MCOUNT_RECORD
|
||||||
|
|
|
@ -95,6 +95,11 @@ ifeq ($(CONFIG_ARM64_MODULE_PLTS),y)
|
||||||
KBUILD_LDS_MODULE += $(srctree)/arch/arm64/kernel/module.lds
|
KBUILD_LDS_MODULE += $(srctree)/arch/arm64/kernel/module.lds
|
||||||
endif
|
endif
|
||||||
|
|
||||||
|
ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_REGS),y)
|
||||||
|
KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY
|
||||||
|
CC_FLAGS_FTRACE := -fpatchable-function-entry=2
|
||||||
|
endif
|
||||||
|
|
||||||
# Default value
|
# Default value
|
||||||
head-y := arch/arm64/kernel/head.o
|
head-y := arch/arm64/kernel/head.o
|
||||||
|
|
||||||
|
|
|
@ -11,9 +11,20 @@
|
||||||
#include <asm/insn.h>
|
#include <asm/insn.h>
|
||||||
|
|
||||||
#define HAVE_FUNCTION_GRAPH_FP_TEST
|
#define HAVE_FUNCTION_GRAPH_FP_TEST
|
||||||
|
|
||||||
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
|
||||||
|
#define ARCH_SUPPORTS_FTRACE_OPS 1
|
||||||
|
#else
|
||||||
#define MCOUNT_ADDR ((unsigned long)_mcount)
|
#define MCOUNT_ADDR ((unsigned long)_mcount)
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* The BL at the callsite's adjusted rec->ip */
|
||||||
#define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE
|
#define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE
|
||||||
|
|
||||||
|
#define FTRACE_PLT_IDX 0
|
||||||
|
#define FTRACE_REGS_PLT_IDX 1
|
||||||
|
#define NR_FTRACE_PLTS 2
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Currently, gcc tends to save the link register after the local variables
|
* Currently, gcc tends to save the link register after the local variables
|
||||||
* on the stack. This causes the max stack tracer to report the function
|
* on the stack. This causes the max stack tracer to report the function
|
||||||
|
@ -43,6 +54,12 @@ extern void return_to_handler(void);
|
||||||
|
|
||||||
static inline unsigned long ftrace_call_adjust(unsigned long addr)
|
static inline unsigned long ftrace_call_adjust(unsigned long addr)
|
||||||
{
|
{
|
||||||
|
/*
|
||||||
|
* Adjust addr to point at the BL in the callsite.
|
||||||
|
* See ftrace_init_nop() for the callsite sequence.
|
||||||
|
*/
|
||||||
|
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
|
||||||
|
return addr + AARCH64_INSN_SIZE;
|
||||||
/*
|
/*
|
||||||
* addr is the address of the mcount call instruction.
|
* addr is the address of the mcount call instruction.
|
||||||
* recordmcount does the necessary offset calculation.
|
* recordmcount does the necessary offset calculation.
|
||||||
|
@ -50,6 +67,12 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
|
||||||
return addr;
|
return addr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
|
||||||
|
struct dyn_ftrace;
|
||||||
|
int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec);
|
||||||
|
#define ftrace_init_nop ftrace_init_nop
|
||||||
|
#endif
|
||||||
|
|
||||||
#define ftrace_return_address(n) return_address(n)
|
#define ftrace_return_address(n) return_address(n)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -440,6 +440,9 @@ u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,
|
||||||
int shift,
|
int shift,
|
||||||
enum aarch64_insn_variant variant,
|
enum aarch64_insn_variant variant,
|
||||||
enum aarch64_insn_logic_type type);
|
enum aarch64_insn_logic_type type);
|
||||||
|
u32 aarch64_insn_gen_move_reg(enum aarch64_insn_register dst,
|
||||||
|
enum aarch64_insn_register src,
|
||||||
|
enum aarch64_insn_variant variant);
|
||||||
u32 aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type,
|
u32 aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type,
|
||||||
enum aarch64_insn_variant variant,
|
enum aarch64_insn_variant variant,
|
||||||
enum aarch64_insn_register Rn,
|
enum aarch64_insn_register Rn,
|
||||||
|
|
|
@ -21,7 +21,7 @@ struct mod_arch_specific {
|
||||||
struct mod_plt_sec init;
|
struct mod_plt_sec init;
|
||||||
|
|
||||||
/* for CONFIG_DYNAMIC_FTRACE */
|
/* for CONFIG_DYNAMIC_FTRACE */
|
||||||
struct plt_entry *ftrace_trampoline;
|
struct plt_entry *ftrace_trampolines;
|
||||||
};
|
};
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
|
@ -56,6 +56,7 @@ int main(void)
|
||||||
DEFINE(S_X24, offsetof(struct pt_regs, regs[24]));
|
DEFINE(S_X24, offsetof(struct pt_regs, regs[24]));
|
||||||
DEFINE(S_X26, offsetof(struct pt_regs, regs[26]));
|
DEFINE(S_X26, offsetof(struct pt_regs, regs[26]));
|
||||||
DEFINE(S_X28, offsetof(struct pt_regs, regs[28]));
|
DEFINE(S_X28, offsetof(struct pt_regs, regs[28]));
|
||||||
|
DEFINE(S_FP, offsetof(struct pt_regs, regs[29]));
|
||||||
DEFINE(S_LR, offsetof(struct pt_regs, regs[30]));
|
DEFINE(S_LR, offsetof(struct pt_regs, regs[30]));
|
||||||
DEFINE(S_SP, offsetof(struct pt_regs, sp));
|
DEFINE(S_SP, offsetof(struct pt_regs, sp));
|
||||||
DEFINE(S_PSTATE, offsetof(struct pt_regs, pstate));
|
DEFINE(S_PSTATE, offsetof(struct pt_regs, pstate));
|
||||||
|
|
|
@ -7,10 +7,137 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <linux/linkage.h>
|
#include <linux/linkage.h>
|
||||||
|
#include <asm/asm-offsets.h>
|
||||||
#include <asm/assembler.h>
|
#include <asm/assembler.h>
|
||||||
#include <asm/ftrace.h>
|
#include <asm/ftrace.h>
|
||||||
#include <asm/insn.h>
|
#include <asm/insn.h>
|
||||||
|
|
||||||
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
|
||||||
|
/*
|
||||||
|
* Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
|
||||||
|
* the regular function prologue. For an enabled callsite, ftrace_init_nop() and
|
||||||
|
* ftrace_make_call() have patched those NOPs to:
|
||||||
|
*
|
||||||
|
* MOV X9, LR
|
||||||
|
* BL <entry>
|
||||||
|
*
|
||||||
|
* ... where <entry> is either ftrace_caller or ftrace_regs_caller.
|
||||||
|
*
|
||||||
|
* Each instrumented function follows the AAPCS, so here x0-x8 and x19-x30 are
|
||||||
|
* live, and x9-x18 are safe to clobber.
|
||||||
|
*
|
||||||
|
* We save the callsite's context into a pt_regs before invoking any ftrace
|
||||||
|
* callbacks. So that we can get a sensible backtrace, we create a stack record
|
||||||
|
* for the callsite and the ftrace entry assembly. This is not sufficient for
|
||||||
|
* reliable stacktrace: until we create the callsite stack record, its caller
|
||||||
|
* is missing from the LR and existing chain of frame records.
|
||||||
|
*/
|
||||||
|
.macro ftrace_regs_entry, allregs=0
|
||||||
|
/* Make room for pt_regs, plus a callee frame */
|
||||||
|
sub sp, sp, #(S_FRAME_SIZE + 16)
|
||||||
|
|
||||||
|
/* Save function arguments (and x9 for simplicity) */
|
||||||
|
stp x0, x1, [sp, #S_X0]
|
||||||
|
stp x2, x3, [sp, #S_X2]
|
||||||
|
stp x4, x5, [sp, #S_X4]
|
||||||
|
stp x6, x7, [sp, #S_X6]
|
||||||
|
stp x8, x9, [sp, #S_X8]
|
||||||
|
|
||||||
|
/* Optionally save the callee-saved registers, always save the FP */
|
||||||
|
.if \allregs == 1
|
||||||
|
stp x10, x11, [sp, #S_X10]
|
||||||
|
stp x12, x13, [sp, #S_X12]
|
||||||
|
stp x14, x15, [sp, #S_X14]
|
||||||
|
stp x16, x17, [sp, #S_X16]
|
||||||
|
stp x18, x19, [sp, #S_X18]
|
||||||
|
stp x20, x21, [sp, #S_X20]
|
||||||
|
stp x22, x23, [sp, #S_X22]
|
||||||
|
stp x24, x25, [sp, #S_X24]
|
||||||
|
stp x26, x27, [sp, #S_X26]
|
||||||
|
stp x28, x29, [sp, #S_X28]
|
||||||
|
.else
|
||||||
|
str x29, [sp, #S_FP]
|
||||||
|
.endif
|
||||||
|
|
||||||
|
/* Save the callsite's SP and LR */
|
||||||
|
add x10, sp, #(S_FRAME_SIZE + 16)
|
||||||
|
stp x9, x10, [sp, #S_LR]
|
||||||
|
|
||||||
|
/* Save the PC after the ftrace callsite */
|
||||||
|
str x30, [sp, #S_PC]
|
||||||
|
|
||||||
|
/* Create a frame record for the callsite above pt_regs */
|
||||||
|
stp x29, x9, [sp, #S_FRAME_SIZE]
|
||||||
|
add x29, sp, #S_FRAME_SIZE
|
||||||
|
|
||||||
|
/* Create our frame record within pt_regs. */
|
||||||
|
stp x29, x30, [sp, #S_STACKFRAME]
|
||||||
|
add x29, sp, #S_STACKFRAME
|
||||||
|
.endm
|
||||||
|
|
||||||
|
ENTRY(ftrace_regs_caller)
|
||||||
|
ftrace_regs_entry 1
|
||||||
|
b ftrace_common
|
||||||
|
ENDPROC(ftrace_regs_caller)
|
||||||
|
|
||||||
|
ENTRY(ftrace_caller)
|
||||||
|
ftrace_regs_entry 0
|
||||||
|
b ftrace_common
|
||||||
|
ENDPROC(ftrace_caller)
|
||||||
|
|
||||||
|
ENTRY(ftrace_common)
|
||||||
|
sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
|
||||||
|
mov x1, x9 // parent_ip (callsite's LR)
|
||||||
|
ldr_l x2, function_trace_op // op
|
||||||
|
mov x3, sp // regs
|
||||||
|
|
||||||
|
GLOBAL(ftrace_call)
|
||||||
|
bl ftrace_stub
|
||||||
|
|
||||||
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
||||||
|
GLOBAL(ftrace_graph_call) // ftrace_graph_caller();
|
||||||
|
nop // If enabled, this will be replaced
|
||||||
|
// "b ftrace_graph_caller"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/*
|
||||||
|
* At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
|
||||||
|
* x19-x29 per the AAPCS, and we created frame records upon entry, so we need
|
||||||
|
* to restore x0-x8, x29, and x30.
|
||||||
|
*/
|
||||||
|
ftrace_common_return:
|
||||||
|
/* Restore function arguments */
|
||||||
|
ldp x0, x1, [sp]
|
||||||
|
ldp x2, x3, [sp, #S_X2]
|
||||||
|
ldp x4, x5, [sp, #S_X4]
|
||||||
|
ldp x6, x7, [sp, #S_X6]
|
||||||
|
ldr x8, [sp, #S_X8]
|
||||||
|
|
||||||
|
/* Restore the callsite's FP, LR, PC */
|
||||||
|
ldr x29, [sp, #S_FP]
|
||||||
|
ldr x30, [sp, #S_LR]
|
||||||
|
ldr x9, [sp, #S_PC]
|
||||||
|
|
||||||
|
/* Restore the callsite's SP */
|
||||||
|
add sp, sp, #S_FRAME_SIZE + 16
|
||||||
|
|
||||||
|
ret x9
|
||||||
|
ENDPROC(ftrace_common)
|
||||||
|
|
||||||
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
||||||
|
ENTRY(ftrace_graph_caller)
|
||||||
|
ldr x0, [sp, #S_PC]
|
||||||
|
sub x0, x0, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
|
||||||
|
add x1, sp, #S_LR // parent_ip (callsite's LR)
|
||||||
|
ldr x2, [sp, #S_FRAME_SIZE] // parent fp (callsite's FP)
|
||||||
|
bl prepare_ftrace_return
|
||||||
|
b ftrace_common_return
|
||||||
|
ENDPROC(ftrace_graph_caller)
|
||||||
|
#else
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Gcc with -pg will put the following code in the beginning of each function:
|
* Gcc with -pg will put the following code in the beginning of each function:
|
||||||
* mov x0, x30
|
* mov x0, x30
|
||||||
|
@ -160,11 +287,6 @@ GLOBAL(ftrace_graph_call) // ftrace_graph_caller();
|
||||||
|
|
||||||
mcount_exit
|
mcount_exit
|
||||||
ENDPROC(ftrace_caller)
|
ENDPROC(ftrace_caller)
|
||||||
#endif /* CONFIG_DYNAMIC_FTRACE */
|
|
||||||
|
|
||||||
ENTRY(ftrace_stub)
|
|
||||||
ret
|
|
||||||
ENDPROC(ftrace_stub)
|
|
||||||
|
|
||||||
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
||||||
/*
|
/*
|
||||||
|
@ -184,7 +306,15 @@ ENTRY(ftrace_graph_caller)
|
||||||
|
|
||||||
mcount_exit
|
mcount_exit
|
||||||
ENDPROC(ftrace_graph_caller)
|
ENDPROC(ftrace_graph_caller)
|
||||||
|
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
|
||||||
|
#endif /* CONFIG_DYNAMIC_FTRACE */
|
||||||
|
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
|
||||||
|
|
||||||
|
ENTRY(ftrace_stub)
|
||||||
|
ret
|
||||||
|
ENDPROC(ftrace_stub)
|
||||||
|
|
||||||
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
||||||
/*
|
/*
|
||||||
* void return_to_handler(void)
|
* void return_to_handler(void)
|
||||||
*
|
*
|
||||||
|
|
|
@ -62,6 +62,19 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
|
||||||
return ftrace_modify_code(pc, 0, new, false);
|
return ftrace_modify_code(pc, 0, new, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_ARM64_MODULE_PLTS
|
||||||
|
struct plt_entry *plt = mod->arch.ftrace_trampolines;
|
||||||
|
|
||||||
|
if (addr == FTRACE_ADDR)
|
||||||
|
return &plt[FTRACE_PLT_IDX];
|
||||||
|
if (addr == FTRACE_REGS_ADDR && IS_ENABLED(CONFIG_FTRACE_WITH_REGS))
|
||||||
|
return &plt[FTRACE_REGS_PLT_IDX];
|
||||||
|
#endif
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Turn on the call to ftrace_caller() in instrumented function
|
* Turn on the call to ftrace_caller() in instrumented function
|
||||||
*/
|
*/
|
||||||
|
@ -72,9 +85,11 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
|
||||||
long offset = (long)pc - (long)addr;
|
long offset = (long)pc - (long)addr;
|
||||||
|
|
||||||
if (offset < -SZ_128M || offset >= SZ_128M) {
|
if (offset < -SZ_128M || offset >= SZ_128M) {
|
||||||
#ifdef CONFIG_ARM64_MODULE_PLTS
|
|
||||||
struct plt_entry trampoline, *dst;
|
|
||||||
struct module *mod;
|
struct module *mod;
|
||||||
|
struct plt_entry *plt;
|
||||||
|
|
||||||
|
if (!IS_ENABLED(CONFIG_ARM64_MODULE_PLTS))
|
||||||
|
return -EINVAL;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* On kernels that support module PLTs, the offset between the
|
* On kernels that support module PLTs, the offset between the
|
||||||
|
@ -93,49 +108,13 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
|
||||||
if (WARN_ON(!mod))
|
if (WARN_ON(!mod))
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
|
||||||
/*
|
plt = get_ftrace_plt(mod, addr);
|
||||||
* There is only one ftrace trampoline per module. For now,
|
if (!plt) {
|
||||||
* this is not a problem since on arm64, all dynamic ftrace
|
pr_err("ftrace: no module PLT for %ps\n", (void *)addr);
|
||||||
* invocations are routed via ftrace_caller(). This will need
|
return -EINVAL;
|
||||||
* to be revisited if support for multiple ftrace entry points
|
|
||||||
* is added in the future, but for now, the pr_err() below
|
|
||||||
* deals with a theoretical issue only.
|
|
||||||
*
|
|
||||||
* Note that PLTs are place relative, and plt_entries_equal()
|
|
||||||
* checks whether they point to the same target. Here, we need
|
|
||||||
* to check if the actual opcodes are in fact identical,
|
|
||||||
* regardless of the offset in memory so use memcmp() instead.
|
|
||||||
*/
|
|
||||||
dst = mod->arch.ftrace_trampoline;
|
|
||||||
trampoline = get_plt_entry(addr, dst);
|
|
||||||
if (memcmp(dst, &trampoline, sizeof(trampoline))) {
|
|
||||||
if (plt_entry_is_initialized(dst)) {
|
|
||||||
pr_err("ftrace: far branches to multiple entry points unsupported inside a single module\n");
|
|
||||||
return -EINVAL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* point the trampoline to our ftrace entry point */
|
|
||||||
module_disable_ro(mod);
|
|
||||||
*dst = trampoline;
|
|
||||||
module_enable_ro(mod, true);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Ensure updated trampoline is visible to instruction
|
|
||||||
* fetch before we patch in the branch. Although the
|
|
||||||
* architecture doesn't require an IPI in this case,
|
|
||||||
* Neoverse-N1 erratum #1542419 does require one
|
|
||||||
* if the TLB maintenance in module_enable_ro() is
|
|
||||||
* skipped due to rodata_enabled. It doesn't seem worth
|
|
||||||
* it to make it conditional given that this is
|
|
||||||
* certainly not a fast-path.
|
|
||||||
*/
|
|
||||||
flush_icache_range((unsigned long)&dst[0],
|
|
||||||
(unsigned long)&dst[1]);
|
|
||||||
}
|
}
|
||||||
addr = (unsigned long)dst;
|
|
||||||
#else /* CONFIG_ARM64_MODULE_PLTS */
|
addr = (unsigned long)plt;
|
||||||
return -EINVAL;
|
|
||||||
#endif /* CONFIG_ARM64_MODULE_PLTS */
|
|
||||||
}
|
}
|
||||||
|
|
||||||
old = aarch64_insn_gen_nop();
|
old = aarch64_insn_gen_nop();
|
||||||
|
@ -144,6 +123,55 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
|
||||||
return ftrace_modify_code(pc, old, new, true);
|
return ftrace_modify_code(pc, old, new, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
|
||||||
|
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
|
||||||
|
unsigned long addr)
|
||||||
|
{
|
||||||
|
unsigned long pc = rec->ip;
|
||||||
|
u32 old, new;
|
||||||
|
|
||||||
|
old = aarch64_insn_gen_branch_imm(pc, old_addr,
|
||||||
|
AARCH64_INSN_BRANCH_LINK);
|
||||||
|
new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
|
||||||
|
|
||||||
|
return ftrace_modify_code(pc, old, new, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The compiler has inserted two NOPs before the regular function prologue.
|
||||||
|
* All instrumented functions follow the AAPCS, so x0-x8 and x19-x30 are live,
|
||||||
|
* and x9-x18 are free for our use.
|
||||||
|
*
|
||||||
|
* At runtime we want to be able to swing a single NOP <-> BL to enable or
|
||||||
|
* disable the ftrace call. The BL requires us to save the original LR value,
|
||||||
|
* so here we insert a <MOV X9, LR> over the first NOP so the instructions
|
||||||
|
* before the regular prologue are:
|
||||||
|
*
|
||||||
|
* | Compiled | Disabled | Enabled |
|
||||||
|
* +----------+------------+------------+
|
||||||
|
* | NOP | MOV X9, LR | MOV X9, LR |
|
||||||
|
* | NOP | NOP | BL <entry> |
|
||||||
|
*
|
||||||
|
* The LR value will be recovered by ftrace_regs_entry, and restored into LR
|
||||||
|
* before returning to the regular function prologue. When a function is not
|
||||||
|
* being traced, the MOV is not harmful given x9 is not live per the AAPCS.
|
||||||
|
*
|
||||||
|
* Note: ftrace_process_locs() has pre-adjusted rec->ip to be the address of
|
||||||
|
* the BL.
|
||||||
|
*/
|
||||||
|
int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
|
||||||
|
{
|
||||||
|
unsigned long pc = rec->ip - AARCH64_INSN_SIZE;
|
||||||
|
u32 old, new;
|
||||||
|
|
||||||
|
old = aarch64_insn_gen_nop();
|
||||||
|
new = aarch64_insn_gen_move_reg(AARCH64_INSN_REG_9,
|
||||||
|
AARCH64_INSN_REG_LR,
|
||||||
|
AARCH64_INSN_VARIANT_64BIT);
|
||||||
|
return ftrace_modify_code(pc, old, new, true);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Turn off the call to ftrace_caller() in instrumented function
|
* Turn off the call to ftrace_caller() in instrumented function
|
||||||
*/
|
*/
|
||||||
|
@ -156,9 +184,11 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
|
||||||
long offset = (long)pc - (long)addr;
|
long offset = (long)pc - (long)addr;
|
||||||
|
|
||||||
if (offset < -SZ_128M || offset >= SZ_128M) {
|
if (offset < -SZ_128M || offset >= SZ_128M) {
|
||||||
#ifdef CONFIG_ARM64_MODULE_PLTS
|
|
||||||
u32 replaced;
|
u32 replaced;
|
||||||
|
|
||||||
|
if (!IS_ENABLED(CONFIG_ARM64_MODULE_PLTS))
|
||||||
|
return -EINVAL;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* 'mod' is only set at module load time, but if we end up
|
* 'mod' is only set at module load time, but if we end up
|
||||||
* dealing with an out-of-range condition, we can assume it
|
* dealing with an out-of-range condition, we can assume it
|
||||||
|
@ -189,9 +219,6 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
|
||||||
validate = false;
|
validate = false;
|
||||||
#else /* CONFIG_ARM64_MODULE_PLTS */
|
|
||||||
return -EINVAL;
|
|
||||||
#endif /* CONFIG_ARM64_MODULE_PLTS */
|
|
||||||
} else {
|
} else {
|
||||||
old = aarch64_insn_gen_branch_imm(pc, addr,
|
old = aarch64_insn_gen_branch_imm(pc, addr,
|
||||||
AARCH64_INSN_BRANCH_LINK);
|
AARCH64_INSN_BRANCH_LINK);
|
||||||
|
|
|
@ -1268,6 +1268,19 @@ u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,
|
||||||
return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
|
return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* MOV (register) is architecturally an alias of ORR (shifted register) where
|
||||||
|
* MOV <*d>, <*m> is equivalent to ORR <*d>, <*ZR>, <*m>
|
||||||
|
*/
|
||||||
|
u32 aarch64_insn_gen_move_reg(enum aarch64_insn_register dst,
|
||||||
|
enum aarch64_insn_register src,
|
||||||
|
enum aarch64_insn_variant variant)
|
||||||
|
{
|
||||||
|
return aarch64_insn_gen_logical_shifted_reg(dst, AARCH64_INSN_REG_ZR,
|
||||||
|
src, 0, variant,
|
||||||
|
AARCH64_INSN_LOGIC_ORR);
|
||||||
|
}
|
||||||
|
|
||||||
u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr,
|
u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr,
|
||||||
enum aarch64_insn_register reg,
|
enum aarch64_insn_register reg,
|
||||||
enum aarch64_insn_adr_type type)
|
enum aarch64_insn_adr_type type)
|
||||||
|
|
|
@ -4,6 +4,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <linux/elf.h>
|
#include <linux/elf.h>
|
||||||
|
#include <linux/ftrace.h>
|
||||||
#include <linux/kernel.h>
|
#include <linux/kernel.h>
|
||||||
#include <linux/module.h>
|
#include <linux/module.h>
|
||||||
#include <linux/sort.h>
|
#include <linux/sort.h>
|
||||||
|
@ -330,7 +331,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
|
||||||
tramp->sh_type = SHT_NOBITS;
|
tramp->sh_type = SHT_NOBITS;
|
||||||
tramp->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
|
tramp->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
|
||||||
tramp->sh_addralign = __alignof__(struct plt_entry);
|
tramp->sh_addralign = __alignof__(struct plt_entry);
|
||||||
tramp->sh_size = sizeof(struct plt_entry);
|
tramp->sh_size = NR_FTRACE_PLTS * sizeof(struct plt_entry);
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
|
@ -9,6 +9,7 @@
|
||||||
|
|
||||||
#include <linux/bitops.h>
|
#include <linux/bitops.h>
|
||||||
#include <linux/elf.h>
|
#include <linux/elf.h>
|
||||||
|
#include <linux/ftrace.h>
|
||||||
#include <linux/gfp.h>
|
#include <linux/gfp.h>
|
||||||
#include <linux/kasan.h>
|
#include <linux/kasan.h>
|
||||||
#include <linux/kernel.h>
|
#include <linux/kernel.h>
|
||||||
|
@ -470,22 +471,58 @@ overflow:
|
||||||
return -ENOEXEC;
|
return -ENOEXEC;
|
||||||
}
|
}
|
||||||
|
|
||||||
int module_finalize(const Elf_Ehdr *hdr,
|
static const Elf_Shdr *find_section(const Elf_Ehdr *hdr,
|
||||||
const Elf_Shdr *sechdrs,
|
const Elf_Shdr *sechdrs,
|
||||||
struct module *me)
|
const char *name)
|
||||||
{
|
{
|
||||||
const Elf_Shdr *s, *se;
|
const Elf_Shdr *s, *se;
|
||||||
const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
|
const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
|
||||||
|
|
||||||
for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
|
for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
|
||||||
if (strcmp(".altinstructions", secstrs + s->sh_name) == 0)
|
if (strcmp(name, secstrs + s->sh_name) == 0)
|
||||||
apply_alternatives_module((void *)s->sh_addr, s->sh_size);
|
return s;
|
||||||
#ifdef CONFIG_ARM64_MODULE_PLTS
|
|
||||||
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE) &&
|
|
||||||
!strcmp(".text.ftrace_trampoline", secstrs + s->sh_name))
|
|
||||||
me->arch.ftrace_trampoline = (void *)s->sh_addr;
|
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void __init_plt(struct plt_entry *plt, unsigned long addr)
|
||||||
|
{
|
||||||
|
*plt = get_plt_entry(addr, plt);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int module_init_ftrace_plt(const Elf_Ehdr *hdr,
|
||||||
|
const Elf_Shdr *sechdrs,
|
||||||
|
struct module *mod)
|
||||||
|
{
|
||||||
|
#if defined(CONFIG_ARM64_MODULE_PLTS) && defined(CONFIG_DYNAMIC_FTRACE)
|
||||||
|
const Elf_Shdr *s;
|
||||||
|
struct plt_entry *plts;
|
||||||
|
|
||||||
|
s = find_section(hdr, sechdrs, ".text.ftrace_trampoline");
|
||||||
|
if (!s)
|
||||||
|
return -ENOEXEC;
|
||||||
|
|
||||||
|
plts = (void *)s->sh_addr;
|
||||||
|
|
||||||
|
__init_plt(&plts[FTRACE_PLT_IDX], FTRACE_ADDR);
|
||||||
|
|
||||||
|
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
|
||||||
|
__init_plt(&plts[FTRACE_REGS_PLT_IDX], FTRACE_REGS_ADDR);
|
||||||
|
|
||||||
|
mod->arch.ftrace_trampolines = plts;
|
||||||
|
#endif
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int module_finalize(const Elf_Ehdr *hdr,
|
||||||
|
const Elf_Shdr *sechdrs,
|
||||||
|
struct module *me)
|
||||||
|
{
|
||||||
|
const Elf_Shdr *s;
|
||||||
|
s = find_section(hdr, sechdrs, ".altinstructions");
|
||||||
|
if (s)
|
||||||
|
apply_alternatives_module((void *)s->sh_addr, s->sh_size);
|
||||||
|
|
||||||
|
return module_init_ftrace_plt(hdr, sechdrs, me);
|
||||||
|
}
|
||||||
|
|
|
@ -60,7 +60,6 @@ KBUILD_CFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY=1 \
|
||||||
-DFTRACE_PATCHABLE_FUNCTION_SIZE=$(NOP_COUNT)
|
-DFTRACE_PATCHABLE_FUNCTION_SIZE=$(NOP_COUNT)
|
||||||
|
|
||||||
CC_FLAGS_FTRACE := -fpatchable-function-entry=$(NOP_COUNT),$(shell echo $$(($(NOP_COUNT)-1)))
|
CC_FLAGS_FTRACE := -fpatchable-function-entry=$(NOP_COUNT),$(shell echo $$(($(NOP_COUNT)-1)))
|
||||||
KBUILD_LDS_MODULE += $(srctree)/arch/parisc/kernel/module.lds
|
|
||||||
endif
|
endif
|
||||||
|
|
||||||
OBJCOPY_FLAGS =-O binary -R .note -R .comment -S
|
OBJCOPY_FLAGS =-O binary -R .note -R .comment -S
|
||||||
|
|
|
@ -43,6 +43,7 @@
|
||||||
#include <linux/elf.h>
|
#include <linux/elf.h>
|
||||||
#include <linux/vmalloc.h>
|
#include <linux/vmalloc.h>
|
||||||
#include <linux/fs.h>
|
#include <linux/fs.h>
|
||||||
|
#include <linux/ftrace.h>
|
||||||
#include <linux/string.h>
|
#include <linux/string.h>
|
||||||
#include <linux/kernel.h>
|
#include <linux/kernel.h>
|
||||||
#include <linux/bug.h>
|
#include <linux/bug.h>
|
||||||
|
@ -862,7 +863,7 @@ int module_finalize(const Elf_Ehdr *hdr,
|
||||||
const char *strtab = NULL;
|
const char *strtab = NULL;
|
||||||
const Elf_Shdr *s;
|
const Elf_Shdr *s;
|
||||||
char *secstrings;
|
char *secstrings;
|
||||||
int err, symindex = -1;
|
int symindex = -1;
|
||||||
Elf_Sym *newptr, *oldptr;
|
Elf_Sym *newptr, *oldptr;
|
||||||
Elf_Shdr *symhdr = NULL;
|
Elf_Shdr *symhdr = NULL;
|
||||||
#ifdef DEBUG
|
#ifdef DEBUG
|
||||||
|
@ -946,11 +947,13 @@ int module_finalize(const Elf_Ehdr *hdr,
|
||||||
/* patch .altinstructions */
|
/* patch .altinstructions */
|
||||||
apply_alternatives(aseg, aseg + s->sh_size, me->name);
|
apply_alternatives(aseg, aseg + s->sh_size, me->name);
|
||||||
|
|
||||||
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
||||||
/* For 32 bit kernels we're compiling modules with
|
/* For 32 bit kernels we're compiling modules with
|
||||||
* -ffunction-sections so we must relocate the addresses in the
|
* -ffunction-sections so we must relocate the addresses in the
|
||||||
*__mcount_loc section.
|
* ftrace callsite section.
|
||||||
*/
|
*/
|
||||||
if (symindex != -1 && !strcmp(secname, "__mcount_loc")) {
|
if (symindex != -1 && !strcmp(secname, FTRACE_CALLSITE_SECTION)) {
|
||||||
|
int err;
|
||||||
if (s->sh_type == SHT_REL)
|
if (s->sh_type == SHT_REL)
|
||||||
err = apply_relocate((Elf_Shdr *)sechdrs,
|
err = apply_relocate((Elf_Shdr *)sechdrs,
|
||||||
strtab, symindex,
|
strtab, symindex,
|
||||||
|
@ -962,6 +965,7 @@ int module_finalize(const Elf_Ehdr *hdr,
|
||||||
if (err)
|
if (err)
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +0,0 @@
|
||||||
/* SPDX-License-Identifier: GPL-2.0 */
|
|
||||||
|
|
||||||
SECTIONS {
|
|
||||||
__mcount_loc : {
|
|
||||||
*(__patchable_function_entries)
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -110,17 +110,17 @@
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
|
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
|
||||||
#ifdef CC_USING_PATCHABLE_FUNCTION_ENTRY
|
/*
|
||||||
#define MCOUNT_REC() . = ALIGN(8); \
|
* The ftrace call sites are logged to a section whose name depends on the
|
||||||
__start_mcount_loc = .; \
|
* compiler option used. A given kernel image will only use one, AKA
|
||||||
KEEP(*(__patchable_function_entries)) \
|
* FTRACE_CALLSITE_SECTION. We capture all of them here to avoid header
|
||||||
__stop_mcount_loc = .;
|
* dependencies for FTRACE_CALLSITE_SECTION's definition.
|
||||||
#else
|
*/
|
||||||
#define MCOUNT_REC() . = ALIGN(8); \
|
#define MCOUNT_REC() . = ALIGN(8); \
|
||||||
__start_mcount_loc = .; \
|
__start_mcount_loc = .; \
|
||||||
KEEP(*(__mcount_loc)) \
|
KEEP(*(__mcount_loc)) \
|
||||||
|
KEEP(*(__patchable_function_entries)) \
|
||||||
__stop_mcount_loc = .;
|
__stop_mcount_loc = .;
|
||||||
#endif
|
|
||||||
#else
|
#else
|
||||||
#define MCOUNT_REC()
|
#define MCOUNT_REC()
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -499,7 +499,7 @@ static inline int ftrace_disable_ftrace_graph_caller(void) { return 0; }
|
||||||
/**
|
/**
|
||||||
* ftrace_make_nop - convert code into nop
|
* ftrace_make_nop - convert code into nop
|
||||||
* @mod: module structure if called by module load initialization
|
* @mod: module structure if called by module load initialization
|
||||||
* @rec: the mcount call site record
|
* @rec: the call site record (e.g. mcount/fentry)
|
||||||
* @addr: the address that the call site should be calling
|
* @addr: the address that the call site should be calling
|
||||||
*
|
*
|
||||||
* This is a very sensitive operation and great care needs
|
* This is a very sensitive operation and great care needs
|
||||||
|
@ -520,9 +520,38 @@ static inline int ftrace_disable_ftrace_graph_caller(void) { return 0; }
|
||||||
extern int ftrace_make_nop(struct module *mod,
|
extern int ftrace_make_nop(struct module *mod,
|
||||||
struct dyn_ftrace *rec, unsigned long addr);
|
struct dyn_ftrace *rec, unsigned long addr);
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ftrace_init_nop - initialize a nop call site
|
||||||
|
* @mod: module structure if called by module load initialization
|
||||||
|
* @rec: the call site record (e.g. mcount/fentry)
|
||||||
|
*
|
||||||
|
* This is a very sensitive operation and great care needs
|
||||||
|
* to be taken by the arch. The operation should carefully
|
||||||
|
* read the location, check to see if what is read is indeed
|
||||||
|
* what we expect it to be, and then on success of the compare,
|
||||||
|
* it should write to the location.
|
||||||
|
*
|
||||||
|
* The code segment at @rec->ip should contain the contents created by
|
||||||
|
* the compiler
|
||||||
|
*
|
||||||
|
* Return must be:
|
||||||
|
* 0 on success
|
||||||
|
* -EFAULT on error reading the location
|
||||||
|
* -EINVAL on a failed compare of the contents
|
||||||
|
* -EPERM on error writing to the location
|
||||||
|
* Any other value will be considered a failure.
|
||||||
|
*/
|
||||||
|
#ifndef ftrace_init_nop
|
||||||
|
static inline int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
|
||||||
|
{
|
||||||
|
return ftrace_make_nop(mod, rec, MCOUNT_ADDR);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ftrace_make_call - convert a nop call site into a call to addr
|
* ftrace_make_call - convert a nop call site into a call to addr
|
||||||
* @rec: the mcount call site record
|
* @rec: the call site record (e.g. mcount/fentry)
|
||||||
* @addr: the address that the call site should call
|
* @addr: the address that the call site should call
|
||||||
*
|
*
|
||||||
* This is a very sensitive operation and great care needs
|
* This is a very sensitive operation and great care needs
|
||||||
|
@ -545,7 +574,7 @@ extern int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr);
|
||||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
|
||||||
/**
|
/**
|
||||||
* ftrace_modify_call - convert from one addr to another (no nop)
|
* ftrace_modify_call - convert from one addr to another (no nop)
|
||||||
* @rec: the mcount call site record
|
* @rec: the call site record (e.g. mcount/fentry)
|
||||||
* @old_addr: the address expected to be currently called to
|
* @old_addr: the address expected to be currently called to
|
||||||
* @addr: the address to change to
|
* @addr: the address to change to
|
||||||
*
|
*
|
||||||
|
@ -709,6 +738,11 @@ static inline unsigned long get_lock_parent_ip(void)
|
||||||
|
|
||||||
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
|
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
|
||||||
extern void ftrace_init(void);
|
extern void ftrace_init(void);
|
||||||
|
#ifdef CC_USING_PATCHABLE_FUNCTION_ENTRY
|
||||||
|
#define FTRACE_CALLSITE_SECTION "__patchable_function_entries"
|
||||||
|
#else
|
||||||
|
#define FTRACE_CALLSITE_SECTION "__mcount_loc"
|
||||||
|
#endif
|
||||||
#else
|
#else
|
||||||
static inline void ftrace_init(void) { }
|
static inline void ftrace_init(void) { }
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -3222,7 +3222,7 @@ static int find_module_sections(struct module *mod, struct load_info *info)
|
||||||
#endif
|
#endif
|
||||||
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
|
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
|
||||||
/* sechdrs[0].sh_size is always zero */
|
/* sechdrs[0].sh_size is always zero */
|
||||||
mod->ftrace_callsites = section_objs(info, "__mcount_loc",
|
mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION,
|
||||||
sizeof(*mod->ftrace_callsites),
|
sizeof(*mod->ftrace_callsites),
|
||||||
&mod->num_ftrace_callsites);
|
&mod->num_ftrace_callsites);
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -2494,14 +2494,14 @@ struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
|
||||||
}
|
}
|
||||||
|
|
||||||
static int
|
static int
|
||||||
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
|
ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec)
|
||||||
{
|
{
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
if (unlikely(ftrace_disabled))
|
if (unlikely(ftrace_disabled))
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
|
ret = ftrace_init_nop(mod, rec);
|
||||||
if (ret) {
|
if (ret) {
|
||||||
ftrace_bug_type = FTRACE_BUG_INIT;
|
ftrace_bug_type = FTRACE_BUG_INIT;
|
||||||
ftrace_bug(ret, rec);
|
ftrace_bug(ret, rec);
|
||||||
|
@ -2943,7 +2943,7 @@ static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
|
||||||
* to the NOP instructions.
|
* to the NOP instructions.
|
||||||
*/
|
*/
|
||||||
if (!__is_defined(CC_USING_NOP_MCOUNT) &&
|
if (!__is_defined(CC_USING_NOP_MCOUNT) &&
|
||||||
!ftrace_code_disable(mod, p))
|
!ftrace_nop_initialize(mod, p))
|
||||||
break;
|
break;
|
||||||
|
|
||||||
update_cnt++;
|
update_cnt++;
|
||||||
|
|
Loading…
Reference in a new issue