Merge branch 'for-next/entry-s-to-c' into for-next/core
Move the synchronous exception paths from entry.S into a C file to improve the code readability. * for-next/entry-s-to-c: arm64: entry-common: don't touch daif before bp-hardening arm64: Remove asmlinkage from updated functions arm64: entry: convert el0_sync to C arm64: entry: convert el1_sync to C arm64: add local_daif_inherit() arm64: Add prototypes for functions called by entry.S arm64: remove __exception annotations
This commit is contained in:
commit
8301ae822d
|
@ -74,13 +74,4 @@ alternative_if ARM64_ALT_PAN_NOT_UAO
|
|||
SET_PSTATE_PAN(0)
|
||||
alternative_else_nop_endif
|
||||
.endm
|
||||
|
||||
/*
|
||||
* Remove the address tag from a virtual address, if present.
|
||||
*/
|
||||
.macro untagged_addr, dst, addr
|
||||
sbfx \dst, \addr, #0, #56
|
||||
and \dst, \dst, \addr
|
||||
.endm
|
||||
|
||||
#endif
|
||||
|
|
|
@ -9,6 +9,7 @@
|
|||
|
||||
#include <asm/arch_gicv3.h>
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/ptrace.h>
|
||||
|
||||
#define DAIF_PROCCTX 0
|
||||
#define DAIF_PROCCTX_NOIRQ PSR_I_BIT
|
||||
|
@ -109,4 +110,19 @@ static inline void local_daif_restore(unsigned long flags)
|
|||
trace_hardirqs_off();
|
||||
}
|
||||
|
||||
/*
|
||||
* Called by synchronous exception handlers to restore the DAIF bits that were
|
||||
* modified by taking an exception.
|
||||
*/
|
||||
static inline void local_daif_inherit(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long flags = regs->pstate & DAIF_MASK;
|
||||
|
||||
/*
|
||||
* We can't use local_daif_restore(regs->pstate) here as
|
||||
* system_has_prio_mask_debugging() won't restore the I bit if it can
|
||||
* use the pmr instead.
|
||||
*/
|
||||
write_sysreg(flags, daif);
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -8,14 +8,15 @@
|
|||
#define __ASM_EXCEPTION_H
|
||||
|
||||
#include <asm/esr.h>
|
||||
#include <asm/kprobes.h>
|
||||
#include <asm/ptrace.h>
|
||||
|
||||
#include <linux/interrupt.h>
|
||||
|
||||
#define __exception __attribute__((section(".exception.text")))
|
||||
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
||||
#define __exception_irq_entry __irq_entry
|
||||
#else
|
||||
#define __exception_irq_entry __exception
|
||||
#define __exception_irq_entry __kprobes
|
||||
#endif
|
||||
|
||||
static inline u32 disr_to_esr(u64 disr)
|
||||
|
@ -31,5 +32,22 @@ static inline u32 disr_to_esr(u64 disr)
|
|||
}
|
||||
|
||||
asmlinkage void enter_from_user_mode(void);
|
||||
void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs);
|
||||
void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs);
|
||||
void do_undefinstr(struct pt_regs *regs);
|
||||
asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr);
|
||||
void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
|
||||
struct pt_regs *regs);
|
||||
void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs);
|
||||
void do_sve_acc(unsigned int esr, struct pt_regs *regs);
|
||||
void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs);
|
||||
void do_sysinstr(unsigned int esr, struct pt_regs *regs);
|
||||
void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs);
|
||||
void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr);
|
||||
void do_cp15instr(unsigned int esr, struct pt_regs *regs);
|
||||
void el0_svc_handler(struct pt_regs *regs);
|
||||
void el0_svc_compat_handler(struct pt_regs *regs);
|
||||
void do_el0_ia_bp_hardening(unsigned long addr, unsigned int esr,
|
||||
struct pt_regs *regs);
|
||||
|
||||
#endif /* __ASM_EXCEPTION_H */
|
||||
|
|
|
@ -26,10 +26,12 @@
|
|||
#include <linux/init.h>
|
||||
#include <linux/stddef.h>
|
||||
#include <linux/string.h>
|
||||
#include <linux/thread_info.h>
|
||||
|
||||
#include <asm/alternative.h>
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/hw_breakpoint.h>
|
||||
#include <asm/kasan.h>
|
||||
#include <asm/lse.h>
|
||||
#include <asm/pgtable-hwdef.h>
|
||||
#include <asm/pointer_auth.h>
|
||||
|
@ -214,6 +216,18 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc,
|
|||
regs->sp = sp;
|
||||
}
|
||||
|
||||
static inline bool is_ttbr0_addr(unsigned long addr)
|
||||
{
|
||||
/* entry assembly clears tags for TTBR0 addrs */
|
||||
return addr < TASK_SIZE;
|
||||
}
|
||||
|
||||
static inline bool is_ttbr1_addr(unsigned long addr)
|
||||
{
|
||||
/* TTBR1 addresses may have a tag if KASAN_SW_TAGS is in use */
|
||||
return arch_kasan_reset_tag(addr) >= PAGE_OFFSET;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_COMPAT
|
||||
static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc,
|
||||
unsigned long sp)
|
||||
|
|
|
@ -42,16 +42,6 @@ static inline int __in_irqentry_text(unsigned long ptr)
|
|||
ptr < (unsigned long)&__irqentry_text_end;
|
||||
}
|
||||
|
||||
static inline int in_exception_text(unsigned long ptr)
|
||||
{
|
||||
int in;
|
||||
|
||||
in = ptr >= (unsigned long)&__exception_text_start &&
|
||||
ptr < (unsigned long)&__exception_text_end;
|
||||
|
||||
return in ? : __in_irqentry_text(ptr);
|
||||
}
|
||||
|
||||
static inline int in_entry_text(unsigned long ptr)
|
||||
{
|
||||
return ptr >= (unsigned long)&__entry_text_start &&
|
||||
|
|
|
@ -13,9 +13,9 @@ CFLAGS_REMOVE_return_address.o = $(CC_FLAGS_FTRACE)
|
|||
|
||||
# Object file lists.
|
||||
obj-y := debug-monitors.o entry.o irq.o fpsimd.o \
|
||||
entry-fpsimd.o process.o ptrace.o setup.o signal.o \
|
||||
sys.o stacktrace.o time.o traps.o io.o vdso.o \
|
||||
hyp-stub.o psci.o cpu_ops.o insn.o \
|
||||
entry-common.o entry-fpsimd.o process.o ptrace.o \
|
||||
setup.o signal.o sys.o stacktrace.o time.o traps.o \
|
||||
io.o vdso.o hyp-stub.o psci.o cpu_ops.o insn.o \
|
||||
return_address.o cpuinfo.o cpu_errata.o \
|
||||
cpufeature.o alternative.o cacheinfo.o \
|
||||
smp.o smp_spin_table.o topology.o smccc-call.o \
|
||||
|
|
332
arch/arm64/kernel/entry-common.c
Normal file
332
arch/arm64/kernel/entry-common.c
Normal file
|
@ -0,0 +1,332 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Exception handling code
|
||||
*
|
||||
* Copyright (C) 2019 ARM Ltd.
|
||||
*/
|
||||
|
||||
#include <linux/context_tracking.h>
|
||||
#include <linux/ptrace.h>
|
||||
#include <linux/thread_info.h>
|
||||
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/daifflags.h>
|
||||
#include <asm/esr.h>
|
||||
#include <asm/exception.h>
|
||||
#include <asm/kprobes.h>
|
||||
#include <asm/mmu.h>
|
||||
#include <asm/sysreg.h>
|
||||
|
||||
static void notrace el1_abort(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
unsigned long far = read_sysreg(far_el1);
|
||||
|
||||
local_daif_inherit(regs);
|
||||
far = untagged_addr(far);
|
||||
do_mem_abort(far, esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el1_abort);
|
||||
|
||||
static void notrace el1_pc(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
unsigned long far = read_sysreg(far_el1);
|
||||
|
||||
local_daif_inherit(regs);
|
||||
do_sp_pc_abort(far, esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el1_pc);
|
||||
|
||||
static void el1_undef(struct pt_regs *regs)
|
||||
{
|
||||
local_daif_inherit(regs);
|
||||
do_undefinstr(regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el1_undef);
|
||||
|
||||
static void el1_inv(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
local_daif_inherit(regs);
|
||||
bad_mode(regs, 0, esr);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el1_inv);
|
||||
|
||||
static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
unsigned long far = read_sysreg(far_el1);
|
||||
|
||||
/*
|
||||
* The CPU masked interrupts, and we are leaving them masked during
|
||||
* do_debug_exception(). Update PMR as if we had called
|
||||
* local_mask_daif().
|
||||
*/
|
||||
if (system_uses_irq_prio_masking())
|
||||
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
|
||||
|
||||
do_debug_exception(far, esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el1_dbg);
|
||||
|
||||
asmlinkage void notrace el1_sync_handler(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long esr = read_sysreg(esr_el1);
|
||||
|
||||
switch (ESR_ELx_EC(esr)) {
|
||||
case ESR_ELx_EC_DABT_CUR:
|
||||
case ESR_ELx_EC_IABT_CUR:
|
||||
el1_abort(regs, esr);
|
||||
break;
|
||||
/*
|
||||
* We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a
|
||||
* recursive exception when trying to push the initial pt_regs.
|
||||
*/
|
||||
case ESR_ELx_EC_PC_ALIGN:
|
||||
el1_pc(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_SYS64:
|
||||
case ESR_ELx_EC_UNKNOWN:
|
||||
el1_undef(regs);
|
||||
break;
|
||||
case ESR_ELx_EC_BREAKPT_CUR:
|
||||
case ESR_ELx_EC_SOFTSTP_CUR:
|
||||
case ESR_ELx_EC_WATCHPT_CUR:
|
||||
case ESR_ELx_EC_BRK64:
|
||||
el1_dbg(regs, esr);
|
||||
break;
|
||||
default:
|
||||
el1_inv(regs, esr);
|
||||
};
|
||||
}
|
||||
NOKPROBE_SYMBOL(el1_sync_handler);
|
||||
|
||||
static void notrace el0_da(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
unsigned long far = read_sysreg(far_el1);
|
||||
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
far = untagged_addr(far);
|
||||
do_mem_abort(far, esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_da);
|
||||
|
||||
static void notrace el0_ia(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
unsigned long far = read_sysreg(far_el1);
|
||||
|
||||
/*
|
||||
* We've taken an instruction abort from userspace and not yet
|
||||
* re-enabled IRQs. If the address is a kernel address, apply
|
||||
* BP hardening prior to enabling IRQs and pre-emption.
|
||||
*/
|
||||
if (!is_ttbr0_addr(far))
|
||||
arm64_apply_bp_hardening();
|
||||
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
do_mem_abort(far, esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_ia);
|
||||
|
||||
static void notrace el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
do_fpsimd_acc(esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_fpsimd_acc);
|
||||
|
||||
static void notrace el0_sve_acc(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
do_sve_acc(esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_sve_acc);
|
||||
|
||||
static void notrace el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
do_fpsimd_exc(esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_fpsimd_exc);
|
||||
|
||||
static void notrace el0_sys(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
do_sysinstr(esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_sys);
|
||||
|
||||
static void notrace el0_pc(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
unsigned long far = read_sysreg(far_el1);
|
||||
|
||||
if (!is_ttbr0_addr(instruction_pointer(regs)))
|
||||
arm64_apply_bp_hardening();
|
||||
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
do_sp_pc_abort(far, esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_pc);
|
||||
|
||||
static void notrace el0_sp(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX_NOIRQ);
|
||||
do_sp_pc_abort(regs->sp, esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_sp);
|
||||
|
||||
static void notrace el0_undef(struct pt_regs *regs)
|
||||
{
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
do_undefinstr(regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_undef);
|
||||
|
||||
static void notrace el0_inv(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
bad_el0_sync(regs, 0, esr);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_inv);
|
||||
|
||||
static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
/* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
|
||||
unsigned long far = read_sysreg(far_el1);
|
||||
|
||||
if (system_uses_irq_prio_masking())
|
||||
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
|
||||
|
||||
user_exit_irqoff();
|
||||
do_debug_exception(far, esr, regs);
|
||||
local_daif_restore(DAIF_PROCCTX_NOIRQ);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_dbg);
|
||||
|
||||
static void notrace el0_svc(struct pt_regs *regs)
|
||||
{
|
||||
if (system_uses_irq_prio_masking())
|
||||
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
|
||||
|
||||
el0_svc_handler(regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_svc);
|
||||
|
||||
asmlinkage void notrace el0_sync_handler(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long esr = read_sysreg(esr_el1);
|
||||
|
||||
switch (ESR_ELx_EC(esr)) {
|
||||
case ESR_ELx_EC_SVC64:
|
||||
el0_svc(regs);
|
||||
break;
|
||||
case ESR_ELx_EC_DABT_LOW:
|
||||
el0_da(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_IABT_LOW:
|
||||
el0_ia(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_FP_ASIMD:
|
||||
el0_fpsimd_acc(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_SVE:
|
||||
el0_sve_acc(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_FP_EXC64:
|
||||
el0_fpsimd_exc(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_SYS64:
|
||||
case ESR_ELx_EC_WFx:
|
||||
el0_sys(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_SP_ALIGN:
|
||||
el0_sp(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_PC_ALIGN:
|
||||
el0_pc(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_UNKNOWN:
|
||||
el0_undef(regs);
|
||||
break;
|
||||
case ESR_ELx_EC_BREAKPT_LOW:
|
||||
case ESR_ELx_EC_SOFTSTP_LOW:
|
||||
case ESR_ELx_EC_WATCHPT_LOW:
|
||||
case ESR_ELx_EC_BRK64:
|
||||
el0_dbg(regs, esr);
|
||||
break;
|
||||
default:
|
||||
el0_inv(regs, esr);
|
||||
}
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_sync_handler);
|
||||
|
||||
#ifdef CONFIG_COMPAT
|
||||
static void notrace el0_cp15(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
user_exit_irqoff();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
do_cp15instr(esr, regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_cp15);
|
||||
|
||||
static void notrace el0_svc_compat(struct pt_regs *regs)
|
||||
{
|
||||
if (system_uses_irq_prio_masking())
|
||||
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
|
||||
|
||||
el0_svc_compat_handler(regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_svc_compat);
|
||||
|
||||
asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long esr = read_sysreg(esr_el1);
|
||||
|
||||
switch (ESR_ELx_EC(esr)) {
|
||||
case ESR_ELx_EC_SVC32:
|
||||
el0_svc_compat(regs);
|
||||
break;
|
||||
case ESR_ELx_EC_DABT_LOW:
|
||||
el0_da(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_IABT_LOW:
|
||||
el0_ia(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_FP_ASIMD:
|
||||
el0_fpsimd_acc(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_FP_EXC32:
|
||||
el0_fpsimd_exc(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_PC_ALIGN:
|
||||
el0_pc(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_UNKNOWN:
|
||||
case ESR_ELx_EC_CP14_MR:
|
||||
case ESR_ELx_EC_CP14_LS:
|
||||
case ESR_ELx_EC_CP14_64:
|
||||
el0_undef(regs);
|
||||
break;
|
||||
case ESR_ELx_EC_CP15_32:
|
||||
case ESR_ELx_EC_CP15_64:
|
||||
el0_cp15(regs, esr);
|
||||
break;
|
||||
case ESR_ELx_EC_BREAKPT_LOW:
|
||||
case ESR_ELx_EC_SOFTSTP_LOW:
|
||||
case ESR_ELx_EC_WATCHPT_LOW:
|
||||
case ESR_ELx_EC_BKPT32:
|
||||
el0_dbg(regs, esr);
|
||||
break;
|
||||
default:
|
||||
el0_inv(regs, esr);
|
||||
}
|
||||
}
|
||||
NOKPROBE_SYMBOL(el0_sync_compat_handler);
|
||||
#endif /* CONFIG_COMPAT */
|
|
@ -578,76 +578,9 @@ ENDPROC(el1_error_invalid)
|
|||
.align 6
|
||||
el1_sync:
|
||||
kernel_entry 1
|
||||
mrs x1, esr_el1 // read the syndrome register
|
||||
lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class
|
||||
cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1
|
||||
b.eq el1_da
|
||||
cmp x24, #ESR_ELx_EC_IABT_CUR // instruction abort in EL1
|
||||
b.eq el1_ia
|
||||
cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
|
||||
b.eq el1_undef
|
||||
cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
|
||||
b.eq el1_pc
|
||||
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1
|
||||
b.eq el1_undef
|
||||
cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1
|
||||
b.ge el1_dbg
|
||||
b el1_inv
|
||||
|
||||
el1_ia:
|
||||
/*
|
||||
* Fall through to the Data abort case
|
||||
*/
|
||||
el1_da:
|
||||
/*
|
||||
* Data abort handling
|
||||
*/
|
||||
mrs x3, far_el1
|
||||
inherit_daif pstate=x23, tmp=x2
|
||||
untagged_addr x0, x3
|
||||
mov x2, sp // struct pt_regs
|
||||
bl do_mem_abort
|
||||
|
||||
kernel_exit 1
|
||||
el1_pc:
|
||||
/*
|
||||
* PC alignment exception handling. We don't handle SP alignment faults,
|
||||
* since we will have hit a recursive exception when trying to push the
|
||||
* initial pt_regs.
|
||||
*/
|
||||
mrs x0, far_el1
|
||||
inherit_daif pstate=x23, tmp=x2
|
||||
mov x2, sp
|
||||
bl do_sp_pc_abort
|
||||
ASM_BUG()
|
||||
el1_undef:
|
||||
/*
|
||||
* Undefined instruction
|
||||
*/
|
||||
inherit_daif pstate=x23, tmp=x2
|
||||
mov x0, sp
|
||||
bl do_undefinstr
|
||||
bl el1_sync_handler
|
||||
kernel_exit 1
|
||||
el1_dbg:
|
||||
/*
|
||||
* Debug exception handling
|
||||
*/
|
||||
cmp x24, #ESR_ELx_EC_BRK64 // if BRK64
|
||||
cinc x24, x24, eq // set bit '0'
|
||||
tbz x24, #0, el1_inv // EL1 only
|
||||
gic_prio_kentry_setup tmp=x3
|
||||
mrs x0, far_el1
|
||||
mov x2, sp // struct pt_regs
|
||||
bl do_debug_exception
|
||||
kernel_exit 1
|
||||
el1_inv:
|
||||
// TODO: add support for undefined instructions in kernel mode
|
||||
inherit_daif pstate=x23, tmp=x2
|
||||
mov x0, sp
|
||||
mov x2, x1
|
||||
mov x1, #BAD_SYNC
|
||||
bl bad_mode
|
||||
ASM_BUG()
|
||||
ENDPROC(el1_sync)
|
||||
|
||||
.align 6
|
||||
|
@ -714,71 +647,18 @@ ENDPROC(el1_irq)
|
|||
.align 6
|
||||
el0_sync:
|
||||
kernel_entry 0
|
||||
mrs x25, esr_el1 // read the syndrome register
|
||||
lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
|
||||
cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state
|
||||
b.eq el0_svc
|
||||
cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
|
||||
b.eq el0_da
|
||||
cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
|
||||
b.eq el0_ia
|
||||
cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
|
||||
b.eq el0_fpsimd_acc
|
||||
cmp x24, #ESR_ELx_EC_SVE // SVE access
|
||||
b.eq el0_sve_acc
|
||||
cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception
|
||||
b.eq el0_fpsimd_exc
|
||||
cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
|
||||
ccmp x24, #ESR_ELx_EC_WFx, #4, ne
|
||||
b.eq el0_sys
|
||||
cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
|
||||
b.eq el0_sp
|
||||
cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
|
||||
b.eq el0_pc
|
||||
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
|
||||
b.ge el0_dbg
|
||||
b el0_inv
|
||||
mov x0, sp
|
||||
bl el0_sync_handler
|
||||
b ret_to_user
|
||||
|
||||
#ifdef CONFIG_COMPAT
|
||||
.align 6
|
||||
el0_sync_compat:
|
||||
kernel_entry 0, 32
|
||||
mrs x25, esr_el1 // read the syndrome register
|
||||
lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
|
||||
cmp x24, #ESR_ELx_EC_SVC32 // SVC in 32-bit state
|
||||
b.eq el0_svc_compat
|
||||
cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
|
||||
b.eq el0_da
|
||||
cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
|
||||
b.eq el0_ia
|
||||
cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
|
||||
b.eq el0_fpsimd_acc
|
||||
cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception
|
||||
b.eq el0_fpsimd_exc
|
||||
cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
|
||||
b.eq el0_pc
|
||||
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap
|
||||
b.eq el0_cp15
|
||||
cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap
|
||||
b.eq el0_cp15
|
||||
cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_ELx_EC_CP14_64 // CP14 MRRC/MCRR trap
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
|
||||
b.ge el0_dbg
|
||||
b el0_inv
|
||||
el0_svc_compat:
|
||||
gic_prio_kentry_setup tmp=x1
|
||||
mov x0, sp
|
||||
bl el0_svc_compat_handler
|
||||
bl el0_sync_compat_handler
|
||||
b ret_to_user
|
||||
ENDPROC(el0_sync)
|
||||
|
||||
.align 6
|
||||
el0_irq_compat:
|
||||
|
@ -788,140 +668,8 @@ el0_irq_compat:
|
|||
el0_error_compat:
|
||||
kernel_entry 0, 32
|
||||
b el0_error_naked
|
||||
|
||||
el0_cp15:
|
||||
/*
|
||||
* Trapped CP15 (MRC, MCR, MRRC, MCRR) instructions
|
||||
*/
|
||||
ct_user_exit_irqoff
|
||||
enable_daif
|
||||
mov x0, x25
|
||||
mov x1, sp
|
||||
bl do_cp15instr
|
||||
b ret_to_user
|
||||
#endif
|
||||
|
||||
el0_da:
|
||||
/*
|
||||
* Data abort handling
|
||||
*/
|
||||
mrs x26, far_el1
|
||||
ct_user_exit_irqoff
|
||||
enable_daif
|
||||
untagged_addr x0, x26
|
||||
mov x1, x25
|
||||
mov x2, sp
|
||||
bl do_mem_abort
|
||||
b ret_to_user
|
||||
el0_ia:
|
||||
/*
|
||||
* Instruction abort handling
|
||||
*/
|
||||
mrs x26, far_el1
|
||||
gic_prio_kentry_setup tmp=x0
|
||||
ct_user_exit_irqoff
|
||||
enable_da_f
|
||||
#ifdef CONFIG_TRACE_IRQFLAGS
|
||||
bl trace_hardirqs_off
|
||||
#endif
|
||||
mov x0, x26
|
||||
mov x1, x25
|
||||
mov x2, sp
|
||||
bl do_el0_ia_bp_hardening
|
||||
b ret_to_user
|
||||
el0_fpsimd_acc:
|
||||
/*
|
||||
* Floating Point or Advanced SIMD access
|
||||
*/
|
||||
ct_user_exit_irqoff
|
||||
enable_daif
|
||||
mov x0, x25
|
||||
mov x1, sp
|
||||
bl do_fpsimd_acc
|
||||
b ret_to_user
|
||||
el0_sve_acc:
|
||||
/*
|
||||
* Scalable Vector Extension access
|
||||
*/
|
||||
ct_user_exit_irqoff
|
||||
enable_daif
|
||||
mov x0, x25
|
||||
mov x1, sp
|
||||
bl do_sve_acc
|
||||
b ret_to_user
|
||||
el0_fpsimd_exc:
|
||||
/*
|
||||
* Floating Point, Advanced SIMD or SVE exception
|
||||
*/
|
||||
ct_user_exit_irqoff
|
||||
enable_daif
|
||||
mov x0, x25
|
||||
mov x1, sp
|
||||
bl do_fpsimd_exc
|
||||
b ret_to_user
|
||||
el0_sp:
|
||||
ldr x26, [sp, #S_SP]
|
||||
b el0_sp_pc
|
||||
el0_pc:
|
||||
mrs x26, far_el1
|
||||
el0_sp_pc:
|
||||
/*
|
||||
* Stack or PC alignment exception handling
|
||||
*/
|
||||
gic_prio_kentry_setup tmp=x0
|
||||
ct_user_exit_irqoff
|
||||
enable_da_f
|
||||
#ifdef CONFIG_TRACE_IRQFLAGS
|
||||
bl trace_hardirqs_off
|
||||
#endif
|
||||
mov x0, x26
|
||||
mov x1, x25
|
||||
mov x2, sp
|
||||
bl do_sp_pc_abort
|
||||
b ret_to_user
|
||||
el0_undef:
|
||||
/*
|
||||
* Undefined instruction
|
||||
*/
|
||||
ct_user_exit_irqoff
|
||||
enable_daif
|
||||
mov x0, sp
|
||||
bl do_undefinstr
|
||||
b ret_to_user
|
||||
el0_sys:
|
||||
/*
|
||||
* System instructions, for trapped cache maintenance instructions
|
||||
*/
|
||||
ct_user_exit_irqoff
|
||||
enable_daif
|
||||
mov x0, x25
|
||||
mov x1, sp
|
||||
bl do_sysinstr
|
||||
b ret_to_user
|
||||
el0_dbg:
|
||||
/*
|
||||
* Debug exception handling
|
||||
*/
|
||||
tbnz x24, #0, el0_inv // EL0 only
|
||||
mrs x24, far_el1
|
||||
gic_prio_kentry_setup tmp=x3
|
||||
ct_user_exit_irqoff
|
||||
mov x0, x24
|
||||
mov x1, x25
|
||||
mov x2, sp
|
||||
bl do_debug_exception
|
||||
enable_da_f
|
||||
b ret_to_user
|
||||
el0_inv:
|
||||
ct_user_exit_irqoff
|
||||
enable_daif
|
||||
mov x0, sp
|
||||
mov x1, #BAD_SYNC
|
||||
mov x2, x25
|
||||
bl bad_el0_sync
|
||||
b ret_to_user
|
||||
ENDPROC(el0_sync)
|
||||
|
||||
.align 6
|
||||
el0_irq:
|
||||
kernel_entry 0
|
||||
|
@ -999,17 +747,6 @@ finish_ret_to_user:
|
|||
kernel_exit 0
|
||||
ENDPROC(ret_to_user)
|
||||
|
||||
/*
|
||||
* SVC handler.
|
||||
*/
|
||||
.align 6
|
||||
el0_svc:
|
||||
gic_prio_kentry_setup tmp=x1
|
||||
mov x0, sp
|
||||
bl el0_svc_handler
|
||||
b ret_to_user
|
||||
ENDPROC(el0_svc)
|
||||
|
||||
.popsection // .entry.text
|
||||
|
||||
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
|
||||
|
|
|
@ -920,7 +920,7 @@ void fpsimd_release_task(struct task_struct *dead_task)
|
|||
* would have disabled the SVE access trap for userspace during
|
||||
* ret_to_user, making an SVE access trap impossible in that case.
|
||||
*/
|
||||
asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
|
||||
void do_sve_acc(unsigned int esr, struct pt_regs *regs)
|
||||
{
|
||||
/* Even if we chose not to use SVE, the hardware could still trap: */
|
||||
if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
|
||||
|
@ -947,7 +947,7 @@ asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
|
|||
/*
|
||||
* Trapped FP/ASIMD access.
|
||||
*/
|
||||
asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
|
||||
void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
|
||||
{
|
||||
/* TODO: implement lazy context saving/restoring */
|
||||
WARN_ON(1);
|
||||
|
@ -956,7 +956,7 @@ asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
|
|||
/*
|
||||
* Raise a SIGFPE for the current process.
|
||||
*/
|
||||
asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
|
||||
void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
|
||||
{
|
||||
unsigned int si_code = FPE_FLTUNK;
|
||||
|
||||
|
|
|
@ -455,10 +455,6 @@ int __init arch_populate_kprobe_blacklist(void)
|
|||
(unsigned long)__irqentry_text_end);
|
||||
if (ret)
|
||||
return ret;
|
||||
ret = kprobe_add_area_blacklist((unsigned long)__exception_text_start,
|
||||
(unsigned long)__exception_text_end);
|
||||
if (ret)
|
||||
return ret;
|
||||
ret = kprobe_add_area_blacklist((unsigned long)__idmap_text_start,
|
||||
(unsigned long)__idmap_text_end);
|
||||
if (ret)
|
||||
|
|
|
@ -154,14 +154,14 @@ static inline void sve_user_discard(void)
|
|||
sve_user_disable();
|
||||
}
|
||||
|
||||
asmlinkage void el0_svc_handler(struct pt_regs *regs)
|
||||
void el0_svc_handler(struct pt_regs *regs)
|
||||
{
|
||||
sve_user_discard();
|
||||
el0_svc_common(regs, regs->regs[8], __NR_syscalls, sys_call_table);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_COMPAT
|
||||
asmlinkage void el0_svc_compat_handler(struct pt_regs *regs)
|
||||
void el0_svc_compat_handler(struct pt_regs *regs)
|
||||
{
|
||||
el0_svc_common(regs, regs->regs[7], __NR_compat_syscalls,
|
||||
compat_sys_call_table);
|
||||
|
|
|
@ -35,6 +35,7 @@
|
|||
#include <asm/debug-monitors.h>
|
||||
#include <asm/esr.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/kprobes.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/smp.h>
|
||||
#include <asm/stack_pointer.h>
|
||||
|
@ -393,7 +394,7 @@ void arm64_notify_segfault(unsigned long addr)
|
|||
force_signal_inject(SIGSEGV, code, addr);
|
||||
}
|
||||
|
||||
asmlinkage void __exception do_undefinstr(struct pt_regs *regs)
|
||||
void do_undefinstr(struct pt_regs *regs)
|
||||
{
|
||||
/* check for AArch32 breakpoint instructions */
|
||||
if (!aarch32_break_handler(regs))
|
||||
|
@ -405,6 +406,7 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs)
|
|||
BUG_ON(!user_mode(regs));
|
||||
force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
|
||||
}
|
||||
NOKPROBE_SYMBOL(do_undefinstr);
|
||||
|
||||
#define __user_cache_maint(insn, address, res) \
|
||||
if (address >= user_addr_max()) { \
|
||||
|
@ -676,7 +678,7 @@ static const struct sys64_hook cp15_64_hooks[] = {
|
|||
{},
|
||||
};
|
||||
|
||||
asmlinkage void __exception do_cp15instr(unsigned int esr, struct pt_regs *regs)
|
||||
void do_cp15instr(unsigned int esr, struct pt_regs *regs)
|
||||
{
|
||||
const struct sys64_hook *hook, *hook_base;
|
||||
|
||||
|
@ -714,9 +716,10 @@ asmlinkage void __exception do_cp15instr(unsigned int esr, struct pt_regs *regs)
|
|||
*/
|
||||
do_undefinstr(regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(do_cp15instr);
|
||||
#endif
|
||||
|
||||
asmlinkage void __exception do_sysinstr(unsigned int esr, struct pt_regs *regs)
|
||||
void do_sysinstr(unsigned int esr, struct pt_regs *regs)
|
||||
{
|
||||
const struct sys64_hook *hook;
|
||||
|
||||
|
@ -733,6 +736,7 @@ asmlinkage void __exception do_sysinstr(unsigned int esr, struct pt_regs *regs)
|
|||
*/
|
||||
do_undefinstr(regs);
|
||||
}
|
||||
NOKPROBE_SYMBOL(do_sysinstr);
|
||||
|
||||
static const char *esr_class_str[] = {
|
||||
[0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
|
||||
|
@ -802,7 +806,7 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
|
|||
* bad_el0_sync handles unexpected, but potentially recoverable synchronous
|
||||
* exceptions taken from EL0. Unlike bad_mode, this returns.
|
||||
*/
|
||||
asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
|
||||
void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
|
||||
{
|
||||
void __user *pc = (void __user *)instruction_pointer(regs);
|
||||
|
||||
|
|
|
@ -111,9 +111,6 @@ SECTIONS
|
|||
}
|
||||
.text : { /* Real text segment */
|
||||
_stext = .; /* Text and read-only data */
|
||||
__exception_text_start = .;
|
||||
*(.exception.text)
|
||||
__exception_text_end = .;
|
||||
IRQENTRY_TEXT
|
||||
SOFTIRQENTRY_TEXT
|
||||
ENTRY_TEXT
|
||||
|
|
|
@ -32,7 +32,8 @@
|
|||
#include <asm/daifflags.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/esr.h>
|
||||
#include <asm/kasan.h>
|
||||
#include <asm/kprobes.h>
|
||||
#include <asm/processor.h>
|
||||
#include <asm/sysreg.h>
|
||||
#include <asm/system_misc.h>
|
||||
#include <asm/pgtable.h>
|
||||
|
@ -101,18 +102,6 @@ static void mem_abort_decode(unsigned int esr)
|
|||
data_abort_decode(esr);
|
||||
}
|
||||
|
||||
static inline bool is_ttbr0_addr(unsigned long addr)
|
||||
{
|
||||
/* entry assembly clears tags for TTBR0 addrs */
|
||||
return addr < TASK_SIZE;
|
||||
}
|
||||
|
||||
static inline bool is_ttbr1_addr(unsigned long addr)
|
||||
{
|
||||
/* TTBR1 addresses may have a tag if KASAN_SW_TAGS is in use */
|
||||
return arch_kasan_reset_tag(addr) >= PAGE_OFFSET;
|
||||
}
|
||||
|
||||
static inline unsigned long mm_to_pgd_phys(struct mm_struct *mm)
|
||||
{
|
||||
/* Either init_pg_dir or swapper_pg_dir */
|
||||
|
@ -736,8 +725,7 @@ static const struct fault_info fault_info[] = {
|
|||
{ do_bad, SIGKILL, SI_KERNEL, "unknown 63" },
|
||||
};
|
||||
|
||||
asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
|
||||
struct pt_regs *regs)
|
||||
void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
|
||||
{
|
||||
const struct fault_info *inf = esr_to_fault_info(esr);
|
||||
|
||||
|
@ -753,43 +741,21 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
|
|||
arm64_notify_die(inf->name, regs,
|
||||
inf->sig, inf->code, (void __user *)addr, esr);
|
||||
}
|
||||
NOKPROBE_SYMBOL(do_mem_abort);
|
||||
|
||||
asmlinkage void __exception do_el0_irq_bp_hardening(void)
|
||||
void do_el0_irq_bp_hardening(void)
|
||||
{
|
||||
/* PC has already been checked in entry.S */
|
||||
arm64_apply_bp_hardening();
|
||||
}
|
||||
NOKPROBE_SYMBOL(do_el0_irq_bp_hardening);
|
||||
|
||||
asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr,
|
||||
unsigned int esr,
|
||||
struct pt_regs *regs)
|
||||
void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
|
||||
{
|
||||
/*
|
||||
* We've taken an instruction abort from userspace and not yet
|
||||
* re-enabled IRQs. If the address is a kernel address, apply
|
||||
* BP hardening prior to enabling IRQs and pre-emption.
|
||||
*/
|
||||
if (!is_ttbr0_addr(addr))
|
||||
arm64_apply_bp_hardening();
|
||||
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
do_mem_abort(addr, esr, regs);
|
||||
}
|
||||
|
||||
|
||||
asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
|
||||
unsigned int esr,
|
||||
struct pt_regs *regs)
|
||||
{
|
||||
if (user_mode(regs)) {
|
||||
if (!is_ttbr0_addr(instruction_pointer(regs)))
|
||||
arm64_apply_bp_hardening();
|
||||
local_daif_restore(DAIF_PROCCTX);
|
||||
}
|
||||
|
||||
arm64_notify_die("SP/PC alignment exception", regs,
|
||||
SIGBUS, BUS_ADRALN, (void __user *)addr, esr);
|
||||
}
|
||||
NOKPROBE_SYMBOL(do_sp_pc_abort);
|
||||
|
||||
int __init early_brk64(unsigned long addr, unsigned int esr,
|
||||
struct pt_regs *regs);
|
||||
|
@ -872,8 +838,7 @@ NOKPROBE_SYMBOL(debug_exception_exit);
|
|||
#ifdef CONFIG_ARM64_ERRATUM_1463225
|
||||
DECLARE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
|
||||
|
||||
static int __exception
|
||||
cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
|
||||
static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
|
||||
{
|
||||
if (user_mode(regs))
|
||||
return 0;
|
||||
|
@ -892,15 +857,14 @@ cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
|
|||
return 1;
|
||||
}
|
||||
#else
|
||||
static int __exception
|
||||
cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
|
||||
static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif /* CONFIG_ARM64_ERRATUM_1463225 */
|
||||
NOKPROBE_SYMBOL(cortex_a76_erratum_1463225_debug_handler);
|
||||
|
||||
asmlinkage void __exception do_debug_exception(unsigned long addr_if_watchpoint,
|
||||
unsigned int esr,
|
||||
void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
|
||||
struct pt_regs *regs)
|
||||
{
|
||||
const struct fault_info *inf = esr_to_debug_fault_info(esr);
|
||||
|
|
Loading…
Reference in a new issue