arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
// Copyright (C) 2017 Arm Ltd.
|
|
|
|
#define pr_fmt(fmt) "sdei: " fmt
|
|
|
|
|
|
|
|
#include <linux/arm_sdei.h>
|
|
|
|
#include <linux/hardirq.h>
|
|
|
|
#include <linux/irqflags.h>
|
|
|
|
#include <linux/sched/task_stack.h>
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
|
|
|
|
#include <asm/alternative.h>
|
|
|
|
#include <asm/kprobes.h>
|
2018-01-08 08:38:18 -07:00
|
|
|
#include <asm/mmu.h>
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
#include <asm/ptrace.h>
|
2018-01-08 08:38:18 -07:00
|
|
|
#include <asm/sections.h>
|
2018-07-20 15:41:53 -06:00
|
|
|
#include <asm/stacktrace.h>
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
#include <asm/sysreg.h>
|
|
|
|
#include <asm/vmap_stack.h>
|
|
|
|
|
|
|
|
unsigned long sdei_exit_mode;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* VMAP'd stacks checking for stack overflow on exception using sp as a scratch
|
|
|
|
* register, meaning SDEI has to switch to its own stack. We need two stacks as
|
|
|
|
* a critical event may interrupt a normal event that has just taken a
|
|
|
|
* synchronous exception, and is using sp as scratch register. For a critical
|
|
|
|
* event interrupting a normal event, we can't reliably tell if we were on the
|
|
|
|
* sdei stack.
|
|
|
|
* For now, we allocate stacks when the driver is probed.
|
|
|
|
*/
|
|
|
|
DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
|
|
|
|
DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
|
|
|
|
|
|
|
|
#ifdef CONFIG_VMAP_STACK
|
|
|
|
DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
|
|
|
|
DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
|
|
|
|
{
|
|
|
|
unsigned long *p;
|
|
|
|
|
|
|
|
p = per_cpu(*ptr, cpu);
|
|
|
|
if (p) {
|
|
|
|
per_cpu(*ptr, cpu) = NULL;
|
|
|
|
vfree(p);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void free_sdei_stacks(void)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
_free_sdei_stack(&sdei_stack_normal_ptr, cpu);
|
|
|
|
_free_sdei_stack(&sdei_stack_critical_ptr, cpu);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int _init_sdei_stack(unsigned long * __percpu *ptr, int cpu)
|
|
|
|
{
|
|
|
|
unsigned long *p;
|
|
|
|
|
|
|
|
p = arch_alloc_vmap_stack(SDEI_STACK_SIZE, cpu_to_node(cpu));
|
|
|
|
if (!p)
|
|
|
|
return -ENOMEM;
|
|
|
|
per_cpu(*ptr, cpu) = p;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int init_sdei_stacks(void)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
|
|
|
|
if (err)
|
|
|
|
break;
|
|
|
|
err = _init_sdei_stack(&sdei_stack_critical_ptr, cpu);
|
|
|
|
if (err)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (err)
|
|
|
|
free_sdei_stacks();
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2018-07-31 05:02:18 -06:00
|
|
|
static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info)
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
{
|
2018-07-20 15:41:53 -06:00
|
|
|
unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
|
|
|
|
unsigned long high = low + SDEI_STACK_SIZE;
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
|
2018-07-20 15:41:53 -06:00
|
|
|
if (sp < low || sp >= high)
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
return false;
|
|
|
|
|
2018-07-20 15:41:53 -06:00
|
|
|
if (info) {
|
|
|
|
info->low = low;
|
|
|
|
info->high = high;
|
|
|
|
info->type = STACK_TYPE_SDEI_NORMAL;
|
|
|
|
}
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
|
2018-07-20 15:41:53 -06:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2018-07-31 05:02:18 -06:00
|
|
|
static bool on_sdei_critical_stack(unsigned long sp, struct stack_info *info)
|
2018-07-20 15:41:53 -06:00
|
|
|
{
|
|
|
|
unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
|
|
|
|
unsigned long high = low + SDEI_STACK_SIZE;
|
|
|
|
|
|
|
|
if (sp < low || sp >= high)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (info) {
|
|
|
|
info->low = low;
|
|
|
|
info->high = high;
|
|
|
|
info->type = STACK_TYPE_SDEI_CRITICAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2018-07-31 05:02:18 -06:00
|
|
|
bool _on_sdei_stack(unsigned long sp, struct stack_info *info)
|
2018-07-20 15:41:53 -06:00
|
|
|
{
|
|
|
|
if (!IS_ENABLED(CONFIG_VMAP_STACK))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (on_sdei_critical_stack(sp, info))
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
return true;
|
|
|
|
|
2018-07-20 15:41:53 -06:00
|
|
|
if (on_sdei_normal_stack(sp, info))
|
|
|
|
return true;
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
|
2018-07-20 15:41:53 -06:00
|
|
|
return false;
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long sdei_arch_get_entry_point(int conduit)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* SDEI works between adjacent exception levels. If we booted at EL1 we
|
|
|
|
* assume a hypervisor is marshalling events. If we booted at EL2 and
|
|
|
|
* dropped to EL1 because we don't support VHE, then we can't support
|
|
|
|
* SDEI.
|
|
|
|
*/
|
|
|
|
if (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
|
|
|
|
pr_err("Not supported on this hardware/boot configuration\n");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (IS_ENABLED(CONFIG_VMAP_STACK)) {
|
|
|
|
if (init_sdei_stacks())
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
sdei_exit_mode = (conduit == CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
|
2018-01-08 08:38:18 -07:00
|
|
|
|
|
|
|
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
|
|
|
|
if (arm64_kernel_unmapped_at_el0()) {
|
|
|
|
unsigned long offset;
|
|
|
|
|
|
|
|
offset = (unsigned long)__sdei_asm_entry_trampoline -
|
|
|
|
(unsigned long)__entry_tramp_text_start;
|
|
|
|
return TRAMP_VALIAS + offset;
|
|
|
|
} else
|
|
|
|
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
|
|
|
|
return (unsigned long)__sdei_asm_handler;
|
|
|
|
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* __sdei_handler() returns one of:
|
|
|
|
* SDEI_EV_HANDLED - success, return to the interrupted context.
|
|
|
|
* SDEI_EV_FAILED - failure, return this error code to firmare.
|
|
|
|
* virtual-address - success, return to this address.
|
|
|
|
*/
|
|
|
|
static __kprobes unsigned long _sdei_handler(struct pt_regs *regs,
|
|
|
|
struct sdei_registered_event *arg)
|
|
|
|
{
|
|
|
|
u32 mode;
|
|
|
|
int i, err = 0;
|
2018-01-08 08:38:18 -07:00
|
|
|
int clobbered_registers = 4;
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
u64 elr = read_sysreg(elr_el1);
|
|
|
|
u32 kernel_mode = read_sysreg(CurrentEL) | 1; /* +SPSel */
|
|
|
|
unsigned long vbar = read_sysreg(vbar_el1);
|
|
|
|
|
2018-01-08 08:38:18 -07:00
|
|
|
if (arm64_kernel_unmapped_at_el0())
|
|
|
|
clobbered_registers++;
|
|
|
|
|
arm64: kernel: Add arch-specific SDEI entry code and CPU masking
The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.
Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),
Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.
This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.
Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.
Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.
When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.
Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-01-08 08:38:12 -07:00
|
|
|
/* Retrieve the missing registers values */
|
|
|
|
for (i = 0; i < clobbered_registers; i++) {
|
|
|
|
/* from within the handler, this call always succeeds */
|
|
|
|
sdei_api_event_context(i, ®s->regs[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We didn't take an exception to get here, set PAN. UAO will be cleared
|
|
|
|
* by sdei_event_handler()s set_fs(USER_DS) call.
|
|
|
|
*/
|
|
|
|
__uaccess_enable_hw_pan();
|
|
|
|
|
|
|
|
err = sdei_event_handler(regs, arg);
|
|
|
|
if (err)
|
|
|
|
return SDEI_EV_FAILED;
|
|
|
|
|
|
|
|
if (elr != read_sysreg(elr_el1)) {
|
|
|
|
/*
|
|
|
|
* We took a synchronous exception from the SDEI handler.
|
|
|
|
* This could deadlock, and if you interrupt KVM it will
|
|
|
|
* hyp-panic instead.
|
|
|
|
*/
|
|
|
|
pr_warn("unsafe: exception during handler\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
mode = regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we interrupted the kernel with interrupts masked, we always go
|
|
|
|
* back to wherever we came from.
|
|
|
|
*/
|
|
|
|
if (mode == kernel_mode && !interrupts_enabled(regs))
|
|
|
|
return SDEI_EV_HANDLED;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Otherwise, we pretend this was an IRQ. This lets user space tasks
|
|
|
|
* receive signals before we return to them, and KVM to invoke it's
|
|
|
|
* world switch to do the same.
|
|
|
|
*
|
|
|
|
* See DDI0487B.a Table D1-7 'Vector offsets from vector table base
|
|
|
|
* address'.
|
|
|
|
*/
|
|
|
|
if (mode == kernel_mode)
|
|
|
|
return vbar + 0x280;
|
|
|
|
else if (mode & PSR_MODE32_BIT)
|
|
|
|
return vbar + 0x680;
|
|
|
|
|
|
|
|
return vbar + 0x480;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
asmlinkage __kprobes notrace unsigned long
|
|
|
|
__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
|
|
|
|
{
|
|
|
|
unsigned long ret;
|
|
|
|
bool do_nmi_exit = false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* nmi_enter() deals with printk() re-entrance and use of RCU when
|
|
|
|
* RCU believed this CPU was idle. Because critical events can
|
|
|
|
* interrupt normal events, we may already be in_nmi().
|
|
|
|
*/
|
|
|
|
if (!in_nmi()) {
|
|
|
|
nmi_enter();
|
|
|
|
do_nmi_exit = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = _sdei_handler(regs, arg);
|
|
|
|
|
|
|
|
if (do_nmi_exit)
|
|
|
|
nmi_exit();
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|