2019-05-27 00:55:01 -06:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
2016-05-17 19:16:50 -06:00
|
|
|
#ifndef _ASM_POWERPC_ASM_PROTOTYPES_H
|
|
|
|
|
#define _ASM_POWERPC_ASM_PROTOTYPES_H
|
|
|
|
|
/*
|
|
|
|
|
* This file is for prototypes of C functions that are only called
|
|
|
|
|
* from asm, and any associated variables.
|
|
|
|
|
*
|
|
|
|
|
* Copyright 2016, Daniel Axtens, IBM Corporation.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <linux/threads.h>
|
2016-11-21 20:52:22 -07:00
|
|
|
#include <asm/cacheflush.h>
|
|
|
|
|
#include <asm/checksum.h>
|
2016-12-24 12:46:01 -07:00
|
|
|
#include <linux/uaccess.h>
|
2016-11-21 20:52:22 -07:00
|
|
|
#include <asm/epapr_hcalls.h>
|
2016-12-01 19:38:38 -07:00
|
|
|
#include <asm/dcr.h>
|
|
|
|
|
#include <asm/mmu_context.h>
|
powerpc/kernel: Add ucall_norets() ultravisor call handler
The ultracalls (ucalls for short) allow the Secure Virtual Machines
(SVM)s and hypervisor to request services from the ultravisor such as
accessing a register or memory region that can only be accessed when
running in ultravisor-privileged mode.
This patch adds the ucall_norets() ultravisor call handler.
The specific service needed from an ucall is specified in register
R3 (the first parameter to the ucall). Other parameters to the
ucall, if any, are specified in registers R4 through R12.
Return value of all ucalls is in register R3. Other output values
from the ucall, if any, are returned in registers R4 through R12.
Each ucall returns specific error codes, applicable in the context
of the ucall. However, like with the PowerPC Architecture Platform
Reference (PAPR), if no specific error code is defined for a particular
situation, then the ucall will fallback to an erroneous
parameter-position based code. i.e U_PARAMETER, U_P2, U_P3 etc depending
on the ucall parameter that may have caused the error.
Every host kernel (powernv) needs to be able to do ucalls in case it
ends up being run in a machine with ultravisor enabled. Otherwise, the
kernel may crash early in boot trying to access ultravisor resources,
for instance, trying to set the partition table entry 0. Secure guests
also need to be able to do ucalls and its kernel may not have
CONFIG_PPC_POWERNV=y. For that reason, the ucall.S file is placed under
arch/powerpc/kernel.
If ultravisor is not enabled, the ucalls will be redirected to the
hypervisor which must handle/fail the call.
Thanks to inputs from Ram Pai and Michael Anderson.
Signed-off-by: Claudio Carvalho <cclaudio@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20190822034838.27876-3-cclaudio@linux.ibm.com
2019-08-21 21:48:33 -06:00
|
|
|
#include <asm/ultravisor-api.h>
|
2016-05-17 19:16:50 -06:00
|
|
|
|
2016-09-05 23:32:43 -06:00
|
|
|
#include <uapi/asm/ucontext.h>
|
|
|
|
|
|
2016-05-17 19:16:50 -06:00
|
|
|
/* SMP */
|
2019-01-31 03:09:02 -07:00
|
|
|
extern struct task_struct *current_set[NR_CPUS];
|
|
|
|
|
extern struct task_struct *secondary_current;
|
2016-05-17 19:16:50 -06:00
|
|
|
void start_secondary(void *unused);
|
|
|
|
|
|
|
|
|
|
/* kexec */
|
|
|
|
|
struct paca_struct;
|
|
|
|
|
struct kimage;
|
|
|
|
|
extern struct paca_struct kexec_paca;
|
|
|
|
|
void kexec_copy_flush(struct kimage *image);
|
|
|
|
|
|
|
|
|
|
/* pseries hcall tracing */
|
|
|
|
|
extern struct static_key hcall_tracepoint_key;
|
|
|
|
|
void __trace_hcall_entry(unsigned long opcode, unsigned long *args);
|
2018-05-07 07:03:55 -06:00
|
|
|
void __trace_hcall_exit(long opcode, long retval, unsigned long *retbuf);
|
2016-05-17 19:16:50 -06:00
|
|
|
|
powerpc/kernel: Add ucall_norets() ultravisor call handler
The ultracalls (ucalls for short) allow the Secure Virtual Machines
(SVM)s and hypervisor to request services from the ultravisor such as
accessing a register or memory region that can only be accessed when
running in ultravisor-privileged mode.
This patch adds the ucall_norets() ultravisor call handler.
The specific service needed from an ucall is specified in register
R3 (the first parameter to the ucall). Other parameters to the
ucall, if any, are specified in registers R4 through R12.
Return value of all ucalls is in register R3. Other output values
from the ucall, if any, are returned in registers R4 through R12.
Each ucall returns specific error codes, applicable in the context
of the ucall. However, like with the PowerPC Architecture Platform
Reference (PAPR), if no specific error code is defined for a particular
situation, then the ucall will fallback to an erroneous
parameter-position based code. i.e U_PARAMETER, U_P2, U_P3 etc depending
on the ucall parameter that may have caused the error.
Every host kernel (powernv) needs to be able to do ucalls in case it
ends up being run in a machine with ultravisor enabled. Otherwise, the
kernel may crash early in boot trying to access ultravisor resources,
for instance, trying to set the partition table entry 0. Secure guests
also need to be able to do ucalls and its kernel may not have
CONFIG_PPC_POWERNV=y. For that reason, the ucall.S file is placed under
arch/powerpc/kernel.
If ultravisor is not enabled, the ucalls will be redirected to the
hypervisor which must handle/fail the call.
Thanks to inputs from Ram Pai and Michael Anderson.
Signed-off-by: Claudio Carvalho <cclaudio@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20190822034838.27876-3-cclaudio@linux.ibm.com
2019-08-21 21:48:33 -06:00
|
|
|
/* Ultravisor */
|
2019-08-19 20:13:12 -06:00
|
|
|
#if defined(CONFIG_PPC_POWERNV) || defined(CONFIG_PPC_SVM)
|
powerpc/kernel: Add ucall_norets() ultravisor call handler
The ultracalls (ucalls for short) allow the Secure Virtual Machines
(SVM)s and hypervisor to request services from the ultravisor such as
accessing a register or memory region that can only be accessed when
running in ultravisor-privileged mode.
This patch adds the ucall_norets() ultravisor call handler.
The specific service needed from an ucall is specified in register
R3 (the first parameter to the ucall). Other parameters to the
ucall, if any, are specified in registers R4 through R12.
Return value of all ucalls is in register R3. Other output values
from the ucall, if any, are returned in registers R4 through R12.
Each ucall returns specific error codes, applicable in the context
of the ucall. However, like with the PowerPC Architecture Platform
Reference (PAPR), if no specific error code is defined for a particular
situation, then the ucall will fallback to an erroneous
parameter-position based code. i.e U_PARAMETER, U_P2, U_P3 etc depending
on the ucall parameter that may have caused the error.
Every host kernel (powernv) needs to be able to do ucalls in case it
ends up being run in a machine with ultravisor enabled. Otherwise, the
kernel may crash early in boot trying to access ultravisor resources,
for instance, trying to set the partition table entry 0. Secure guests
also need to be able to do ucalls and its kernel may not have
CONFIG_PPC_POWERNV=y. For that reason, the ucall.S file is placed under
arch/powerpc/kernel.
If ultravisor is not enabled, the ucalls will be redirected to the
hypervisor which must handle/fail the call.
Thanks to inputs from Ram Pai and Michael Anderson.
Signed-off-by: Claudio Carvalho <cclaudio@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20190822034838.27876-3-cclaudio@linux.ibm.com
2019-08-21 21:48:33 -06:00
|
|
|
long ucall_norets(unsigned long opcode, ...);
|
|
|
|
|
#else
|
|
|
|
|
static inline long ucall_norets(unsigned long opcode, ...)
|
|
|
|
|
{
|
|
|
|
|
return U_NOT_AVAILABLE;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
powerpc/powernv: move OPAL call wrapper tracing and interrupt handling to C
The OPAL call wrapper gets interrupt disabling wrong. It disables
interrupts just by clearing MSR[EE], which has two problems:
- It doesn't call into the IRQ tracing subsystem, which means tracing
across OPAL calls does not always notice IRQs have been disabled.
- It doesn't go through the IRQ soft-mask code, which causes a minor
bug. MSR[EE] can not be restored by saving the MSR then clearing
MSR[EE], because a racing interrupt while soft-masked could clear
MSR[EE] between the two steps. This can cause MSR[EE] to be
incorrectly enabled when the OPAL call returns. Fortunately that
should only result in another masked interrupt being taken to
disable MSR[EE] again, but it's a bit sloppy.
The existing code also saves MSR to PACA, which is not re-entrant if
there is a nested OPAL call from different MSR contexts, which can
happen these days with SRESET interrupts on bare metal.
To fix these issues, move the tracing and IRQ handling code to C, and
call into asm just for the low level call when everything is ready to
go. Save the MSR on stack rather than PACA.
Performance cost is kept to a minimum with a few optimisations:
- The endian switch upon return is combined with the MSR restore,
which avoids an expensive context synchronizing operation for LE
kernels. This makes up for the additional mtmsrd to enable
interrupts with local_irq_enable().
- blr is now used to return from the opal_* functions that are called
as C functions, to avoid link stack corruption. This requires a
skiboot fix as well to keep the call stack balanced.
A NULL call is more costly after this, (410ns->430ns on POWER9), but
OPAL calls are generally not performance critical at this scale.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2019-02-26 02:30:35 -07:00
|
|
|
/* OPAL */
|
|
|
|
|
int64_t __opal_call(int64_t a0, int64_t a1, int64_t a2, int64_t a3,
|
|
|
|
|
int64_t a4, int64_t a5, int64_t a6, int64_t a7,
|
|
|
|
|
int64_t opcode, uint64_t msr);
|
2016-05-17 19:16:50 -06:00
|
|
|
|
|
|
|
|
/* VMX copying */
|
|
|
|
|
int enter_vmx_usercopy(void);
|
|
|
|
|
int exit_vmx_usercopy(void);
|
2018-06-06 19:57:53 -06:00
|
|
|
int enter_vmx_ops(void);
|
|
|
|
|
void *exit_vmx_ops(void *dest);
|
2016-05-17 19:16:50 -06:00
|
|
|
|
|
|
|
|
/* Traps */
|
|
|
|
|
long machine_check_early(struct pt_regs *regs);
|
|
|
|
|
long hmi_exception_realmode(struct pt_regs *regs);
|
|
|
|
|
void SMIException(struct pt_regs *regs);
|
|
|
|
|
void handle_hmi_exception(struct pt_regs *regs);
|
|
|
|
|
void instruction_breakpoint_exception(struct pt_regs *regs);
|
|
|
|
|
void RunModeException(struct pt_regs *regs);
|
2016-09-16 04:48:08 -06:00
|
|
|
void single_step_exception(struct pt_regs *regs);
|
|
|
|
|
void program_check_exception(struct pt_regs *regs);
|
2016-05-17 19:16:50 -06:00
|
|
|
void alignment_exception(struct pt_regs *regs);
|
|
|
|
|
void StackOverflow(struct pt_regs *regs);
|
|
|
|
|
void kernel_fp_unavailable_exception(struct pt_regs *regs);
|
|
|
|
|
void altivec_unavailable_exception(struct pt_regs *regs);
|
|
|
|
|
void vsx_unavailable_exception(struct pt_regs *regs);
|
|
|
|
|
void fp_unavailable_tm(struct pt_regs *regs);
|
|
|
|
|
void altivec_unavailable_tm(struct pt_regs *regs);
|
|
|
|
|
void vsx_unavailable_tm(struct pt_regs *regs);
|
|
|
|
|
void facility_unavailable_exception(struct pt_regs *regs);
|
|
|
|
|
void TAUException(struct pt_regs *regs);
|
|
|
|
|
void altivec_assist_exception(struct pt_regs *regs);
|
|
|
|
|
void unrecoverable_exception(struct pt_regs *regs);
|
|
|
|
|
void kernel_bad_stack(struct pt_regs *regs);
|
|
|
|
|
void system_reset_exception(struct pt_regs *regs);
|
|
|
|
|
void machine_check_exception(struct pt_regs *regs);
|
2016-09-16 04:48:08 -06:00
|
|
|
void emulation_assist_interrupt(struct pt_regs *regs);
|
2018-09-14 09:30:51 -06:00
|
|
|
long do_slb_fault(struct pt_regs *regs, unsigned long ea);
|
|
|
|
|
void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err);
|
2016-05-17 19:16:50 -06:00
|
|
|
|
2016-09-05 23:32:43 -06:00
|
|
|
/* signals, syscalls and interrupts */
|
|
|
|
|
long sys_swapcontext(struct ucontext __user *old_ctx,
|
|
|
|
|
struct ucontext __user *new_ctx,
|
2018-05-02 07:20:47 -06:00
|
|
|
long ctx_size);
|
|
|
|
|
#ifdef CONFIG_PPC32
|
|
|
|
|
long sys_debug_setcontext(struct ucontext __user *ctx,
|
|
|
|
|
int ndbg, struct sig_dbg_op __user *dbg);
|
2018-02-25 10:22:35 -07:00
|
|
|
int
|
|
|
|
|
ppc_select(int n, fd_set __user *inp, fd_set __user *outp, fd_set __user *exp, struct timeval __user *tvp);
|
2018-03-08 04:31:59 -07:00
|
|
|
unsigned long __init early_init(unsigned long dt_ptr);
|
|
|
|
|
void __init machine_init(u64 dt_ptr);
|
2016-09-05 23:32:43 -06:00
|
|
|
#endif
|
2018-03-08 04:31:59 -07:00
|
|
|
|
2018-02-25 10:22:35 -07:00
|
|
|
long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low,
|
|
|
|
|
u32 len_high, u32 len_low);
|
2016-09-05 23:32:43 -06:00
|
|
|
long sys_switch_endian(void);
|
|
|
|
|
notrace unsigned int __check_irq_replay(void);
|
|
|
|
|
void notrace restore_interrupts(void);
|
|
|
|
|
|
|
|
|
|
/* ptrace */
|
|
|
|
|
long do_syscall_trace_enter(struct pt_regs *regs);
|
|
|
|
|
void do_syscall_trace_leave(struct pt_regs *regs);
|
|
|
|
|
|
|
|
|
|
/* process */
|
|
|
|
|
void restore_math(struct pt_regs *regs);
|
|
|
|
|
void restore_tm_state(struct pt_regs *regs);
|
|
|
|
|
|
|
|
|
|
/* prom_init (OpenFirmware) */
|
|
|
|
|
unsigned long __init prom_init(unsigned long r3, unsigned long r4,
|
|
|
|
|
unsigned long pp,
|
|
|
|
|
unsigned long r6, unsigned long r7,
|
|
|
|
|
unsigned long kbase);
|
|
|
|
|
|
|
|
|
|
/* setup */
|
|
|
|
|
void __init early_setup(unsigned long dt_ptr);
|
|
|
|
|
void early_setup_secondary(void);
|
|
|
|
|
|
|
|
|
|
/* time */
|
|
|
|
|
void accumulate_stolen_time(void);
|
|
|
|
|
|
2016-11-21 20:52:22 -07:00
|
|
|
/* misc runtime */
|
|
|
|
|
extern u64 __bswapdi2(u64);
|
|
|
|
|
extern s64 __lshrdi3(s64, int);
|
|
|
|
|
extern s64 __ashldi3(s64, int);
|
|
|
|
|
extern s64 __ashrdi3(s64, int);
|
|
|
|
|
extern int __cmpdi2(s64, s64);
|
|
|
|
|
extern int __ucmpdi2(u64, u64);
|
|
|
|
|
|
2017-03-06 01:49:46 -07:00
|
|
|
/* tracing */
|
2017-02-09 05:55:43 -07:00
|
|
|
void _mcount(void);
|
2019-09-05 12:20:29 -06:00
|
|
|
unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip,
|
|
|
|
|
unsigned long sp);
|
2017-02-09 05:55:43 -07:00
|
|
|
|
powerpc/powernv: Provide a way to force a core into SMT4 mode
POWER9 processors up to and including "Nimbus" v2.2 have hardware
bugs relating to transactional memory and thread reconfiguration.
One of these bugs has a workaround which is to get the core into
SMT4 state temporarily. This workaround is only needed when
running bare-metal.
This patch provides a function which gets the core into SMT4 mode
by preventing threads from going to a stop state, and waking up
those which are already in a stop state. Once at least 3 threads
are not in a stop state, the core will be in SMT4 and we can
continue.
To do this, we add a "dont_stop" flag to the paca to tell the
thread not to go into a stop state. If this flag is set,
power9_idle_stop() just returns immediately with a return value
of 0. The pnv_power9_force_smt4_catch() function does the following:
1. Set the dont_stop flag for each thread in the core, except
ourselves (in fact we use an atomic_inc() in case more than
one thread is calling this function concurrently).
2. See how many threads are awake, indicated by their
requested_psscr field in the paca being 0. If this is at
least 3, skip to step 5.
3. Send a doorbell interrupt to each thread that was seen as
being in a stop state in step 2.
4. Until at least 3 threads are awake, scan the threads to which
we sent a doorbell interrupt and check if they are awake now.
This relies on the following properties:
- Once dont_stop is non-zero, requested_psccr can't go from zero to
non-zero, except transiently (and without the thread doing stop).
- requested_psscr being zero guarantees that the thread isn't in
a state-losing stop state where thread reconfiguration could occur.
- Doing stop with a PSSCR value of 0 won't be a state-losing stop
and thus won't allow thread reconfiguration.
- Once threads_per_core/2 + 1 (i.e. 3) threads are awake, the core
must be in SMT4 mode, since SMT modes are powers of 2.
This does add a sync to power9_idle_stop(), which is necessary to
provide the correct ordering between setting requested_psscr and
checking dont_stop. The overhead of the sync should be unnoticeable
compared to the latency of going into and out of a stop state.
Because some objected to incurring this extra latency on systems where
the XER[SO] bug is not relevant, I have put the test in
power9_idle_stop inside a feature section. This means that
pnv_power9_force_smt4_catch() WILL NOT WORK correctly on systems
without the CPU_FTR_P9_TM_XER_SO_BUG feature bit set, and will
probably hang the system.
In order to cater for uses where the caller has an operation that
has to be done while the core is in SMT4, the core continues to be
kept in SMT4 after pnv_power9_force_smt4_catch() function returns,
until the pnv_power9_force_smt4_release() function is called.
It undoes the effect of step 1 above and allows the other threads
to go into a stop state.
Signed-off-by: Paul Mackerras <paulus@ozlabs.org>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2018-03-21 04:32:00 -06:00
|
|
|
void pnv_power9_force_smt4_catch(void);
|
|
|
|
|
void pnv_power9_force_smt4_release(void);
|
|
|
|
|
|
2018-05-23 01:01:50 -06:00
|
|
|
/* Transaction memory related */
|
2018-05-23 01:01:46 -06:00
|
|
|
void tm_enable(void);
|
|
|
|
|
void tm_disable(void);
|
|
|
|
|
void tm_abort(uint8_t cause);
|
2018-05-23 01:01:50 -06:00
|
|
|
|
|
|
|
|
struct kvm_vcpu;
|
|
|
|
|
void _kvmppc_restore_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr);
|
|
|
|
|
void _kvmppc_save_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr);
|
|
|
|
|
|
2018-07-23 09:07:54 -06:00
|
|
|
/* Patch sites */
|
|
|
|
|
extern s32 patch__call_flush_count_cache;
|
|
|
|
|
extern s32 patch__flush_count_cache_return;
|
2019-11-13 03:05:41 -07:00
|
|
|
extern s32 patch__flush_link_stack_return;
|
2019-11-13 03:05:44 -07:00
|
|
|
extern s32 patch__call_kvm_flush_link_stack;
|
2018-08-09 02:14:41 -06:00
|
|
|
extern s32 patch__memset_nocache, patch__memcpy_nocache;
|
2018-07-23 09:07:54 -06:00
|
|
|
|
|
|
|
|
extern long flush_count_cache;
|
2019-11-13 03:05:44 -07:00
|
|
|
extern long kvm_flush_link_stack;
|
2018-07-23 09:07:54 -06:00
|
|
|
|
KVM: PPC: Book3S: Rework TM save/restore code and make it C-callable
This adds a parameter to __kvmppc_save_tm and __kvmppc_restore_tm
which allows the caller to indicate whether it wants the nonvolatile
register state to be preserved across the call, as required by the C
calling conventions. This parameter being non-zero also causes the
MSR bits that enable TM, FP, VMX and VSX to be preserved. The
condition register and DSCR are now always preserved.
With this, kvmppc_save_tm_hv and kvmppc_restore_tm_hv can be called
from C code provided the 3rd parameter is non-zero. So that these
functions can be called from modules, they now include code to set
the TOC pointer (r2) on entry, as they can call other built-in C
functions which will assume the TOC to have been set.
Also, the fake suspend code in kvmppc_save_tm_hv is modified here to
assume that treclaim in fake-suspend state does not modify any registers,
which is the case on POWER9. This enables the code to be simplified
quite a bit.
_kvmppc_save_tm_pr and _kvmppc_restore_tm_pr become much simpler with
this change, since they now only need to save and restore TAR and pass
1 for the 3rd argument to __kvmppc_{save,restore}_tm.
Reviewed-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Paul Mackerras <paulus@ozlabs.org>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2018-10-07 23:30:53 -06:00
|
|
|
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
|
|
|
|
|
void kvmppc_save_tm_hv(struct kvm_vcpu *vcpu, u64 msr, bool preserve_nv);
|
|
|
|
|
void kvmppc_restore_tm_hv(struct kvm_vcpu *vcpu, u64 msr, bool preserve_nv);
|
|
|
|
|
#else
|
|
|
|
|
static inline void kvmppc_save_tm_hv(struct kvm_vcpu *vcpu, u64 msr,
|
|
|
|
|
bool preserve_nv) { }
|
|
|
|
|
static inline void kvmppc_restore_tm_hv(struct kvm_vcpu *vcpu, u64 msr,
|
|
|
|
|
bool preserve_nv) { }
|
|
|
|
|
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
|
|
|
|
|
|
2018-10-07 23:30:51 -06:00
|
|
|
void kvmhv_save_host_pmu(void);
|
|
|
|
|
void kvmhv_load_host_pmu(void);
|
|
|
|
|
void kvmhv_save_guest_pmu(struct kvm_vcpu *vcpu, bool pmu_in_use);
|
|
|
|
|
void kvmhv_load_guest_pmu(struct kvm_vcpu *vcpu);
|
|
|
|
|
|
2018-10-07 23:30:55 -06:00
|
|
|
int __kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu);
|
|
|
|
|
|
2018-10-07 23:31:06 -06:00
|
|
|
long kvmppc_h_set_dabr(struct kvm_vcpu *vcpu, unsigned long dabr);
|
|
|
|
|
long kvmppc_h_set_xdabr(struct kvm_vcpu *vcpu, unsigned long dabr,
|
|
|
|
|
unsigned long dabrx);
|
|
|
|
|
|
2016-05-17 19:16:50 -06:00
|
|
|
#endif /* _ASM_POWERPC_ASM_PROTOTYPES_H */
|