2019-06-02 23:44:50 -06:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2015-10-21 02:57:10 -06:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2015 - ARM Ltd
|
|
|
|
* Author: Marc Zyngier <marc.zyngier@arm.com>
|
|
|
|
*/
|
|
|
|
|
2018-05-29 06:11:16 -06:00
|
|
|
#include <linux/arm-smccc.h>
|
2019-05-22 11:47:04 -06:00
|
|
|
#include <linux/kvm_host.h>
|
2015-10-28 09:06:47 -06:00
|
|
|
#include <linux/types.h>
|
2016-09-12 08:49:15 -06:00
|
|
|
#include <linux/jump_label.h>
|
2018-01-03 09:38:37 -07:00
|
|
|
#include <uapi/linux/psci.h>
|
2016-09-12 08:49:15 -06:00
|
|
|
|
2018-02-06 10:56:13 -07:00
|
|
|
#include <kvm/arm_psci.h>
|
|
|
|
|
2019-01-31 07:58:48 -07:00
|
|
|
#include <asm/arch_gicv3.h>
|
2018-04-20 09:20:43 -06:00
|
|
|
#include <asm/cpufeature.h>
|
2019-01-24 09:32:54 -07:00
|
|
|
#include <asm/kprobes.h>
|
2015-01-29 08:47:55 -07:00
|
|
|
#include <asm/kvm_asm.h>
|
arm64: KVM: vgic-v2: Add the GICV emulation infrastructure
In order to efficiently perform the GICV access on behalf of the
guest, we need to be able to avoid going back all the way to
the host kernel.
For this, we introduce a new hook in the world switch code,
conveniently placed just after populating the fault info.
At that point, we only have saved/restored the GP registers,
and we can quickly perform all the required checks (data abort,
translation fault, valid faulting syndrome, not an external
abort, not a PTW).
Coming back from the emulation code, we need to skip the emulated
instruction. This involves an additional bit of save/restore in
order to be able to access the guest's PC (and possibly CPSR if
this is a 32bit guest).
At this stage, no emulation code is provided.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2016-09-06 02:28:45 -06:00
|
|
|
#include <asm/kvm_emulate.h>
|
2018-04-06 07:55:59 -06:00
|
|
|
#include <asm/kvm_host.h>
|
2016-01-28 06:44:07 -07:00
|
|
|
#include <asm/kvm_hyp.h>
|
2017-10-23 10:11:14 -06:00
|
|
|
#include <asm/kvm_mmu.h>
|
2016-11-08 06:56:21 -07:00
|
|
|
#include <asm/fpsimd.h>
|
2017-11-23 05:11:34 -07:00
|
|
|
#include <asm/debug-monitors.h>
|
2018-04-20 09:20:43 -06:00
|
|
|
#include <asm/processor.h>
|
2018-04-06 07:55:59 -06:00
|
|
|
#include <asm/thread_info.h>
|
2015-10-21 02:57:10 -06:00
|
|
|
|
2018-04-06 07:55:59 -06:00
|
|
|
/* Check whether the FP regs were dirtied while in the host-side run loop: */
|
|
|
|
static bool __hyp_text update_fp_enabled(struct kvm_vcpu *vcpu)
|
2015-10-28 08:15:45 -06:00
|
|
|
{
|
2018-04-06 07:55:59 -06:00
|
|
|
if (vcpu->arch.host_thread_info->flags & _TIF_FOREIGN_FPSTATE)
|
|
|
|
vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |
|
|
|
|
KVM_ARM64_FP_HOST);
|
2015-10-28 08:15:45 -06:00
|
|
|
|
2018-04-06 07:55:59 -06:00
|
|
|
return !!(vcpu->arch.flags & KVM_ARM64_FP_ENABLED);
|
2015-10-28 08:15:45 -06:00
|
|
|
}
|
|
|
|
|
2017-12-27 14:12:12 -07:00
|
|
|
/* Save the 32-bit only FPSIMD system register state */
|
|
|
|
static void __hyp_text __fpsimd_save_fpexc32(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
if (!vcpu_el1_is_32bit(vcpu))
|
|
|
|
return;
|
|
|
|
|
|
|
|
vcpu->arch.ctxt.sys_regs[FPEXC32_EL2] = read_sysreg(fpexc32_el2);
|
|
|
|
}
|
|
|
|
|
2017-08-04 00:50:25 -06:00
|
|
|
static void __hyp_text __activate_traps_fpsimd32(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We are about to set CPTR_EL2.TFP to trap all floating point
|
|
|
|
* register accesses to EL2, however, the ARM ARM clearly states that
|
|
|
|
* traps are only taken to EL2 if the operation would not otherwise
|
|
|
|
* trap to EL1. Therefore, always make sure that for 32-bit guests,
|
|
|
|
* we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
|
|
|
|
* If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to
|
|
|
|
* it will cause an exception.
|
|
|
|
*/
|
|
|
|
if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd()) {
|
|
|
|
write_sysreg(1 << 30, fpexc32_el2);
|
|
|
|
isb();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __hyp_text __activate_traps_common(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
/* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */
|
|
|
|
write_sysreg(1 << 15, hstr_el2);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Make sure we trap PMU access from EL0 to EL2. Also sanitize
|
|
|
|
* PMSELR_EL0 to make sure it never contains the cycle
|
|
|
|
* counter, which could make a PMXEVCNTR_EL0 access UNDEF at
|
|
|
|
* EL1 instead of being trapped to EL2.
|
|
|
|
*/
|
|
|
|
write_sysreg(0, pmselr_el0);
|
|
|
|
write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
|
|
|
|
write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __hyp_text __deactivate_traps_common(void)
|
|
|
|
{
|
|
|
|
write_sysreg(0, hstr_el2);
|
|
|
|
write_sysreg(0, pmuserenr_el0);
|
|
|
|
}
|
|
|
|
|
2017-10-03 09:06:15 -06:00
|
|
|
static void activate_traps_vhe(struct kvm_vcpu *vcpu)
|
2015-01-29 08:47:55 -07:00
|
|
|
{
|
|
|
|
u64 val;
|
|
|
|
|
|
|
|
val = read_sysreg(cpacr_el1);
|
|
|
|
val |= CPACR_EL1_TTA;
|
2018-04-06 07:55:59 -06:00
|
|
|
val &= ~CPACR_EL1_ZEN;
|
2018-09-28 07:39:17 -06:00
|
|
|
if (update_fp_enabled(vcpu)) {
|
|
|
|
if (vcpu_has_sve(vcpu))
|
|
|
|
val |= CPACR_EL1_ZEN;
|
|
|
|
} else {
|
2018-04-06 07:55:59 -06:00
|
|
|
val &= ~CPACR_EL1_FPEN;
|
2018-08-23 04:51:43 -06:00
|
|
|
__activate_traps_fpsimd32(vcpu);
|
|
|
|
}
|
2018-04-06 07:55:59 -06:00
|
|
|
|
2015-01-29 08:47:55 -07:00
|
|
|
write_sysreg(val, cpacr_el1);
|
|
|
|
|
2018-01-03 09:38:35 -07:00
|
|
|
write_sysreg(kvm_get_hyp_vector(), vbar_el1);
|
2015-01-29 08:47:55 -07:00
|
|
|
}
|
2019-01-24 09:32:54 -07:00
|
|
|
NOKPROBE_SYMBOL(activate_traps_vhe);
|
2015-01-29 08:47:55 -07:00
|
|
|
|
2017-08-04 00:50:25 -06:00
|
|
|
static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu)
|
2015-01-29 08:47:55 -07:00
|
|
|
{
|
|
|
|
u64 val;
|
|
|
|
|
2017-08-04 05:47:18 -06:00
|
|
|
__activate_traps_common(vcpu);
|
|
|
|
|
2015-01-29 08:47:55 -07:00
|
|
|
val = CPTR_EL2_DEFAULT;
|
2018-04-06 07:55:59 -06:00
|
|
|
val |= CPTR_EL2_TTA | CPTR_EL2_TZ;
|
2018-08-23 04:51:43 -06:00
|
|
|
if (!update_fp_enabled(vcpu)) {
|
2018-04-06 07:55:59 -06:00
|
|
|
val |= CPTR_EL2_TFP;
|
2018-08-23 04:51:43 -06:00
|
|
|
__activate_traps_fpsimd32(vcpu);
|
|
|
|
}
|
2018-04-06 07:55:59 -06:00
|
|
|
|
2015-01-29 08:47:55 -07:00
|
|
|
write_sysreg(val, cptr_el2);
|
|
|
|
}
|
|
|
|
|
2015-10-21 02:57:10 -06:00
|
|
|
static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
2017-12-13 14:56:48 -07:00
|
|
|
u64 hcr = vcpu->arch.hcr_el2;
|
2015-10-21 02:57:10 -06:00
|
|
|
|
2017-08-04 00:50:25 -06:00
|
|
|
write_sysreg(hcr, hcr_el2);
|
arm64: KVM: Hide unsupported AArch64 CPU features from guests
Currently, a guest kernel sees the true CPU feature registers
(ID_*_EL1) when it reads them using MRS instructions. This means
that the guest may observe features that are present in the
hardware but the host doesn't understand or doesn't provide support
for. A guest may legimitately try to use such a feature as per the
architecture, but use of the feature may trap instead of working
normally, triggering undef injection into the guest.
This is not a problem for the host, but the guest may go wrong when
running on newer hardware than the host knows about.
This patch hides from guest VMs any AArch64-specific CPU features
that the host doesn't support, by exposing to the guest the
sanitised versions of the registers computed by the cpufeatures
framework, instead of the true hardware registers. To achieve
this, HCR_EL2.TID3 is now set for AArch64 guests, and emulation
code is added to KVM to report the sanitised versions of the
affected registers in response to MRS and register reads from
userspace.
The affected registers are removed from invariant_sys_regs[] (since
the invariant_sys_regs handling is no longer quite correct for
them) and added to sys_reg_desgs[], with appropriate access(),
get_user() and set_user() methods. No runtime vcpu storage is
allocated for the registers: instead, they are read on demand from
the cpufeatures framework. This may need modification in the
future if there is a need for userspace to customise the features
visible to the guest.
Attempts by userspace to write the registers are handled similarly
to the current invariant_sys_regs handling: writes are permitted,
but only if they don't attempt to change the value. This is
sufficient to support VM snapshot/restore from userspace.
Because of the additional registers, restoring a VM on an older
kernel may not work unless userspace knows how to handle the extra
VM registers exposed to the KVM user ABI by this patch.
Under the principle of least damage, this patch makes no attempt to
handle any of the other registers currently in
invariant_sys_regs[], or to emulate registers for AArch32: however,
these could be handled in a similar way in future, as necessary.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 09:50:56 -06:00
|
|
|
|
2017-12-13 14:56:48 -07:00
|
|
|
if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
|
2018-01-15 12:39:01 -07:00
|
|
|
write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);
|
|
|
|
|
2017-10-03 09:06:15 -06:00
|
|
|
if (has_vhe())
|
|
|
|
activate_traps_vhe(vcpu);
|
|
|
|
else
|
|
|
|
__activate_traps_nvhe(vcpu);
|
2015-01-29 08:47:55 -07:00
|
|
|
}
|
2016-01-19 09:20:18 -07:00
|
|
|
|
2017-10-03 09:06:15 -06:00
|
|
|
static void deactivate_traps_vhe(void)
|
2015-01-29 08:47:55 -07:00
|
|
|
{
|
|
|
|
extern char vectors[]; /* kernel exception vectors */
|
|
|
|
write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
|
2018-12-06 10:31:24 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* ARM erratum 1165522 requires the actual execution of the above
|
|
|
|
* before we can switch to the EL2/EL0 translation regime used by
|
|
|
|
* the host.
|
|
|
|
*/
|
|
|
|
asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_1165522));
|
|
|
|
|
arm64/sve: KVM: Prevent guests from using SVE
Until KVM has full SVE support, guests must not be allowed to
execute SVE instructions.
This patch enables the necessary traps, and also ensures that the
traps are disabled again on exit from the guest so that the host
can still use SVE if it wants to.
On guest exit, high bits of the SVE Zn registers may have been
clobbered as a side-effect the execution of FPSIMD instructions in
the guest. The existing KVM host FPSIMD restore code is not
sufficient to restore these bits, so this patch explicitly marks
the CPU as not containing cached vector state for any task, thus
forcing a reload on the next return to userspace. This is an
interim measure, in advance of adding full SVE awareness to KVM.
This marking of cached vector state in the CPU as invalid is done
using __this_cpu_write(fpsimd_last_state, NULL) in fpsimd.c. Due
to the repeated use of this rather obscure operation, it makes
sense to factor it out as a separate helper with a clearer name.
This patch factors it out as fpsimd_flush_cpu_state(), and ports
all callers to use it.
As a side effect of this refactoring, a this_cpu_write() in
fpsimd_cpu_pm_notifier() is changed to __this_cpu_write(). This
should be fine, since cpu_pm_enter() is supposed to be called only
with interrupts disabled.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 09:51:16 -06:00
|
|
|
write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
|
2015-01-29 08:47:55 -07:00
|
|
|
write_sysreg(vectors, vbar_el1);
|
2015-10-21 02:57:10 -06:00
|
|
|
}
|
2019-01-24 09:32:54 -07:00
|
|
|
NOKPROBE_SYMBOL(deactivate_traps_vhe);
|
2015-10-21 02:57:10 -06:00
|
|
|
|
2015-01-29 08:47:55 -07:00
|
|
|
static void __hyp_text __deactivate_traps_nvhe(void)
|
2015-10-21 02:57:10 -06:00
|
|
|
{
|
2016-09-22 04:35:43 -06:00
|
|
|
u64 mdcr_el2 = read_sysreg(mdcr_el2);
|
|
|
|
|
2017-08-04 05:47:18 -06:00
|
|
|
__deactivate_traps_common();
|
|
|
|
|
2016-09-22 04:35:43 -06:00
|
|
|
mdcr_el2 &= MDCR_EL2_HPMN_MASK;
|
|
|
|
mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;
|
|
|
|
|
|
|
|
write_sysreg(mdcr_el2, mdcr_el2);
|
2018-12-07 11:39:21 -07:00
|
|
|
write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2);
|
2015-01-29 08:47:55 -07:00
|
|
|
write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
2016-09-06 07:02:00 -06:00
|
|
|
/*
|
|
|
|
* If we pended a virtual abort, preserve it until it gets
|
|
|
|
* cleared. See D1.14.3 (Virtual Interrupts) for details, but
|
|
|
|
* the crucial bit is "On taking a vSError interrupt,
|
|
|
|
* HCR_EL2.VSE is cleared to 0."
|
|
|
|
*/
|
|
|
|
if (vcpu->arch.hcr_el2 & HCR_VSE)
|
|
|
|
vcpu->arch.hcr_el2 = read_sysreg(hcr_el2);
|
|
|
|
|
2017-10-03 09:06:15 -06:00
|
|
|
if (has_vhe())
|
|
|
|
deactivate_traps_vhe();
|
|
|
|
else
|
|
|
|
__deactivate_traps_nvhe();
|
2015-10-21 02:57:10 -06:00
|
|
|
}
|
|
|
|
|
2017-08-04 05:47:18 -06:00
|
|
|
void activate_traps_vhe_load(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
__activate_traps_common(vcpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
void deactivate_traps_vhe_put(void)
|
|
|
|
{
|
|
|
|
u64 mdcr_el2 = read_sysreg(mdcr_el2);
|
|
|
|
|
|
|
|
mdcr_el2 &= MDCR_EL2_HPMN_MASK |
|
|
|
|
MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT |
|
|
|
|
MDCR_EL2_TPMS;
|
|
|
|
|
|
|
|
write_sysreg(mdcr_el2, mdcr_el2);
|
|
|
|
|
|
|
|
__deactivate_traps_common();
|
|
|
|
}
|
|
|
|
|
2017-10-10 05:25:21 -06:00
|
|
|
static void __hyp_text __activate_vm(struct kvm *kvm)
|
2015-10-21 02:57:10 -06:00
|
|
|
{
|
2018-09-26 10:32:39 -06:00
|
|
|
__load_guest_stage2(kvm);
|
2015-10-21 02:57:10 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
write_sysreg(0, vttbr_el2);
|
|
|
|
}
|
|
|
|
|
2017-10-04 15:42:32 -06:00
|
|
|
/* Save VGICv3 state on non-VHE systems */
|
|
|
|
static void __hyp_text __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
|
2015-10-21 02:57:10 -06:00
|
|
|
{
|
2017-10-05 09:19:19 -06:00
|
|
|
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
|
2016-09-12 08:49:15 -06:00
|
|
|
__vgic_v3_save_state(vcpu);
|
2017-10-05 09:19:19 -06:00
|
|
|
__vgic_v3_deactivate_traps(vcpu);
|
|
|
|
}
|
2015-10-21 02:57:10 -06:00
|
|
|
}
|
|
|
|
|
2017-10-04 15:42:32 -06:00
|
|
|
/* Restore VGICv3 state on non_VEH systems */
|
|
|
|
static void __hyp_text __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)
|
2015-10-21 02:57:10 -06:00
|
|
|
{
|
2017-10-05 09:19:19 -06:00
|
|
|
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
|
|
|
|
__vgic_v3_activate_traps(vcpu);
|
2016-09-12 08:49:15 -06:00
|
|
|
__vgic_v3_restore_state(vcpu);
|
2017-10-05 09:19:19 -06:00
|
|
|
}
|
2015-10-21 02:57:10 -06:00
|
|
|
}
|
|
|
|
|
2015-10-28 09:06:47 -06:00
|
|
|
static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar)
|
|
|
|
{
|
|
|
|
u64 par, tmp;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Resolve the IPA the hard way using the guest VA.
|
|
|
|
*
|
|
|
|
* Stage-1 translation already validated the memory access
|
|
|
|
* rights. As such, we can use the EL1 translation regime, and
|
|
|
|
* don't have to distinguish between EL0 and EL1 access.
|
|
|
|
*
|
|
|
|
* We do need to save/restore PAR_EL1 though, as we haven't
|
|
|
|
* saved the guest context yet, and we may return early...
|
|
|
|
*/
|
|
|
|
par = read_sysreg(par_el1);
|
|
|
|
asm volatile("at s1e1r, %0" : : "r" (far));
|
|
|
|
isb();
|
|
|
|
|
|
|
|
tmp = read_sysreg(par_el1);
|
|
|
|
write_sysreg(par, par_el1);
|
|
|
|
|
2019-08-22 10:21:21 -06:00
|
|
|
if (unlikely(tmp & SYS_PAR_EL1_F))
|
2015-10-28 09:06:47 -06:00
|
|
|
return false; /* Translation failed, back to guest */
|
|
|
|
|
|
|
|
/* Convert PAR to HPFAR format */
|
2018-09-26 10:32:51 -06:00
|
|
|
*hpfar = PAR_TO_HPFAR(tmp);
|
2015-10-28 09:06:47 -06:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
2018-01-15 12:39:03 -07:00
|
|
|
u8 ec;
|
|
|
|
u64 esr;
|
2015-10-28 09:06:47 -06:00
|
|
|
u64 hpfar, far;
|
|
|
|
|
2018-01-15 12:39:03 -07:00
|
|
|
esr = vcpu->arch.fault.esr_el2;
|
|
|
|
ec = ESR_ELx_EC(esr);
|
2015-10-28 09:06:47 -06:00
|
|
|
|
|
|
|
if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
|
|
|
|
return true;
|
|
|
|
|
KVM: arm64: Migrate _elx sysreg accessors to msr_s/mrs_s
Currently, the {read,write}_sysreg_el*() accessors for accessing
particular ELs' sysregs in the presence of VHE rely on some local
hacks and define their system register encodings in a way that is
inconsistent with the core definitions in <asm/sysreg.h>.
As a result, it is necessary to add duplicate definitions for any
system register that already needs a definition in sysreg.h for
other reasons.
This is a bit of a maintenance headache, and the reasons for the
_el*() accessors working the way they do is a bit historical.
This patch gets rid of the shadow sysreg definitions in
<asm/kvm_hyp.h>, converts the _el*() accessors to use the core
__msr_s/__mrs_s interface, and converts all call sites to use the
standard sysreg #define names (i.e., upper case, with SYS_ prefix).
This patch will conflict heavily anyway, so the opportunity
to clean up some bad whitespace in the context of the changes is
taken.
The change exposes a few system registers that have no sysreg.h
definition, due to msr_s/mrs_s being used in place of msr/mrs:
additions are made in order to fill in the gaps.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christoffer Dall <christoffer.dall@arm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Link: https://www.spinics.net/lists/kvm-arm/msg31717.html
[Rebased to v4.21-rc1]
Signed-off-by: Sudeep Holla <sudeep.holla@arm.com>
[Rebased to v5.2-rc5, changelog updates]
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2019-04-06 04:29:40 -06:00
|
|
|
far = read_sysreg_el2(SYS_FAR);
|
2015-10-28 09:06:47 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The HPFAR can be invalid if the stage 2 fault did not
|
|
|
|
* happen during a stage 1 page table walk (the ESR_EL2.S1PTW
|
|
|
|
* bit is clear) and one of the two following cases are true:
|
|
|
|
* 1. The fault was due to a permission fault
|
|
|
|
* 2. The processor carries errata 834220
|
|
|
|
*
|
|
|
|
* Therefore, for all non S1PTW faults where we either have a
|
|
|
|
* permission fault or the errata workaround is enabled, we
|
|
|
|
* resolve the IPA using the AT instruction.
|
|
|
|
*/
|
|
|
|
if (!(esr & ESR_ELx_S1PTW) &&
|
2019-09-01 15:12:35 -06:00
|
|
|
(cpus_have_const_cap(ARM64_WORKAROUND_834220) ||
|
|
|
|
(esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) {
|
2015-10-28 09:06:47 -06:00
|
|
|
if (!__translate_far_to_hpfar(far, &hpfar))
|
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
hpfar = read_sysreg(hpfar_el2);
|
|
|
|
}
|
|
|
|
|
|
|
|
vcpu->arch.fault.far_el2 = far;
|
|
|
|
vcpu->arch.fault.hpfar_el2 = hpfar;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2018-09-28 07:39:17 -06:00
|
|
|
/* Check for an FPSIMD/SVE trap and handle as appropriate */
|
|
|
|
static bool __hyp_text __hyp_handle_fpsimd(struct kvm_vcpu *vcpu)
|
2018-02-16 09:35:32 -07:00
|
|
|
{
|
2018-09-28 07:39:17 -06:00
|
|
|
bool vhe, sve_guest, sve_host;
|
|
|
|
u8 hsr_ec;
|
2018-04-20 09:20:43 -06:00
|
|
|
|
2018-09-28 07:39:17 -06:00
|
|
|
if (!system_supports_fpsimd())
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (system_supports_sve()) {
|
|
|
|
sve_guest = vcpu_has_sve(vcpu);
|
|
|
|
sve_host = vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE;
|
|
|
|
vhe = true;
|
|
|
|
} else {
|
|
|
|
sve_guest = false;
|
|
|
|
sve_host = false;
|
|
|
|
vhe = has_vhe();
|
|
|
|
}
|
|
|
|
|
|
|
|
hsr_ec = kvm_vcpu_trap_get_class(vcpu);
|
|
|
|
if (hsr_ec != ESR_ELx_EC_FP_ASIMD &&
|
|
|
|
hsr_ec != ESR_ELx_EC_SVE)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* Don't handle SVE traps for non-SVE vcpus here: */
|
|
|
|
if (!sve_guest)
|
|
|
|
if (hsr_ec != ESR_ELx_EC_FP_ASIMD)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* Valid trap. Switch the context: */
|
|
|
|
|
|
|
|
if (vhe) {
|
|
|
|
u64 reg = read_sysreg(cpacr_el1) | CPACR_EL1_FPEN;
|
|
|
|
|
|
|
|
if (sve_guest)
|
|
|
|
reg |= CPACR_EL1_ZEN;
|
|
|
|
|
|
|
|
write_sysreg(reg, cpacr_el1);
|
|
|
|
} else {
|
2018-02-16 09:35:32 -07:00
|
|
|
write_sysreg(read_sysreg(cptr_el2) & ~(u64)CPTR_EL2_TFP,
|
|
|
|
cptr_el2);
|
2018-09-28 07:39:17 -06:00
|
|
|
}
|
2018-02-16 09:35:32 -07:00
|
|
|
|
|
|
|
isb();
|
|
|
|
|
2018-04-06 07:55:59 -06:00
|
|
|
if (vcpu->arch.flags & KVM_ARM64_FP_HOST) {
|
2018-04-20 09:20:43 -06:00
|
|
|
/*
|
|
|
|
* In the SVE case, VHE is assumed: it is enforced by
|
|
|
|
* Kconfig and kvm_arch_init().
|
|
|
|
*/
|
2018-09-28 07:39:17 -06:00
|
|
|
if (sve_host) {
|
2018-04-20 09:20:43 -06:00
|
|
|
struct thread_struct *thread = container_of(
|
2018-09-28 07:39:17 -06:00
|
|
|
vcpu->arch.host_fpsimd_state,
|
2018-04-20 09:20:43 -06:00
|
|
|
struct thread_struct, uw.fpsimd_state);
|
|
|
|
|
2018-09-28 07:39:17 -06:00
|
|
|
sve_save_state(sve_pffr(thread),
|
|
|
|
&vcpu->arch.host_fpsimd_state->fpsr);
|
2018-04-20 09:20:43 -06:00
|
|
|
} else {
|
2018-09-28 07:39:17 -06:00
|
|
|
__fpsimd_save_state(vcpu->arch.host_fpsimd_state);
|
2018-04-20 09:20:43 -06:00
|
|
|
}
|
|
|
|
|
2018-04-06 07:55:59 -06:00
|
|
|
vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;
|
|
|
|
}
|
|
|
|
|
2018-09-28 07:39:17 -06:00
|
|
|
if (sve_guest) {
|
|
|
|
sve_load_state(vcpu_sve_pffr(vcpu),
|
|
|
|
&vcpu->arch.ctxt.gp_regs.fp_regs.fpsr,
|
|
|
|
sve_vq_from_vl(vcpu->arch.sve_max_vl) - 1);
|
2018-09-28 07:39:16 -06:00
|
|
|
write_sysreg_s(vcpu->arch.ctxt.sys_regs[ZCR_EL1], SYS_ZCR_EL12);
|
2018-09-28 07:39:17 -06:00
|
|
|
} else {
|
|
|
|
__fpsimd_restore_state(&vcpu->arch.ctxt.gp_regs.fp_regs);
|
|
|
|
}
|
2018-09-28 07:39:16 -06:00
|
|
|
|
2018-02-16 09:35:32 -07:00
|
|
|
/* Skip restoring fpexc32 for AArch64 guests */
|
|
|
|
if (!(read_sysreg(hcr_el2) & HCR_RW))
|
|
|
|
write_sysreg(vcpu->arch.ctxt.sys_regs[FPEXC32_EL2],
|
|
|
|
fpexc32_el2);
|
2018-04-06 07:55:59 -06:00
|
|
|
|
|
|
|
vcpu->arch.flags |= KVM_ARM64_FP_ENABLED;
|
2018-05-02 07:18:02 -06:00
|
|
|
|
|
|
|
return true;
|
2018-02-16 09:35:32 -07:00
|
|
|
}
|
|
|
|
|
2017-10-03 05:16:04 -06:00
|
|
|
/*
|
|
|
|
* Return true when we were able to fixup the guest exit and should return to
|
|
|
|
* the guest, false when we should restore the host state and return to the
|
|
|
|
* main run loop.
|
|
|
|
*/
|
|
|
|
static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
|
2015-10-21 02:57:10 -06:00
|
|
|
{
|
2017-10-03 05:16:04 -06:00
|
|
|
if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ)
|
KVM: arm64: Migrate _elx sysreg accessors to msr_s/mrs_s
Currently, the {read,write}_sysreg_el*() accessors for accessing
particular ELs' sysregs in the presence of VHE rely on some local
hacks and define their system register encodings in a way that is
inconsistent with the core definitions in <asm/sysreg.h>.
As a result, it is necessary to add duplicate definitions for any
system register that already needs a definition in sysreg.h for
other reasons.
This is a bit of a maintenance headache, and the reasons for the
_el*() accessors working the way they do is a bit historical.
This patch gets rid of the shadow sysreg definitions in
<asm/kvm_hyp.h>, converts the _el*() accessors to use the core
__msr_s/__mrs_s interface, and converts all call sites to use the
standard sysreg #define names (i.e., upper case, with SYS_ prefix).
This patch will conflict heavily anyway, so the opportunity
to clean up some bad whitespace in the context of the changes is
taken.
The change exposes a few system registers that have no sysreg.h
definition, due to msr_s/mrs_s being used in place of msr/mrs:
additions are made in order to fill in the gaps.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christoffer Dall <christoffer.dall@arm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Link: https://www.spinics.net/lists/kvm-arm/msg31717.html
[Rebased to v4.21-rc1]
Signed-off-by: Sudeep Holla <sudeep.holla@arm.com>
[Rebased to v5.2-rc5, changelog updates]
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2019-04-06 04:29:40 -06:00
|
|
|
vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR);
|
2017-10-03 05:16:04 -06:00
|
|
|
|
2016-09-06 07:02:07 -06:00
|
|
|
/*
|
|
|
|
* We're using the raw exception code in order to only process
|
|
|
|
* the trap if no SError is pending. We will come back to the
|
|
|
|
* same PC once the SError has been injected, and replay the
|
|
|
|
* trapping instruction.
|
|
|
|
*/
|
2018-05-02 06:36:48 -06:00
|
|
|
if (*exit_code != ARM_EXCEPTION_TRAP)
|
|
|
|
goto exit;
|
|
|
|
|
2018-05-02 07:18:02 -06:00
|
|
|
/*
|
|
|
|
* We trap the first access to the FP/SIMD to save the host context
|
|
|
|
* and restore the guest context lazily.
|
|
|
|
* If FP/SIMD is not implemented, handle the trap and inject an
|
|
|
|
* undefined instruction exception to the guest.
|
2018-09-28 07:39:17 -06:00
|
|
|
* Similarly for trapped SVE accesses.
|
2018-05-02 07:18:02 -06:00
|
|
|
*/
|
2018-09-28 07:39:17 -06:00
|
|
|
if (__hyp_handle_fpsimd(vcpu))
|
|
|
|
return true;
|
2018-05-02 07:18:02 -06:00
|
|
|
|
2018-05-02 06:36:48 -06:00
|
|
|
if (!__populate_fault_info(vcpu))
|
2017-10-03 05:16:04 -06:00
|
|
|
return true;
|
2015-10-28 09:06:47 -06:00
|
|
|
|
2018-05-02 06:36:48 -06:00
|
|
|
if (static_branch_unlikely(&vgic_v2_cpuif_trap)) {
|
arm64: KVM: vgic-v2: Add the GICV emulation infrastructure
In order to efficiently perform the GICV access on behalf of the
guest, we need to be able to avoid going back all the way to
the host kernel.
For this, we introduce a new hook in the world switch code,
conveniently placed just after populating the fault info.
At that point, we only have saved/restored the GP registers,
and we can quickly perform all the required checks (data abort,
translation fault, valid faulting syndrome, not an external
abort, not a PTW).
Coming back from the emulation code, we need to skip the emulated
instruction. This involves an additional bit of save/restore in
order to be able to access the guest's PC (and possibly CPSR if
this is a 32bit guest).
At this stage, no emulation code is provided.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2016-09-06 02:28:45 -06:00
|
|
|
bool valid;
|
|
|
|
|
|
|
|
valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&
|
|
|
|
kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&
|
|
|
|
kvm_vcpu_dabt_isvalid(vcpu) &&
|
|
|
|
!kvm_vcpu_dabt_isextabt(vcpu) &&
|
|
|
|
!kvm_vcpu_dabt_iss1tw(vcpu);
|
|
|
|
|
2016-09-06 07:02:17 -06:00
|
|
|
if (valid) {
|
|
|
|
int ret = __vgic_v2_perform_cpuif_access(vcpu);
|
|
|
|
|
2018-11-09 08:07:11 -07:00
|
|
|
if (ret == 1)
|
2018-05-02 06:23:07 -06:00
|
|
|
return true;
|
2016-09-06 07:02:17 -06:00
|
|
|
|
2018-11-09 08:07:11 -07:00
|
|
|
/* Promote an illegal access to an SError.*/
|
|
|
|
if (ret == -1)
|
2017-10-03 05:16:04 -06:00
|
|
|
*exit_code = ARM_EXCEPTION_EL1_SERROR;
|
2018-05-02 06:36:48 -06:00
|
|
|
|
|
|
|
goto exit;
|
arm64: KVM: vgic-v2: Add the GICV emulation infrastructure
In order to efficiently perform the GICV access on behalf of the
guest, we need to be able to avoid going back all the way to
the host kernel.
For this, we introduce a new hook in the world switch code,
conveniently placed just after populating the fault info.
At that point, we only have saved/restored the GP registers,
and we can quickly perform all the required checks (data abort,
translation fault, valid faulting syndrome, not an external
abort, not a PTW).
Coming back from the emulation code, we need to skip the emulated
instruction. This involves an additional bit of save/restore in
order to be able to access the guest's PC (and possibly CPSR if
this is a 32bit guest).
At this stage, no emulation code is provided.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2016-09-06 02:28:45 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-06-09 05:49:33 -06:00
|
|
|
if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
|
|
|
|
(kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 ||
|
|
|
|
kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) {
|
|
|
|
int ret = __vgic_v3_perform_cpuif_access(vcpu);
|
|
|
|
|
2018-11-09 08:07:11 -07:00
|
|
|
if (ret == 1)
|
2018-05-02 06:23:07 -06:00
|
|
|
return true;
|
2017-06-09 05:49:33 -06:00
|
|
|
}
|
|
|
|
|
2018-05-02 06:36:48 -06:00
|
|
|
exit:
|
2017-10-03 05:16:04 -06:00
|
|
|
/* Return to the host kernel and handle the exit */
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2018-05-29 06:11:16 -06:00
|
|
|
static inline bool __hyp_text __needs_ssbd_off(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
if (!cpus_have_const_cap(ARM64_SSBD))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return !(vcpu->arch.workaround_flags & VCPU_WORKAROUND_2_FLAG);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __hyp_text __set_guest_arch_workaround_state(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_ARM64_SSBD
|
|
|
|
/*
|
|
|
|
* The host runs with the workaround always present. If the
|
|
|
|
* guest wants it disabled, so be it...
|
|
|
|
*/
|
|
|
|
if (__needs_ssbd_off(vcpu) &&
|
|
|
|
__hyp_this_cpu_read(arm64_ssbd_callback_required))
|
|
|
|
arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, 0, NULL);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __hyp_text __set_host_arch_workaround_state(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_ARM64_SSBD
|
|
|
|
/*
|
|
|
|
* If the guest has disabled the workaround, bring it back on.
|
|
|
|
*/
|
|
|
|
if (__needs_ssbd_off(vcpu) &&
|
|
|
|
__hyp_this_cpu_read(arm64_ssbd_callback_required))
|
|
|
|
arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, 1, NULL);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2019-05-22 11:47:04 -06:00
|
|
|
/**
|
|
|
|
* Disable host events, enable guest events
|
|
|
|
*/
|
|
|
|
static bool __hyp_text __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt)
|
|
|
|
{
|
|
|
|
struct kvm_host_data *host;
|
|
|
|
struct kvm_pmu_events *pmu;
|
|
|
|
|
|
|
|
host = container_of(host_ctxt, struct kvm_host_data, host_ctxt);
|
|
|
|
pmu = &host->pmu_events;
|
|
|
|
|
|
|
|
if (pmu->events_host)
|
|
|
|
write_sysreg(pmu->events_host, pmcntenclr_el0);
|
|
|
|
|
|
|
|
if (pmu->events_guest)
|
|
|
|
write_sysreg(pmu->events_guest, pmcntenset_el0);
|
|
|
|
|
|
|
|
return (pmu->events_host || pmu->events_guest);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Disable guest events, enable host events
|
|
|
|
*/
|
|
|
|
static void __hyp_text __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt)
|
|
|
|
{
|
|
|
|
struct kvm_host_data *host;
|
|
|
|
struct kvm_pmu_events *pmu;
|
|
|
|
|
|
|
|
host = container_of(host_ctxt, struct kvm_host_data, host_ctxt);
|
|
|
|
pmu = &host->pmu_events;
|
|
|
|
|
|
|
|
if (pmu->events_guest)
|
|
|
|
write_sysreg(pmu->events_guest, pmcntenclr_el0);
|
|
|
|
|
|
|
|
if (pmu->events_host)
|
|
|
|
write_sysreg(pmu->events_host, pmcntenset_el0);
|
|
|
|
}
|
|
|
|
|
2017-10-03 06:02:12 -06:00
|
|
|
/* Switch to the guest for VHE systems running in EL2 */
|
|
|
|
int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
struct kvm_cpu_context *host_ctxt;
|
|
|
|
struct kvm_cpu_context *guest_ctxt;
|
|
|
|
u64 exit_code;
|
|
|
|
|
2016-12-22 16:20:38 -07:00
|
|
|
host_ctxt = vcpu->arch.host_cpu_context;
|
2017-10-03 06:02:12 -06:00
|
|
|
host_ctxt->__hyp_running_vcpu = vcpu;
|
|
|
|
guest_ctxt = &vcpu->arch.ctxt;
|
|
|
|
|
2017-10-10 14:19:31 -06:00
|
|
|
sysreg_save_host_state_vhe(host_ctxt);
|
2017-10-03 06:02:12 -06:00
|
|
|
|
2018-12-06 10:31:24 -07:00
|
|
|
/*
|
|
|
|
* ARM erratum 1165522 requires us to configure both stage 1 and
|
|
|
|
* stage 2 translation for the guest context before we clear
|
|
|
|
* HCR_EL2.TGE.
|
|
|
|
*
|
|
|
|
* We have already configured the guest's stage 1 translation in
|
|
|
|
* kvm_vcpu_load_sysregs above. We must now call __activate_vm
|
|
|
|
* before __activate_traps, because __activate_vm configures
|
|
|
|
* stage 2 translation, and __activate_traps clear HCR_EL2.TGE
|
|
|
|
* (among other things).
|
|
|
|
*/
|
2017-10-10 05:25:21 -06:00
|
|
|
__activate_vm(vcpu->kvm);
|
2018-12-06 10:31:21 -07:00
|
|
|
__activate_traps(vcpu);
|
2017-10-03 06:02:12 -06:00
|
|
|
|
2017-10-10 14:19:31 -06:00
|
|
|
sysreg_restore_guest_state_vhe(guest_ctxt);
|
2017-10-03 06:02:12 -06:00
|
|
|
__debug_switch_to_guest(vcpu);
|
|
|
|
|
2018-05-29 06:11:16 -06:00
|
|
|
__set_guest_arch_workaround_state(vcpu);
|
|
|
|
|
2017-10-03 06:02:12 -06:00
|
|
|
do {
|
|
|
|
/* Jump in the fire! */
|
|
|
|
exit_code = __guest_enter(vcpu, host_ctxt);
|
|
|
|
|
|
|
|
/* And we're baaack! */
|
|
|
|
} while (fixup_guest_exit(vcpu, &exit_code));
|
|
|
|
|
2018-05-29 06:11:16 -06:00
|
|
|
__set_host_arch_workaround_state(vcpu);
|
|
|
|
|
2017-10-10 14:19:31 -06:00
|
|
|
sysreg_save_guest_state_vhe(guest_ctxt);
|
2017-10-03 06:02:12 -06:00
|
|
|
|
|
|
|
__deactivate_traps(vcpu);
|
|
|
|
|
2017-10-10 14:19:31 -06:00
|
|
|
sysreg_restore_host_state_vhe(host_ctxt);
|
2017-10-03 06:02:12 -06:00
|
|
|
|
2018-04-06 07:55:59 -06:00
|
|
|
if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
|
2017-12-27 14:12:12 -07:00
|
|
|
__fpsimd_save_fpexc32(vcpu);
|
2017-10-03 06:02:12 -06:00
|
|
|
|
|
|
|
__debug_switch_to_host(vcpu);
|
|
|
|
|
|
|
|
return exit_code;
|
|
|
|
}
|
2019-01-24 09:32:54 -07:00
|
|
|
NOKPROBE_SYMBOL(kvm_vcpu_run_vhe);
|
2017-10-03 06:02:12 -06:00
|
|
|
|
|
|
|
/* Switch to the guest for legacy non-VHE systems */
|
|
|
|
int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
|
2017-10-03 05:16:04 -06:00
|
|
|
{
|
|
|
|
struct kvm_cpu_context *host_ctxt;
|
|
|
|
struct kvm_cpu_context *guest_ctxt;
|
2019-04-09 13:22:14 -06:00
|
|
|
bool pmu_switch_needed;
|
2017-10-03 05:16:04 -06:00
|
|
|
u64 exit_code;
|
|
|
|
|
2019-01-31 07:58:48 -07:00
|
|
|
/*
|
|
|
|
* Having IRQs masked via PMR when entering the guest means the GIC
|
|
|
|
* will not signal the CPU of interrupts of lower priority, and the
|
|
|
|
* only way to get out will be via guest exceptions.
|
|
|
|
* Naturally, we want to avoid this.
|
|
|
|
*/
|
|
|
|
if (system_uses_irq_prio_masking()) {
|
2019-06-11 03:38:10 -06:00
|
|
|
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
|
2019-01-31 07:58:48 -07:00
|
|
|
dsb(sy);
|
|
|
|
}
|
|
|
|
|
2017-10-03 05:16:04 -06:00
|
|
|
vcpu = kern_hyp_va(vcpu);
|
|
|
|
|
|
|
|
host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
|
|
|
|
host_ctxt->__hyp_running_vcpu = vcpu;
|
|
|
|
guest_ctxt = &vcpu->arch.ctxt;
|
|
|
|
|
2019-04-09 13:22:14 -06:00
|
|
|
pmu_switch_needed = __pmu_switch_to_guest(host_ctxt);
|
|
|
|
|
2017-10-10 14:40:13 -06:00
|
|
|
__sysreg_save_state_nvhe(host_ctxt);
|
2017-10-03 05:16:04 -06:00
|
|
|
|
2017-10-10 05:25:21 -06:00
|
|
|
__activate_vm(kern_hyp_va(vcpu->kvm));
|
2018-12-06 10:31:21 -07:00
|
|
|
__activate_traps(vcpu);
|
2017-10-03 05:16:04 -06:00
|
|
|
|
2017-10-04 15:42:32 -06:00
|
|
|
__hyp_vgic_restore_state(vcpu);
|
2017-10-03 05:16:04 -06:00
|
|
|
__timer_enable_traps(vcpu);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We must restore the 32-bit state before the sysregs, thanks
|
|
|
|
* to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
|
|
|
|
*/
|
|
|
|
__sysreg32_restore_state(vcpu);
|
2017-10-10 14:40:13 -06:00
|
|
|
__sysreg_restore_state_nvhe(guest_ctxt);
|
2017-10-03 05:16:04 -06:00
|
|
|
__debug_switch_to_guest(vcpu);
|
|
|
|
|
2018-05-29 06:11:16 -06:00
|
|
|
__set_guest_arch_workaround_state(vcpu);
|
|
|
|
|
2017-10-03 05:16:04 -06:00
|
|
|
do {
|
|
|
|
/* Jump in the fire! */
|
|
|
|
exit_code = __guest_enter(vcpu, host_ctxt);
|
|
|
|
|
|
|
|
/* And we're baaack! */
|
|
|
|
} while (fixup_guest_exit(vcpu, &exit_code));
|
|
|
|
|
2018-05-29 06:11:16 -06:00
|
|
|
__set_host_arch_workaround_state(vcpu);
|
|
|
|
|
2017-10-10 14:40:13 -06:00
|
|
|
__sysreg_save_state_nvhe(guest_ctxt);
|
2015-10-21 02:57:10 -06:00
|
|
|
__sysreg32_save_state(vcpu);
|
2017-01-04 08:10:28 -07:00
|
|
|
__timer_disable_traps(vcpu);
|
2017-10-04 15:42:32 -06:00
|
|
|
__hyp_vgic_save_state(vcpu);
|
2015-10-21 02:57:10 -06:00
|
|
|
|
|
|
|
__deactivate_traps(vcpu);
|
|
|
|
__deactivate_vm(vcpu);
|
|
|
|
|
2017-10-10 14:40:13 -06:00
|
|
|
__sysreg_restore_state_nvhe(host_ctxt);
|
2015-10-21 02:57:10 -06:00
|
|
|
|
2018-04-06 07:55:59 -06:00
|
|
|
if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
|
2017-12-27 14:12:12 -07:00
|
|
|
__fpsimd_save_fpexc32(vcpu);
|
2015-10-26 02:34:09 -06:00
|
|
|
|
2016-09-22 04:35:43 -06:00
|
|
|
/*
|
|
|
|
* This must come after restoring the host sysregs, since a non-VHE
|
|
|
|
* system may enable SPE here and make use of the TTBRs.
|
|
|
|
*/
|
2017-10-10 12:10:08 -06:00
|
|
|
__debug_switch_to_host(vcpu);
|
2015-10-21 02:57:10 -06:00
|
|
|
|
2019-04-09 13:22:14 -06:00
|
|
|
if (pmu_switch_needed)
|
|
|
|
__pmu_switch_to_host(host_ctxt);
|
|
|
|
|
2019-01-31 07:58:48 -07:00
|
|
|
/* Returning to host will clear PSR.I, remask PMR if needed */
|
|
|
|
if (system_uses_irq_prio_masking())
|
|
|
|
gic_write_pmr(GIC_PRIO_IRQOFF);
|
|
|
|
|
2015-10-21 02:57:10 -06:00
|
|
|
return exit_code;
|
|
|
|
}
|
2015-10-25 09:21:52 -06:00
|
|
|
|
|
|
|
static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";
|
|
|
|
|
2018-01-08 08:38:05 -07:00
|
|
|
static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par,
|
2017-10-09 13:43:50 -06:00
|
|
|
struct kvm_cpu_context *__host_ctxt)
|
2015-10-25 09:21:52 -06:00
|
|
|
{
|
2017-10-09 13:43:50 -06:00
|
|
|
struct kvm_vcpu *vcpu;
|
2016-06-30 11:40:35 -06:00
|
|
|
unsigned long str_va;
|
2015-11-17 07:07:45 -07:00
|
|
|
|
2017-10-09 13:43:50 -06:00
|
|
|
vcpu = __host_ctxt->__hyp_running_vcpu;
|
|
|
|
|
|
|
|
if (read_sysreg(vttbr_el2)) {
|
|
|
|
__timer_disable_traps(vcpu);
|
|
|
|
__deactivate_traps(vcpu);
|
|
|
|
__deactivate_vm(vcpu);
|
2017-10-10 14:40:13 -06:00
|
|
|
__sysreg_restore_state_nvhe(__host_ctxt);
|
2017-10-09 13:43:50 -06:00
|
|
|
}
|
|
|
|
|
2016-06-30 11:40:35 -06:00
|
|
|
/*
|
|
|
|
* Force the panic string to be loaded from the literal pool,
|
|
|
|
* making sure it is a kernel address and not a PC-relative
|
|
|
|
* reference.
|
|
|
|
*/
|
|
|
|
asm volatile("ldr %0, =__hyp_panic_string" : "=r" (str_va));
|
|
|
|
|
|
|
|
__hyp_do_panic(str_va,
|
KVM: arm64: Migrate _elx sysreg accessors to msr_s/mrs_s
Currently, the {read,write}_sysreg_el*() accessors for accessing
particular ELs' sysregs in the presence of VHE rely on some local
hacks and define their system register encodings in a way that is
inconsistent with the core definitions in <asm/sysreg.h>.
As a result, it is necessary to add duplicate definitions for any
system register that already needs a definition in sysreg.h for
other reasons.
This is a bit of a maintenance headache, and the reasons for the
_el*() accessors working the way they do is a bit historical.
This patch gets rid of the shadow sysreg definitions in
<asm/kvm_hyp.h>, converts the _el*() accessors to use the core
__msr_s/__mrs_s interface, and converts all call sites to use the
standard sysreg #define names (i.e., upper case, with SYS_ prefix).
This patch will conflict heavily anyway, so the opportunity
to clean up some bad whitespace in the context of the changes is
taken.
The change exposes a few system registers that have no sysreg.h
definition, due to msr_s/mrs_s being used in place of msr/mrs:
additions are made in order to fill in the gaps.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christoffer Dall <christoffer.dall@arm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Link: https://www.spinics.net/lists/kvm-arm/msg31717.html
[Rebased to v4.21-rc1]
Signed-off-by: Sudeep Holla <sudeep.holla@arm.com>
[Rebased to v5.2-rc5, changelog updates]
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2019-04-06 04:29:40 -06:00
|
|
|
spsr, elr,
|
|
|
|
read_sysreg(esr_el2), read_sysreg_el2(SYS_FAR),
|
2018-01-08 08:38:05 -07:00
|
|
|
read_sysreg(hpfar_el2), par, vcpu);
|
2015-11-17 07:07:45 -07:00
|
|
|
}
|
|
|
|
|
2017-10-09 13:43:50 -06:00
|
|
|
static void __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par,
|
|
|
|
struct kvm_cpu_context *host_ctxt)
|
2015-11-17 07:07:45 -07:00
|
|
|
{
|
2017-10-09 13:43:50 -06:00
|
|
|
struct kvm_vcpu *vcpu;
|
|
|
|
vcpu = host_ctxt->__hyp_running_vcpu;
|
|
|
|
|
|
|
|
__deactivate_traps(vcpu);
|
2017-10-10 14:19:31 -06:00
|
|
|
sysreg_restore_host_state_vhe(host_ctxt);
|
2017-10-09 13:43:50 -06:00
|
|
|
|
2015-11-17 07:07:45 -07:00
|
|
|
panic(__hyp_panic_string,
|
|
|
|
spsr, elr,
|
KVM: arm64: Migrate _elx sysreg accessors to msr_s/mrs_s
Currently, the {read,write}_sysreg_el*() accessors for accessing
particular ELs' sysregs in the presence of VHE rely on some local
hacks and define their system register encodings in a way that is
inconsistent with the core definitions in <asm/sysreg.h>.
As a result, it is necessary to add duplicate definitions for any
system register that already needs a definition in sysreg.h for
other reasons.
This is a bit of a maintenance headache, and the reasons for the
_el*() accessors working the way they do is a bit historical.
This patch gets rid of the shadow sysreg definitions in
<asm/kvm_hyp.h>, converts the _el*() accessors to use the core
__msr_s/__mrs_s interface, and converts all call sites to use the
standard sysreg #define names (i.e., upper case, with SYS_ prefix).
This patch will conflict heavily anyway, so the opportunity
to clean up some bad whitespace in the context of the changes is
taken.
The change exposes a few system registers that have no sysreg.h
definition, due to msr_s/mrs_s being used in place of msr/mrs:
additions are made in order to fill in the gaps.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christoffer Dall <christoffer.dall@arm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Link: https://www.spinics.net/lists/kvm-arm/msg31717.html
[Rebased to v4.21-rc1]
Signed-off-by: Sudeep Holla <sudeep.holla@arm.com>
[Rebased to v5.2-rc5, changelog updates]
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2019-04-06 04:29:40 -06:00
|
|
|
read_sysreg_el2(SYS_ESR), read_sysreg_el2(SYS_FAR),
|
2018-01-08 08:38:05 -07:00
|
|
|
read_sysreg(hpfar_el2), par, vcpu);
|
2015-11-17 07:07:45 -07:00
|
|
|
}
|
2019-01-24 09:32:54 -07:00
|
|
|
NOKPROBE_SYMBOL(__hyp_call_panic_vhe);
|
2015-11-17 07:07:45 -07:00
|
|
|
|
2017-10-08 09:01:56 -06:00
|
|
|
void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
|
2015-11-17 07:07:45 -07:00
|
|
|
{
|
KVM: arm64: Migrate _elx sysreg accessors to msr_s/mrs_s
Currently, the {read,write}_sysreg_el*() accessors for accessing
particular ELs' sysregs in the presence of VHE rely on some local
hacks and define their system register encodings in a way that is
inconsistent with the core definitions in <asm/sysreg.h>.
As a result, it is necessary to add duplicate definitions for any
system register that already needs a definition in sysreg.h for
other reasons.
This is a bit of a maintenance headache, and the reasons for the
_el*() accessors working the way they do is a bit historical.
This patch gets rid of the shadow sysreg definitions in
<asm/kvm_hyp.h>, converts the _el*() accessors to use the core
__msr_s/__mrs_s interface, and converts all call sites to use the
standard sysreg #define names (i.e., upper case, with SYS_ prefix).
This patch will conflict heavily anyway, so the opportunity
to clean up some bad whitespace in the context of the changes is
taken.
The change exposes a few system registers that have no sysreg.h
definition, due to msr_s/mrs_s being used in place of msr/mrs:
additions are made in order to fill in the gaps.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christoffer Dall <christoffer.dall@arm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Link: https://www.spinics.net/lists/kvm-arm/msg31717.html
[Rebased to v4.21-rc1]
Signed-off-by: Sudeep Holla <sudeep.holla@arm.com>
[Rebased to v5.2-rc5, changelog updates]
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2019-04-06 04:29:40 -06:00
|
|
|
u64 spsr = read_sysreg_el2(SYS_SPSR);
|
|
|
|
u64 elr = read_sysreg_el2(SYS_ELR);
|
2015-10-25 09:21:52 -06:00
|
|
|
u64 par = read_sysreg(par_el1);
|
|
|
|
|
2017-10-09 13:43:50 -06:00
|
|
|
if (!has_vhe())
|
|
|
|
__hyp_call_panic_nvhe(spsr, elr, par, host_ctxt);
|
|
|
|
else
|
|
|
|
__hyp_call_panic_vhe(spsr, elr, par, host_ctxt);
|
2015-10-25 09:21:52 -06:00
|
|
|
|
|
|
|
unreachable();
|
|
|
|
}
|