Small update for KVM.
* ARM: lazy context-switching of FPSIMD registers on arm64, "split" regions for vGIC redistributor * s390: cleanups for nested, clock handling, crypto, storage keys and control register bits * x86: many bugfixes, implement more Hyper-V super powers, implement lapic_timer_advance_ns even when the LAPIC timer is emulated using the processor's VMX preemption timer. Two security-related bugfixes at the top of the branch. -----BEGIN PGP SIGNATURE----- Version: GnuPG v2.0.22 (GNU/Linux) iQEcBAABAgAGBQJbH8Z/AAoJEL/70l94x66DF+UIAJeOuTp6LGasT/9uAb2OovaN +5kGmOPGFwkTcmg8BQHI2fXT4vhxMXWPFcQnyig9eXJVxhuwluXDOH4P9IMay0yw VDCBsWRdMvZDQad2hn6Z5zR4Jx01XrSaG/KqvXbbDKDCy96mWG7SYAY2m3ZwmeQi 3Pa3O3BTijr7hBYnMhdXGkSn4ZyU8uPaAgIJ8795YKeOJ2JmioGYk6fj6y2WCxA3 ztJymBjTmIoZ/F8bjuVouIyP64xH4q9roAyw4rpu7vnbWGqx1fjPYJoB8yddluWF JqCPsPzhKDO7mjZJy+lfaxIlzz2BN7tKBNCm88s5GefGXgZwk3ByAq/0GQ2M3rk= =H5zI -----END PGP SIGNATURE----- Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm Pull KVM updates from Paolo Bonzini: "Small update for KVM: ARM: - lazy context-switching of FPSIMD registers on arm64 - "split" regions for vGIC redistributor s390: - cleanups for nested - clock handling - crypto - storage keys - control register bits x86: - many bugfixes - implement more Hyper-V super powers - implement lapic_timer_advance_ns even when the LAPIC timer is emulated using the processor's VMX preemption timer. - two security-related bugfixes at the top of the branch" * tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (79 commits) kvm: fix typo in flag name kvm: x86: use correct privilege level for sgdt/sidt/fxsave/fxrstor access KVM: x86: pass kvm_vcpu to kvm_read_guest_virt and kvm_write_guest_virt_system KVM: x86: introduce linear_{read,write}_system kvm: nVMX: Enforce cpl=0 for VMX instructions kvm: nVMX: Add support for "VMWRITE to any supported field" kvm: nVMX: Restrict VMX capability MSR changes KVM: VMX: Optimize tscdeadline timer latency KVM: docs: nVMX: Remove known limitations as they do not exist now KVM: docs: mmu: KVM support exposing SLAT to guests kvm: no need to check return value of debugfs_create functions kvm: Make VM ioctl do valloc for some archs kvm: Change return type to vm_fault_t KVM: docs: mmu: Fix link to NPT presentation from KVM Forum 2008 kvm: x86: Amend the KVM_GET_SUPPORTED_CPUID API documentation KVM: x86: hyperv: declare KVM_CAP_HYPERV_TLBFLUSH capability KVM: x86: hyperv: simplistic HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE}_EX implementation KVM: x86: hyperv: simplistic HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE} implementation KVM: introduce kvm_make_vcpus_request_mask() API KVM: x86: hyperv: do rep check for each hypercall separately ...
This commit is contained in:
commit
b357bf6023
|
@ -1269,12 +1269,18 @@ struct kvm_cpuid_entry2 {
|
|||
__u32 padding[3];
|
||||
};
|
||||
|
||||
This ioctl returns x86 cpuid features which are supported by both the hardware
|
||||
and kvm. Userspace can use the information returned by this ioctl to
|
||||
construct cpuid information (for KVM_SET_CPUID2) that is consistent with
|
||||
hardware, kernel, and userspace capabilities, and with user requirements (for
|
||||
example, the user may wish to constrain cpuid to emulate older hardware,
|
||||
or for feature consistency across a cluster).
|
||||
This ioctl returns x86 cpuid features which are supported by both the
|
||||
hardware and kvm in its default configuration. Userspace can use the
|
||||
information returned by this ioctl to construct cpuid information (for
|
||||
KVM_SET_CPUID2) that is consistent with hardware, kernel, and
|
||||
userspace capabilities, and with user requirements (for example, the
|
||||
user may wish to constrain cpuid to emulate older hardware, or for
|
||||
feature consistency across a cluster).
|
||||
|
||||
Note that certain capabilities, such as KVM_CAP_X86_DISABLE_EXITS, may
|
||||
expose cpuid features (e.g. MONITOR) which are not supported by kvm in
|
||||
its default configuration. If userspace enables such capabilities, it
|
||||
is responsible for modifying the results of this ioctl appropriately.
|
||||
|
||||
Userspace invokes KVM_GET_SUPPORTED_CPUID by passing a kvm_cpuid2 structure
|
||||
with the 'nent' field indicating the number of entries in the variable-size
|
||||
|
@ -4603,3 +4609,12 @@ Architectures: s390
|
|||
This capability indicates that kvm will implement the interfaces to handle
|
||||
reset, migration and nested KVM for branch prediction blocking. The stfle
|
||||
facility 82 should not be provided to the guest without this capability.
|
||||
|
||||
8.14 KVM_CAP_HYPERV_TLBFLUSH
|
||||
|
||||
Architectures: x86
|
||||
|
||||
This capability indicates that KVM supports paravirtualized Hyper-V TLB Flush
|
||||
hypercalls:
|
||||
HvFlushVirtualAddressSpace, HvFlushVirtualAddressSpaceEx,
|
||||
HvFlushVirtualAddressList, HvFlushVirtualAddressListEx.
|
||||
|
|
|
@ -27,16 +27,42 @@ Groups:
|
|||
VCPU and all of the redistributor pages are contiguous.
|
||||
Only valid for KVM_DEV_TYPE_ARM_VGIC_V3.
|
||||
This address needs to be 64K aligned.
|
||||
|
||||
KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION (rw, 64-bit)
|
||||
The attribute data pointed to by kvm_device_attr.addr is a __u64 value:
|
||||
bits: | 63 .... 52 | 51 .... 16 | 15 - 12 |11 - 0
|
||||
values: | count | base | flags | index
|
||||
- index encodes the unique redistributor region index
|
||||
- flags: reserved for future use, currently 0
|
||||
- base field encodes bits [51:16] of the guest physical base address
|
||||
of the first redistributor in the region.
|
||||
- count encodes the number of redistributors in the region. Must be
|
||||
greater than 0.
|
||||
There are two 64K pages for each redistributor in the region and
|
||||
redistributors are laid out contiguously within the region. Regions
|
||||
are filled with redistributors in the index order. The sum of all
|
||||
region count fields must be greater than or equal to the number of
|
||||
VCPUs. Redistributor regions must be registered in the incremental
|
||||
index order, starting from index 0.
|
||||
The characteristics of a specific redistributor region can be read
|
||||
by presetting the index field in the attr data.
|
||||
Only valid for KVM_DEV_TYPE_ARM_VGIC_V3.
|
||||
|
||||
It is invalid to mix calls with KVM_VGIC_V3_ADDR_TYPE_REDIST and
|
||||
KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION attributes.
|
||||
|
||||
Errors:
|
||||
-E2BIG: Address outside of addressable IPA range
|
||||
-EINVAL: Incorrectly aligned address
|
||||
-EINVAL: Incorrectly aligned address, bad redistributor region
|
||||
count/index, mixed redistributor region attribute usage
|
||||
-EEXIST: Address already configured
|
||||
-ENOENT: Attempt to read the characteristics of a non existing
|
||||
redistributor region
|
||||
-ENXIO: The group or attribute is unknown/unsupported for this device
|
||||
or hardware support is missing.
|
||||
-EFAULT: Invalid user pointer for attr->addr.
|
||||
|
||||
|
||||
|
||||
KVM_DEV_ARM_VGIC_GRP_DIST_REGS
|
||||
KVM_DEV_ARM_VGIC_GRP_REDIST_REGS
|
||||
Attributes:
|
||||
|
|
|
@ -49,8 +49,8 @@ The mmu supports first-generation mmu hardware, which allows an atomic switch
|
|||
of the current paging mode and cr3 during guest entry, as well as
|
||||
two-dimensional paging (AMD's NPT and Intel's EPT). The emulated hardware
|
||||
it exposes is the traditional 2/3/4 level x86 mmu, with support for global
|
||||
pages, pae, pse, pse36, cr0.wp, and 1GB pages. Work is in progress to support
|
||||
exposing NPT capable hardware on NPT capable hosts.
|
||||
pages, pae, pse, pse36, cr0.wp, and 1GB pages. Emulated hardware also
|
||||
able to expose NPT capable hardware on NPT capable hosts.
|
||||
|
||||
Translation
|
||||
===========
|
||||
|
@ -465,5 +465,5 @@ Further reading
|
|||
===============
|
||||
|
||||
- NPT presentation from KVM Forum 2008
|
||||
http://www.linux-kvm.org/wiki/images/c/c8/KvmForum2008%24kdf2008_21.pdf
|
||||
http://www.linux-kvm.org/images/c/c8/KvmForum2008%24kdf2008_21.pdf
|
||||
|
||||
|
|
|
@ -31,17 +31,6 @@ L0, the guest hypervisor, which we call L1, and its nested guest, which we
|
|||
call L2.
|
||||
|
||||
|
||||
Known limitations
|
||||
-----------------
|
||||
|
||||
The current code supports running Linux guests under KVM guests.
|
||||
Only 64-bit guest hypervisors are supported.
|
||||
|
||||
Additional patches for running Windows under guest KVM, and Linux under
|
||||
guest VMware server, and support for nested EPT, are currently running in
|
||||
the lab, and will be sent as follow-on patchsets.
|
||||
|
||||
|
||||
Running nested VMX
|
||||
------------------
|
||||
|
||||
|
|
|
@ -281,6 +281,7 @@ void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
|
|||
|
||||
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
|
||||
|
||||
static inline bool kvm_arch_check_sve_has_vhe(void) { return true; }
|
||||
static inline void kvm_arch_hardware_unsetup(void) {}
|
||||
static inline void kvm_arch_sync_events(struct kvm *kvm) {}
|
||||
static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
|
||||
|
@ -304,8 +305,13 @@ int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
|
|||
int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
|
||||
struct kvm_device_attr *attr);
|
||||
|
||||
/* All host FP/SIMD state is restored on guest exit, so nothing to save: */
|
||||
static inline void kvm_fpsimd_flush_cpu_state(void) {}
|
||||
/*
|
||||
* VFP/NEON switching is all done by the hyp switch code, so no need to
|
||||
* coordinate with host context handling for this state:
|
||||
*/
|
||||
static inline void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) {}
|
||||
static inline void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) {}
|
||||
static inline void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) {}
|
||||
|
||||
static inline void kvm_arm_vhe_guest_enter(void) {}
|
||||
static inline void kvm_arm_vhe_guest_exit(void) {}
|
||||
|
@ -340,4 +346,8 @@ static inline int kvm_arm_have_ssbd(void)
|
|||
static inline void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu) {}
|
||||
static inline void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu) {}
|
||||
|
||||
#define __KVM_HAVE_ARCH_VM_ALLOC
|
||||
struct kvm *kvm_arch_alloc_vm(void);
|
||||
void kvm_arch_free_vm(struct kvm *kvm);
|
||||
|
||||
#endif /* __ARM_KVM_HOST_H__ */
|
||||
|
|
|
@ -91,6 +91,7 @@ struct kvm_regs {
|
|||
#define KVM_VGIC_V3_ADDR_TYPE_DIST 2
|
||||
#define KVM_VGIC_V3_ADDR_TYPE_REDIST 3
|
||||
#define KVM_VGIC_ITS_ADDR_TYPE 4
|
||||
#define KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION 5
|
||||
|
||||
#define KVM_VGIC_V3_DIST_SIZE SZ_64K
|
||||
#define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K)
|
||||
|
|
|
@ -1128,6 +1128,7 @@ endmenu
|
|||
config ARM64_SVE
|
||||
bool "ARM Scalable Vector Extension support"
|
||||
default y
|
||||
depends on !KVM || ARM64_VHE
|
||||
help
|
||||
The Scalable Vector Extension (SVE) is an extension to the AArch64
|
||||
execution state which complements and extends the SIMD functionality
|
||||
|
@ -1153,6 +1154,12 @@ config ARM64_SVE
|
|||
booting the kernel. If unsure and you are not observing these
|
||||
symptoms, you should assume that it is safe to say Y.
|
||||
|
||||
CPUs that support SVE are architecturally required to support the
|
||||
Virtualization Host Extensions (VHE), so the kernel makes no
|
||||
provision for supporting SVE alongside KVM without VHE enabled.
|
||||
Thus, you will need to enable CONFIG_ARM64_VHE if you want to support
|
||||
KVM in the same kernel image.
|
||||
|
||||
config ARM64_MODULE_PLTS
|
||||
bool
|
||||
select HAVE_MOD_ARCH_SPECIFIC
|
||||
|
|
|
@ -11,9 +11,7 @@
|
|||
|
||||
#include <asm/cpucaps.h>
|
||||
#include <asm/cputype.h>
|
||||
#include <asm/fpsimd.h>
|
||||
#include <asm/hwcap.h>
|
||||
#include <asm/sigcontext.h>
|
||||
#include <asm/sysreg.h>
|
||||
|
||||
/*
|
||||
|
@ -510,33 +508,6 @@ static inline bool system_supports_sve(void)
|
|||
cpus_have_const_cap(ARM64_SVE);
|
||||
}
|
||||
|
||||
/*
|
||||
* Read the pseudo-ZCR used by cpufeatures to identify the supported SVE
|
||||
* vector length.
|
||||
*
|
||||
* Use only if SVE is present.
|
||||
* This function clobbers the SVE vector length.
|
||||
*/
|
||||
static inline u64 read_zcr_features(void)
|
||||
{
|
||||
u64 zcr;
|
||||
unsigned int vq_max;
|
||||
|
||||
/*
|
||||
* Set the maximum possible VL, and write zeroes to all other
|
||||
* bits to see if they stick.
|
||||
*/
|
||||
sve_kernel_enable(NULL);
|
||||
write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
|
||||
|
||||
zcr = read_sysreg_s(SYS_ZCR_EL1);
|
||||
zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */
|
||||
vq_max = sve_vq_from_vl(sve_get_vl());
|
||||
zcr |= vq_max - 1; /* set LEN field to maximum effective value */
|
||||
|
||||
return zcr;
|
||||
}
|
||||
|
||||
#define ARM64_SSBD_UNKNOWN -1
|
||||
#define ARM64_SSBD_FORCE_DISABLE 0
|
||||
#define ARM64_SSBD_KERNEL 1
|
||||
|
|
|
@ -18,6 +18,8 @@
|
|||
|
||||
#include <asm/ptrace.h>
|
||||
#include <asm/errno.h>
|
||||
#include <asm/processor.h>
|
||||
#include <asm/sigcontext.h>
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
|
@ -41,6 +43,8 @@ struct task_struct;
|
|||
extern void fpsimd_save_state(struct user_fpsimd_state *state);
|
||||
extern void fpsimd_load_state(struct user_fpsimd_state *state);
|
||||
|
||||
extern void fpsimd_save(void);
|
||||
|
||||
extern void fpsimd_thread_switch(struct task_struct *next);
|
||||
extern void fpsimd_flush_thread(void);
|
||||
|
||||
|
@ -49,12 +53,27 @@ extern void fpsimd_preserve_current_state(void);
|
|||
extern void fpsimd_restore_current_state(void);
|
||||
extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
|
||||
|
||||
extern void fpsimd_bind_task_to_cpu(void);
|
||||
extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state);
|
||||
|
||||
extern void fpsimd_flush_task_state(struct task_struct *target);
|
||||
extern void fpsimd_flush_cpu_state(void);
|
||||
extern void sve_flush_cpu_state(void);
|
||||
|
||||
/* Maximum VL that SVE VL-agnostic software can transparently support */
|
||||
#define SVE_VL_ARCH_MAX 0x100
|
||||
|
||||
/* Offset of FFR in the SVE register dump */
|
||||
static inline size_t sve_ffr_offset(int vl)
|
||||
{
|
||||
return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
|
||||
}
|
||||
|
||||
static inline void *sve_pffr(struct thread_struct *thread)
|
||||
{
|
||||
return (char *)thread->sve_state + sve_ffr_offset(thread->sve_vl);
|
||||
}
|
||||
|
||||
extern void sve_save_state(void *state, u32 *pfpsr);
|
||||
extern void sve_load_state(void const *state, u32 const *pfpsr,
|
||||
unsigned long vq_minus_1);
|
||||
|
@ -63,6 +82,8 @@ extern unsigned int sve_get_vl(void);
|
|||
struct arm64_cpu_capabilities;
|
||||
extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused);
|
||||
|
||||
extern u64 read_zcr_features(void);
|
||||
|
||||
extern int __ro_after_init sve_max_vl;
|
||||
|
||||
#ifdef CONFIG_ARM64_SVE
|
||||
|
|
|
@ -33,19 +33,19 @@
|
|||
/* The hyp-stub will return this for any kvm_call_hyp() call */
|
||||
#define ARM_EXCEPTION_HYP_GONE HVC_STUB_ERR
|
||||
|
||||
#define KVM_ARM64_DEBUG_DIRTY_SHIFT 0
|
||||
#define KVM_ARM64_DEBUG_DIRTY (1 << KVM_ARM64_DEBUG_DIRTY_SHIFT)
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
#include <linux/mm.h>
|
||||
|
||||
/* Translate a kernel address of @sym into its equivalent linear mapping */
|
||||
#define kvm_ksym_ref(sym) \
|
||||
({ \
|
||||
void *val = &sym; \
|
||||
if (!is_kernel_in_hyp_mode()) \
|
||||
val = phys_to_virt((u64)&sym - kimage_voffset); \
|
||||
val = lm_alias(&sym); \
|
||||
val; \
|
||||
})
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
struct kvm;
|
||||
struct kvm_vcpu;
|
||||
|
||||
|
|
|
@ -30,6 +30,7 @@
|
|||
#include <asm/kvm.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
#include <asm/kvm_mmio.h>
|
||||
#include <asm/thread_info.h>
|
||||
|
||||
#define __KVM_HAVE_ARCH_INTC_INITIALIZED
|
||||
|
||||
|
@ -219,8 +220,8 @@ struct kvm_vcpu_arch {
|
|||
/* State of various workarounds, see kvm_asm.h for bit assignment */
|
||||
u64 workaround_flags;
|
||||
|
||||
/* Guest debug state */
|
||||
u64 debug_flags;
|
||||
/* Miscellaneous vcpu state flags */
|
||||
u64 flags;
|
||||
|
||||
/*
|
||||
* We maintain more than a single set of debug registers to support
|
||||
|
@ -241,6 +242,10 @@ struct kvm_vcpu_arch {
|
|||
|
||||
/* Pointer to host CPU context */
|
||||
kvm_cpu_context_t *host_cpu_context;
|
||||
|
||||
struct thread_info *host_thread_info; /* hyp VA */
|
||||
struct user_fpsimd_state *host_fpsimd_state; /* hyp VA */
|
||||
|
||||
struct {
|
||||
/* {Break,watch}point registers */
|
||||
struct kvm_guest_debug_arch regs;
|
||||
|
@ -296,6 +301,12 @@ struct kvm_vcpu_arch {
|
|||
bool sysregs_loaded_on_cpu;
|
||||
};
|
||||
|
||||
/* vcpu_arch flags field values: */
|
||||
#define KVM_ARM64_DEBUG_DIRTY (1 << 0)
|
||||
#define KVM_ARM64_FP_ENABLED (1 << 1) /* guest FP regs loaded */
|
||||
#define KVM_ARM64_FP_HOST (1 << 2) /* host FP regs loaded */
|
||||
#define KVM_ARM64_HOST_SVE_IN_USE (1 << 3) /* backup for host TIF_SVE */
|
||||
|
||||
#define vcpu_gp_regs(v) (&(v)->arch.ctxt.gp_regs)
|
||||
|
||||
/*
|
||||
|
@ -397,6 +408,19 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,
|
|||
kvm_call_hyp(__kvm_set_tpidr_el2, tpidr_el2);
|
||||
}
|
||||
|
||||
static inline bool kvm_arch_check_sve_has_vhe(void)
|
||||
{
|
||||
/*
|
||||
* The Arm architecture specifies that implementation of SVE
|
||||
* requires VHE also to be implemented. The KVM code for arm64
|
||||
* relies on this when SVE is present:
|
||||
*/
|
||||
if (system_supports_sve())
|
||||
return has_vhe();
|
||||
else
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline void kvm_arch_hardware_unsetup(void) {}
|
||||
static inline void kvm_arch_sync_events(struct kvm *kvm) {}
|
||||
static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
|
||||
|
@ -423,15 +447,18 @@ static inline void __cpu_init_stage2(void)
|
|||
"PARange is %d bits, unsupported configuration!", parange);
|
||||
}
|
||||
|
||||
/*
|
||||
* All host FP/SIMD state is restored on guest exit, so nothing needs
|
||||
* doing here except in the SVE case:
|
||||
*/
|
||||
static inline void kvm_fpsimd_flush_cpu_state(void)
|
||||
/* Guest/host FPSIMD coordination helpers */
|
||||
int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu);
|
||||
void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu);
|
||||
void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu);
|
||||
void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu);
|
||||
|
||||
#ifdef CONFIG_KVM /* Avoid conflicts with core headers if CONFIG_KVM=n */
|
||||
static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (system_supports_sve())
|
||||
sve_flush_cpu_state();
|
||||
return kvm_arch_vcpu_run_map_fp(vcpu);
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline void kvm_arm_vhe_guest_enter(void)
|
||||
{
|
||||
|
@ -481,4 +508,8 @@ static inline int kvm_arm_have_ssbd(void)
|
|||
void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu);
|
||||
void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu);
|
||||
|
||||
#define __KVM_HAVE_ARCH_VM_ALLOC
|
||||
struct kvm *kvm_arch_alloc_vm(void);
|
||||
void kvm_arch_free_vm(struct kvm *kvm);
|
||||
|
||||
#endif /* __ARM64_KVM_HOST_H__ */
|
||||
|
|
|
@ -158,7 +158,9 @@ static inline void arch_thread_struct_whitelist(unsigned long *offset,
|
|||
/* Sync TPIDR_EL0 back to thread_struct for current */
|
||||
void tls_preserve_current_state(void);
|
||||
|
||||
#define INIT_THREAD { }
|
||||
#define INIT_THREAD { \
|
||||
.fpsimd_cpu = NR_CPUS, \
|
||||
}
|
||||
|
||||
static inline void start_thread_common(struct pt_regs *regs, unsigned long pc)
|
||||
{
|
||||
|
@ -249,6 +251,17 @@ void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused);
|
|||
extern unsigned long __ro_after_init signal_minsigstksz; /* sigframe size */
|
||||
extern void __init minsigstksz_setup(void);
|
||||
|
||||
/*
|
||||
* Not at the top of the file due to a direct #include cycle between
|
||||
* <asm/fpsimd.h> and <asm/processor.h>. Deferring this #include
|
||||
* ensures that contents of processor.h are visible to fpsimd.h even if
|
||||
* processor.h is included first.
|
||||
*
|
||||
* These prctl helpers are the only things in this file that require
|
||||
* fpsimd.h. The core code expects them to be in this header.
|
||||
*/
|
||||
#include <asm/fpsimd.h>
|
||||
|
||||
/* Userspace interface for PR_SVE_{SET,GET}_VL prctl()s: */
|
||||
#define SVE_SET_VL(arg) sve_set_current_vl(arg)
|
||||
#define SVE_GET_VL() sve_get_current_vl()
|
||||
|
|
|
@ -45,12 +45,6 @@ struct thread_info {
|
|||
int preempt_count; /* 0 => preemptable, <0 => bug */
|
||||
};
|
||||
|
||||
#define INIT_THREAD_INFO(tsk) \
|
||||
{ \
|
||||
.preempt_count = INIT_PREEMPT_COUNT, \
|
||||
.addr_limit = KERNEL_DS, \
|
||||
}
|
||||
|
||||
#define thread_saved_pc(tsk) \
|
||||
((unsigned long)(tsk->thread.cpu_context.pc))
|
||||
#define thread_saved_sp(tsk) \
|
||||
|
@ -118,5 +112,12 @@ void arch_release_task_struct(struct task_struct *tsk);
|
|||
_TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
|
||||
_TIF_NOHZ)
|
||||
|
||||
#define INIT_THREAD_INFO(tsk) \
|
||||
{ \
|
||||
.flags = _TIF_FOREIGN_FPSTATE, \
|
||||
.preempt_count = INIT_PREEMPT_COUNT, \
|
||||
.addr_limit = KERNEL_DS, \
|
||||
}
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
#endif /* __ASM_THREAD_INFO_H */
|
||||
|
|
|
@ -91,6 +91,7 @@ struct kvm_regs {
|
|||
#define KVM_VGIC_V3_ADDR_TYPE_DIST 2
|
||||
#define KVM_VGIC_V3_ADDR_TYPE_REDIST 3
|
||||
#define KVM_VGIC_ITS_ADDR_TYPE 4
|
||||
#define KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION 5
|
||||
|
||||
#define KVM_VGIC_V3_DIST_SIZE SZ_64K
|
||||
#define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K)
|
||||
|
|
|
@ -36,12 +36,14 @@
|
|||
#include <linux/sched/task_stack.h>
|
||||
#include <linux/signal.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/stddef.h>
|
||||
#include <linux/sysctl.h>
|
||||
|
||||
#include <asm/esr.h>
|
||||
#include <asm/fpsimd.h>
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/cputype.h>
|
||||
#include <asm/processor.h>
|
||||
#include <asm/simd.h>
|
||||
#include <asm/sigcontext.h>
|
||||
#include <asm/sysreg.h>
|
||||
|
@ -117,7 +119,6 @@
|
|||
*/
|
||||
struct fpsimd_last_state_struct {
|
||||
struct user_fpsimd_state *st;
|
||||
bool sve_in_use;
|
||||
};
|
||||
|
||||
static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
|
||||
|
@ -158,19 +159,6 @@ static void sve_free(struct task_struct *task)
|
|||
__sve_free(task);
|
||||
}
|
||||
|
||||
|
||||
/* Offset of FFR in the SVE register dump */
|
||||
static size_t sve_ffr_offset(int vl)
|
||||
{
|
||||
return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
|
||||
}
|
||||
|
||||
static void *sve_pffr(struct task_struct *task)
|
||||
{
|
||||
return (char *)task->thread.sve_state +
|
||||
sve_ffr_offset(task->thread.sve_vl);
|
||||
}
|
||||
|
||||
static void change_cpacr(u64 val, u64 mask)
|
||||
{
|
||||
u64 cpacr = read_sysreg(CPACR_EL1);
|
||||
|
@ -251,31 +239,24 @@ static void task_fpsimd_load(void)
|
|||
WARN_ON(!in_softirq() && !irqs_disabled());
|
||||
|
||||
if (system_supports_sve() && test_thread_flag(TIF_SVE))
|
||||
sve_load_state(sve_pffr(current),
|
||||
sve_load_state(sve_pffr(¤t->thread),
|
||||
¤t->thread.uw.fpsimd_state.fpsr,
|
||||
sve_vq_from_vl(current->thread.sve_vl) - 1);
|
||||
else
|
||||
fpsimd_load_state(¤t->thread.uw.fpsimd_state);
|
||||
|
||||
if (system_supports_sve()) {
|
||||
/* Toggle SVE trapping for userspace if needed */
|
||||
if (test_thread_flag(TIF_SVE))
|
||||
sve_user_enable();
|
||||
else
|
||||
sve_user_disable();
|
||||
|
||||
/* Serialised by exception return to user */
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Ensure current's FPSIMD/SVE storage in thread_struct is up to date
|
||||
* with respect to the CPU registers.
|
||||
* Ensure FPSIMD/SVE storage in memory for the loaded context is up to
|
||||
* date with respect to the CPU registers.
|
||||
*
|
||||
* Softirqs (and preemption) must be disabled.
|
||||
*/
|
||||
static void task_fpsimd_save(void)
|
||||
void fpsimd_save(void)
|
||||
{
|
||||
struct user_fpsimd_state *st = __this_cpu_read(fpsimd_last_state.st);
|
||||
/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
|
||||
|
||||
WARN_ON(!in_softirq() && !irqs_disabled());
|
||||
|
||||
if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
|
||||
|
@ -290,10 +271,9 @@ static void task_fpsimd_save(void)
|
|||
return;
|
||||
}
|
||||
|
||||
sve_save_state(sve_pffr(current),
|
||||
¤t->thread.uw.fpsimd_state.fpsr);
|
||||
sve_save_state(sve_pffr(¤t->thread), &st->fpsr);
|
||||
} else
|
||||
fpsimd_save_state(¤t->thread.uw.fpsimd_state);
|
||||
fpsimd_save_state(st);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -588,7 +568,7 @@ int sve_set_vector_length(struct task_struct *task,
|
|||
if (task == current) {
|
||||
local_bh_disable();
|
||||
|
||||
task_fpsimd_save();
|
||||
fpsimd_save();
|
||||
set_thread_flag(TIF_FOREIGN_FPSTATE);
|
||||
}
|
||||
|
||||
|
@ -608,10 +588,8 @@ int sve_set_vector_length(struct task_struct *task,
|
|||
task->thread.sve_vl = vl;
|
||||
|
||||
out:
|
||||
if (flags & PR_SVE_VL_INHERIT)
|
||||
set_tsk_thread_flag(task, TIF_SVE_VL_INHERIT);
|
||||
else
|
||||
clear_tsk_thread_flag(task, TIF_SVE_VL_INHERIT);
|
||||
update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT,
|
||||
flags & PR_SVE_VL_INHERIT);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -755,6 +733,33 @@ void sve_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p)
|
|||
isb();
|
||||
}
|
||||
|
||||
/*
|
||||
* Read the pseudo-ZCR used by cpufeatures to identify the supported SVE
|
||||
* vector length.
|
||||
*
|
||||
* Use only if SVE is present.
|
||||
* This function clobbers the SVE vector length.
|
||||
*/
|
||||
u64 read_zcr_features(void)
|
||||
{
|
||||
u64 zcr;
|
||||
unsigned int vq_max;
|
||||
|
||||
/*
|
||||
* Set the maximum possible VL, and write zeroes to all other
|
||||
* bits to see if they stick.
|
||||
*/
|
||||
sve_kernel_enable(NULL);
|
||||
write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
|
||||
|
||||
zcr = read_sysreg_s(SYS_ZCR_EL1);
|
||||
zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */
|
||||
vq_max = sve_vq_from_vl(sve_get_vl());
|
||||
zcr |= vq_max - 1; /* set LEN field to maximum effective value */
|
||||
|
||||
return zcr;
|
||||
}
|
||||
|
||||
void __init sve_setup(void)
|
||||
{
|
||||
u64 zcr;
|
||||
|
@ -829,7 +834,7 @@ asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
|
|||
|
||||
local_bh_disable();
|
||||
|
||||
task_fpsimd_save();
|
||||
fpsimd_save();
|
||||
fpsimd_to_sve(current);
|
||||
|
||||
/* Force ret_to_user to reload the registers: */
|
||||
|
@ -882,31 +887,25 @@ asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
|
|||
|
||||
void fpsimd_thread_switch(struct task_struct *next)
|
||||
{
|
||||
bool wrong_task, wrong_cpu;
|
||||
|
||||
if (!system_supports_fpsimd())
|
||||
return;
|
||||
/*
|
||||
* Save the current FPSIMD state to memory, but only if whatever is in
|
||||
* the registers is in fact the most recent userland FPSIMD state of
|
||||
* 'current'.
|
||||
*/
|
||||
if (current->mm)
|
||||
task_fpsimd_save();
|
||||
|
||||
if (next->mm) {
|
||||
/*
|
||||
* If we are switching to a task whose most recent userland
|
||||
* FPSIMD state is already in the registers of *this* cpu,
|
||||
* we can skip loading the state from memory. Otherwise, set
|
||||
* the TIF_FOREIGN_FPSTATE flag so the state will be loaded
|
||||
* upon the next return to userland.
|
||||
*/
|
||||
if (__this_cpu_read(fpsimd_last_state.st) ==
|
||||
&next->thread.uw.fpsimd_state
|
||||
&& next->thread.fpsimd_cpu == smp_processor_id())
|
||||
clear_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
|
||||
else
|
||||
set_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
|
||||
}
|
||||
/* Save unsaved fpsimd state, if any: */
|
||||
fpsimd_save();
|
||||
|
||||
/*
|
||||
* Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
|
||||
* state. For kernel threads, FPSIMD registers are never loaded
|
||||
* and wrong_task and wrong_cpu will always be true.
|
||||
*/
|
||||
wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
|
||||
&next->thread.uw.fpsimd_state;
|
||||
wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
|
||||
|
||||
update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
|
||||
wrong_task || wrong_cpu);
|
||||
}
|
||||
|
||||
void fpsimd_flush_thread(void)
|
||||
|
@ -972,7 +971,7 @@ void fpsimd_preserve_current_state(void)
|
|||
return;
|
||||
|
||||
local_bh_disable();
|
||||
task_fpsimd_save();
|
||||
fpsimd_save();
|
||||
local_bh_enable();
|
||||
}
|
||||
|
||||
|
@ -992,14 +991,33 @@ void fpsimd_signal_preserve_current_state(void)
|
|||
* Associate current's FPSIMD context with this cpu
|
||||
* Preemption must be disabled when calling this function.
|
||||
*/
|
||||
static void fpsimd_bind_to_cpu(void)
|
||||
void fpsimd_bind_task_to_cpu(void)
|
||||
{
|
||||
struct fpsimd_last_state_struct *last =
|
||||
this_cpu_ptr(&fpsimd_last_state);
|
||||
|
||||
last->st = ¤t->thread.uw.fpsimd_state;
|
||||
last->sve_in_use = test_thread_flag(TIF_SVE);
|
||||
current->thread.fpsimd_cpu = smp_processor_id();
|
||||
|
||||
if (system_supports_sve()) {
|
||||
/* Toggle SVE trapping for userspace if needed */
|
||||
if (test_thread_flag(TIF_SVE))
|
||||
sve_user_enable();
|
||||
else
|
||||
sve_user_disable();
|
||||
|
||||
/* Serialised by exception return to user */
|
||||
}
|
||||
}
|
||||
|
||||
void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st)
|
||||
{
|
||||
struct fpsimd_last_state_struct *last =
|
||||
this_cpu_ptr(&fpsimd_last_state);
|
||||
|
||||
WARN_ON(!in_softirq() && !irqs_disabled());
|
||||
|
||||
last->st = st;
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -1016,7 +1034,7 @@ void fpsimd_restore_current_state(void)
|
|||
|
||||
if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
|
||||
task_fpsimd_load();
|
||||
fpsimd_bind_to_cpu();
|
||||
fpsimd_bind_task_to_cpu();
|
||||
}
|
||||
|
||||
local_bh_enable();
|
||||
|
@ -1039,9 +1057,9 @@ void fpsimd_update_current_state(struct user_fpsimd_state const *state)
|
|||
fpsimd_to_sve(current);
|
||||
|
||||
task_fpsimd_load();
|
||||
fpsimd_bind_task_to_cpu();
|
||||
|
||||
if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE))
|
||||
fpsimd_bind_to_cpu();
|
||||
clear_thread_flag(TIF_FOREIGN_FPSTATE);
|
||||
|
||||
local_bh_enable();
|
||||
}
|
||||
|
@ -1054,29 +1072,12 @@ void fpsimd_flush_task_state(struct task_struct *t)
|
|||
t->thread.fpsimd_cpu = NR_CPUS;
|
||||
}
|
||||
|
||||
static inline void fpsimd_flush_cpu_state(void)
|
||||
void fpsimd_flush_cpu_state(void)
|
||||
{
|
||||
__this_cpu_write(fpsimd_last_state.st, NULL);
|
||||
set_thread_flag(TIF_FOREIGN_FPSTATE);
|
||||
}
|
||||
|
||||
/*
|
||||
* Invalidate any task SVE state currently held in this CPU's regs.
|
||||
*
|
||||
* This is used to prevent the kernel from trying to reuse SVE register data
|
||||
* that is detroyed by KVM guest enter/exit. This function should go away when
|
||||
* KVM SVE support is implemented. Don't use it for anything else.
|
||||
*/
|
||||
#ifdef CONFIG_ARM64_SVE
|
||||
void sve_flush_cpu_state(void)
|
||||
{
|
||||
struct fpsimd_last_state_struct const *last =
|
||||
this_cpu_ptr(&fpsimd_last_state);
|
||||
|
||||
if (last->st && last->sve_in_use)
|
||||
fpsimd_flush_cpu_state();
|
||||
}
|
||||
#endif /* CONFIG_ARM64_SVE */
|
||||
|
||||
#ifdef CONFIG_KERNEL_MODE_NEON
|
||||
|
||||
DEFINE_PER_CPU(bool, kernel_neon_busy);
|
||||
|
@ -1110,11 +1111,8 @@ void kernel_neon_begin(void)
|
|||
|
||||
__this_cpu_write(kernel_neon_busy, true);
|
||||
|
||||
/* Save unsaved task fpsimd state, if any: */
|
||||
if (current->mm) {
|
||||
task_fpsimd_save();
|
||||
set_thread_flag(TIF_FOREIGN_FPSTATE);
|
||||
}
|
||||
/* Save unsaved fpsimd state, if any: */
|
||||
fpsimd_save();
|
||||
|
||||
/* Invalidate any task state remaining in the fpsimd regs: */
|
||||
fpsimd_flush_cpu_state();
|
||||
|
@ -1236,13 +1234,10 @@ static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
|
|||
{
|
||||
switch (cmd) {
|
||||
case CPU_PM_ENTER:
|
||||
if (current->mm)
|
||||
task_fpsimd_save();
|
||||
fpsimd_save();
|
||||
fpsimd_flush_cpu_state();
|
||||
break;
|
||||
case CPU_PM_EXIT:
|
||||
if (current->mm)
|
||||
set_thread_flag(TIF_FOREIGN_FPSTATE);
|
||||
break;
|
||||
case CPU_PM_ENTER_FAILED:
|
||||
default:
|
||||
|
|
|
@ -44,6 +44,7 @@
|
|||
#include <asm/compat.h>
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/fpsimd.h>
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/stacktrace.h>
|
||||
#include <asm/syscall.h>
|
||||
|
|
|
@ -39,6 +39,7 @@ config KVM
|
|||
select HAVE_KVM_IRQ_ROUTING
|
||||
select IRQ_BYPASS_MANAGER
|
||||
select HAVE_KVM_IRQ_BYPASS
|
||||
select HAVE_KVM_VCPU_RUN_PID_CHANGE
|
||||
---help---
|
||||
Support hosting virtualized guest machines.
|
||||
We don't support KVM with 16K page tables yet, due to the multiple
|
||||
|
|
|
@ -19,7 +19,7 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/psci.o $(KVM)/arm/perf.o
|
|||
kvm-$(CONFIG_KVM_ARM_HOST) += inject_fault.o regmap.o va_layout.o
|
||||
kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o
|
||||
kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o
|
||||
kvm-$(CONFIG_KVM_ARM_HOST) += vgic-sys-reg-v3.o
|
||||
kvm-$(CONFIG_KVM_ARM_HOST) += vgic-sys-reg-v3.o fpsimd.o
|
||||
kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/aarch32.o
|
||||
|
||||
kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic.o
|
||||
|
|
|
@ -103,7 +103,7 @@ void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu)
|
|||
*
|
||||
* Additionally, KVM only traps guest accesses to the debug registers if
|
||||
* the guest is not actively using them (see the KVM_ARM64_DEBUG_DIRTY
|
||||
* flag on vcpu->arch.debug_flags). Since the guest must not interfere
|
||||
* flag on vcpu->arch.flags). Since the guest must not interfere
|
||||
* with the hardware state when debugging the guest, we must ensure that
|
||||
* trapping is enabled whenever we are debugging the guest using the
|
||||
* debug registers.
|
||||
|
@ -111,7 +111,7 @@ void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu)
|
|||
|
||||
void kvm_arm_setup_debug(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
bool trap_debug = !(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY);
|
||||
bool trap_debug = !(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY);
|
||||
unsigned long mdscr;
|
||||
|
||||
trace_kvm_arm_setup_debug(vcpu, vcpu->guest_debug);
|
||||
|
@ -184,7 +184,7 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu)
|
|||
vcpu_write_sys_reg(vcpu, mdscr, MDSCR_EL1);
|
||||
|
||||
vcpu->arch.debug_ptr = &vcpu->arch.external_debug_state;
|
||||
vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
trap_debug = true;
|
||||
|
||||
trace_kvm_arm_set_regset("BKPTS", get_num_brps(),
|
||||
|
@ -206,7 +206,7 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu)
|
|||
|
||||
/* If KDE or MDE are set, perform a full save/restore cycle. */
|
||||
if (vcpu_read_sys_reg(vcpu, MDSCR_EL1) & (DBG_MDSCR_KDE | DBG_MDSCR_MDE))
|
||||
vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
|
||||
trace_kvm_arm_set_dreg32("MDCR_EL2", vcpu->arch.mdcr_el2);
|
||||
trace_kvm_arm_set_dreg32("MDSCR_EL1", vcpu_read_sys_reg(vcpu, MDSCR_EL1));
|
||||
|
|
110
arch/arm64/kvm/fpsimd.c
Normal file
110
arch/arm64/kvm/fpsimd.c
Normal file
|
@ -0,0 +1,110 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* arch/arm64/kvm/fpsimd.c: Guest/host FPSIMD context coordination helpers
|
||||
*
|
||||
* Copyright 2018 Arm Limited
|
||||
* Author: Dave Martin <Dave.Martin@arm.com>
|
||||
*/
|
||||
#include <linux/bottom_half.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/thread_info.h>
|
||||
#include <linux/kvm_host.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
#include <asm/kvm_host.h>
|
||||
#include <asm/kvm_mmu.h>
|
||||
|
||||
/*
|
||||
* Called on entry to KVM_RUN unless this vcpu previously ran at least
|
||||
* once and the most recent prior KVM_RUN for this vcpu was called from
|
||||
* the same task as current (highly likely).
|
||||
*
|
||||
* This is guaranteed to execute before kvm_arch_vcpu_load_fp(vcpu),
|
||||
* such that on entering hyp the relevant parts of current are already
|
||||
* mapped.
|
||||
*/
|
||||
int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
int ret;
|
||||
|
||||
struct thread_info *ti = ¤t->thread_info;
|
||||
struct user_fpsimd_state *fpsimd = ¤t->thread.uw.fpsimd_state;
|
||||
|
||||
/*
|
||||
* Make sure the host task thread flags and fpsimd state are
|
||||
* visible to hyp:
|
||||
*/
|
||||
ret = create_hyp_mappings(ti, ti + 1, PAGE_HYP);
|
||||
if (ret)
|
||||
goto error;
|
||||
|
||||
ret = create_hyp_mappings(fpsimd, fpsimd + 1, PAGE_HYP);
|
||||
if (ret)
|
||||
goto error;
|
||||
|
||||
vcpu->arch.host_thread_info = kern_hyp_va(ti);
|
||||
vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);
|
||||
error:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Prepare vcpu for saving the host's FPSIMD state and loading the guest's.
|
||||
* The actual loading is done by the FPSIMD access trap taken to hyp.
|
||||
*
|
||||
* Here, we just set the correct metadata to indicate that the FPSIMD
|
||||
* state in the cpu regs (if any) belongs to current on the host.
|
||||
*
|
||||
* TIF_SVE is backed up here, since it may get clobbered with guest state.
|
||||
* This flag is restored by kvm_arch_vcpu_put_fp(vcpu).
|
||||
*/
|
||||
void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
BUG_ON(!current->mm);
|
||||
|
||||
vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED | KVM_ARM64_HOST_SVE_IN_USE);
|
||||
vcpu->arch.flags |= KVM_ARM64_FP_HOST;
|
||||
if (test_thread_flag(TIF_SVE))
|
||||
vcpu->arch.flags |= KVM_ARM64_HOST_SVE_IN_USE;
|
||||
}
|
||||
|
||||
/*
|
||||
* If the guest FPSIMD state was loaded, update the host's context
|
||||
* tracking data mark the CPU FPSIMD regs as dirty and belonging to vcpu
|
||||
* so that they will be written back if the kernel clobbers them due to
|
||||
* kernel-mode NEON before re-entry into the guest.
|
||||
*/
|
||||
void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
WARN_ON_ONCE(!irqs_disabled());
|
||||
|
||||
if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
|
||||
fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.gp_regs.fp_regs);
|
||||
clear_thread_flag(TIF_FOREIGN_FPSTATE);
|
||||
clear_thread_flag(TIF_SVE);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Write back the vcpu FPSIMD regs if they are dirty, and invalidate the
|
||||
* cpu FPSIMD regs so that they can't be spuriously reused if this vcpu
|
||||
* disappears and another task or vcpu appears that recycles the same
|
||||
* struct fpsimd_state.
|
||||
*/
|
||||
void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
local_bh_disable();
|
||||
|
||||
update_thread_flag(TIF_SVE,
|
||||
vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE);
|
||||
|
||||
if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
|
||||
/* Clean guest FP state to memory and invalidate cpu view */
|
||||
fpsimd_save();
|
||||
fpsimd_flush_cpu_state();
|
||||
} else if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
|
||||
/* Ensure user trap controls are correctly restored */
|
||||
fpsimd_bind_task_to_cpu();
|
||||
}
|
||||
|
||||
local_bh_enable();
|
||||
}
|
|
@ -163,7 +163,7 @@ void __hyp_text __debug_switch_to_guest(struct kvm_vcpu *vcpu)
|
|||
if (!has_vhe())
|
||||
__debug_save_spe_nvhe(&vcpu->arch.host_debug_state.pmscr_el1);
|
||||
|
||||
if (!(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY))
|
||||
if (!(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY))
|
||||
return;
|
||||
|
||||
host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
|
||||
|
@ -185,7 +185,7 @@ void __hyp_text __debug_switch_to_host(struct kvm_vcpu *vcpu)
|
|||
if (!has_vhe())
|
||||
__debug_restore_spe_nvhe(vcpu->arch.host_debug_state.pmscr_el1);
|
||||
|
||||
if (!(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY))
|
||||
if (!(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY))
|
||||
return;
|
||||
|
||||
host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
|
||||
|
@ -196,7 +196,7 @@ void __hyp_text __debug_switch_to_host(struct kvm_vcpu *vcpu)
|
|||
__debug_save_state(vcpu, guest_dbg, guest_ctxt);
|
||||
__debug_restore_state(vcpu, host_dbg, host_ctxt);
|
||||
|
||||
vcpu->arch.debug_flags &= ~KVM_ARM64_DEBUG_DIRTY;
|
||||
vcpu->arch.flags &= ~KVM_ARM64_DEBUG_DIRTY;
|
||||
}
|
||||
|
||||
u32 __hyp_text __kvm_get_mdcr_el2(void)
|
||||
|
|
|
@ -166,46 +166,3 @@ abort_guest_exit_end:
|
|||
orr x0, x0, x5
|
||||
1: ret
|
||||
ENDPROC(__guest_exit)
|
||||
|
||||
ENTRY(__fpsimd_guest_restore)
|
||||
// x0: esr
|
||||
// x1: vcpu
|
||||
// x2-x29,lr: vcpu regs
|
||||
// vcpu x0-x1 on the stack
|
||||
stp x2, x3, [sp, #-16]!
|
||||
stp x4, lr, [sp, #-16]!
|
||||
|
||||
alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
|
||||
mrs x2, cptr_el2
|
||||
bic x2, x2, #CPTR_EL2_TFP
|
||||
msr cptr_el2, x2
|
||||
alternative_else
|
||||
mrs x2, cpacr_el1
|
||||
orr x2, x2, #CPACR_EL1_FPEN
|
||||
msr cpacr_el1, x2
|
||||
alternative_endif
|
||||
isb
|
||||
|
||||
mov x3, x1
|
||||
|
||||
ldr x0, [x3, #VCPU_HOST_CONTEXT]
|
||||
kern_hyp_va x0
|
||||
add x0, x0, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
|
||||
bl __fpsimd_save_state
|
||||
|
||||
add x2, x3, #VCPU_CONTEXT
|
||||
add x0, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
|
||||
bl __fpsimd_restore_state
|
||||
|
||||
// Skip restoring fpexc32 for AArch64 guests
|
||||
mrs x1, hcr_el2
|
||||
tbnz x1, #HCR_RW_SHIFT, 1f
|
||||
ldr x4, [x3, #VCPU_FPEXC32_EL2]
|
||||
msr fpexc32_el2, x4
|
||||
1:
|
||||
ldp x4, lr, [sp], #16
|
||||
ldp x2, x3, [sp], #16
|
||||
ldp x0, x1, [sp], #16
|
||||
|
||||
eret
|
||||
ENDPROC(__fpsimd_guest_restore)
|
||||
|
|
|
@ -149,25 +149,6 @@ wa_epilogue:
|
|||
|
||||
el1_trap:
|
||||
get_vcpu_ptr x1, x0
|
||||
|
||||
mrs x0, esr_el2
|
||||
lsr x0, x0, #ESR_ELx_EC_SHIFT
|
||||
/*
|
||||
* x0: ESR_EC
|
||||
* x1: vcpu pointer
|
||||
*/
|
||||
|
||||
/*
|
||||
* We trap the first access to the FP/SIMD to save the host context
|
||||
* and restore the guest context lazily.
|
||||
* If FP/SIMD is not implemented, handle the trap and inject an
|
||||
* undefined instruction exception to the guest.
|
||||
*/
|
||||
alternative_if_not ARM64_HAS_NO_FPSIMD
|
||||
cmp x0, #ESR_ELx_EC_FP_ASIMD
|
||||
b.eq __fpsimd_guest_restore
|
||||
alternative_else_nop_endif
|
||||
|
||||
mov x0, #ARM_EXCEPTION_TRAP
|
||||
b __guest_exit
|
||||
|
||||
|
|
|
@ -22,21 +22,25 @@
|
|||
|
||||
#include <kvm/arm_psci.h>
|
||||
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
#include <asm/kvm_emulate.h>
|
||||
#include <asm/kvm_host.h>
|
||||
#include <asm/kvm_hyp.h>
|
||||
#include <asm/kvm_mmu.h>
|
||||
#include <asm/fpsimd.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/processor.h>
|
||||
#include <asm/thread_info.h>
|
||||
|
||||
static bool __hyp_text __fpsimd_enabled_nvhe(void)
|
||||
/* Check whether the FP regs were dirtied while in the host-side run loop: */
|
||||
static bool __hyp_text update_fp_enabled(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return !(read_sysreg(cptr_el2) & CPTR_EL2_TFP);
|
||||
}
|
||||
if (vcpu->arch.host_thread_info->flags & _TIF_FOREIGN_FPSTATE)
|
||||
vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |
|
||||
KVM_ARM64_FP_HOST);
|
||||
|
||||
static bool fpsimd_enabled_vhe(void)
|
||||
{
|
||||
return !!(read_sysreg(cpacr_el1) & CPACR_EL1_FPEN);
|
||||
return !!(vcpu->arch.flags & KVM_ARM64_FP_ENABLED);
|
||||
}
|
||||
|
||||
/* Save the 32-bit only FPSIMD system register state */
|
||||
|
@ -93,7 +97,10 @@ static void activate_traps_vhe(struct kvm_vcpu *vcpu)
|
|||
|
||||
val = read_sysreg(cpacr_el1);
|
||||
val |= CPACR_EL1_TTA;
|
||||
val &= ~(CPACR_EL1_FPEN | CPACR_EL1_ZEN);
|
||||
val &= ~CPACR_EL1_ZEN;
|
||||
if (!update_fp_enabled(vcpu))
|
||||
val &= ~CPACR_EL1_FPEN;
|
||||
|
||||
write_sysreg(val, cpacr_el1);
|
||||
|
||||
write_sysreg(kvm_get_hyp_vector(), vbar_el1);
|
||||
|
@ -106,7 +113,10 @@ static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu)
|
|||
__activate_traps_common(vcpu);
|
||||
|
||||
val = CPTR_EL2_DEFAULT;
|
||||
val |= CPTR_EL2_TTA | CPTR_EL2_TFP | CPTR_EL2_TZ;
|
||||
val |= CPTR_EL2_TTA | CPTR_EL2_TZ;
|
||||
if (!update_fp_enabled(vcpu))
|
||||
val |= CPTR_EL2_TFP;
|
||||
|
||||
write_sysreg(val, cptr_el2);
|
||||
}
|
||||
|
||||
|
@ -319,6 +329,50 @@ static bool __hyp_text __skip_instr(struct kvm_vcpu *vcpu)
|
|||
}
|
||||
}
|
||||
|
||||
static bool __hyp_text __hyp_switch_fpsimd(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
struct user_fpsimd_state *host_fpsimd = vcpu->arch.host_fpsimd_state;
|
||||
|
||||
if (has_vhe())
|
||||
write_sysreg(read_sysreg(cpacr_el1) | CPACR_EL1_FPEN,
|
||||
cpacr_el1);
|
||||
else
|
||||
write_sysreg(read_sysreg(cptr_el2) & ~(u64)CPTR_EL2_TFP,
|
||||
cptr_el2);
|
||||
|
||||
isb();
|
||||
|
||||
if (vcpu->arch.flags & KVM_ARM64_FP_HOST) {
|
||||
/*
|
||||
* In the SVE case, VHE is assumed: it is enforced by
|
||||
* Kconfig and kvm_arch_init().
|
||||
*/
|
||||
if (system_supports_sve() &&
|
||||
(vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE)) {
|
||||
struct thread_struct *thread = container_of(
|
||||
host_fpsimd,
|
||||
struct thread_struct, uw.fpsimd_state);
|
||||
|
||||
sve_save_state(sve_pffr(thread), &host_fpsimd->fpsr);
|
||||
} else {
|
||||
__fpsimd_save_state(host_fpsimd);
|
||||
}
|
||||
|
||||
vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;
|
||||
}
|
||||
|
||||
__fpsimd_restore_state(&vcpu->arch.ctxt.gp_regs.fp_regs);
|
||||
|
||||
/* Skip restoring fpexc32 for AArch64 guests */
|
||||
if (!(read_sysreg(hcr_el2) & HCR_RW))
|
||||
write_sysreg(vcpu->arch.ctxt.sys_regs[FPEXC32_EL2],
|
||||
fpexc32_el2);
|
||||
|
||||
vcpu->arch.flags |= KVM_ARM64_FP_ENABLED;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return true when we were able to fixup the guest exit and should return to
|
||||
* the guest, false when we should restore the host state and return to the
|
||||
|
@ -335,11 +389,23 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
* same PC once the SError has been injected, and replay the
|
||||
* trapping instruction.
|
||||
*/
|
||||
if (*exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
|
||||
if (*exit_code != ARM_EXCEPTION_TRAP)
|
||||
goto exit;
|
||||
|
||||
/*
|
||||
* We trap the first access to the FP/SIMD to save the host context
|
||||
* and restore the guest context lazily.
|
||||
* If FP/SIMD is not implemented, handle the trap and inject an
|
||||
* undefined instruction exception to the guest.
|
||||
*/
|
||||
if (system_supports_fpsimd() &&
|
||||
kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_FP_ASIMD)
|
||||
return __hyp_switch_fpsimd(vcpu);
|
||||
|
||||
if (!__populate_fault_info(vcpu))
|
||||
return true;
|
||||
|
||||
if (static_branch_unlikely(&vgic_v2_cpuif_trap) &&
|
||||
*exit_code == ARM_EXCEPTION_TRAP) {
|
||||
if (static_branch_unlikely(&vgic_v2_cpuif_trap)) {
|
||||
bool valid;
|
||||
|
||||
valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&
|
||||
|
@ -351,12 +417,8 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
if (valid) {
|
||||
int ret = __vgic_v2_perform_cpuif_access(vcpu);
|
||||
|
||||
if (ret == 1) {
|
||||
if (__skip_instr(vcpu))
|
||||
return true;
|
||||
else
|
||||
*exit_code = ARM_EXCEPTION_TRAP;
|
||||
}
|
||||
if (ret == 1 && __skip_instr(vcpu))
|
||||
return true;
|
||||
|
||||
if (ret == -1) {
|
||||
/* Promote an illegal access to an
|
||||
|
@ -369,23 +431,21 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
|
||||
*exit_code = ARM_EXCEPTION_EL1_SERROR;
|
||||
}
|
||||
|
||||
goto exit;
|
||||
}
|
||||
}
|
||||
|
||||
if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
|
||||
*exit_code == ARM_EXCEPTION_TRAP &&
|
||||
(kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 ||
|
||||
kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) {
|
||||
int ret = __vgic_v3_perform_cpuif_access(vcpu);
|
||||
|
||||
if (ret == 1) {
|
||||
if (__skip_instr(vcpu))
|
||||
return true;
|
||||
else
|
||||
*exit_code = ARM_EXCEPTION_TRAP;
|
||||
}
|
||||
if (ret == 1 && __skip_instr(vcpu))
|
||||
return true;
|
||||
}
|
||||
|
||||
exit:
|
||||
/* Return to the host kernel and handle the exit */
|
||||
return false;
|
||||
}
|
||||
|
@ -428,7 +488,6 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
|
|||
{
|
||||
struct kvm_cpu_context *host_ctxt;
|
||||
struct kvm_cpu_context *guest_ctxt;
|
||||
bool fp_enabled;
|
||||
u64 exit_code;
|
||||
|
||||
host_ctxt = vcpu->arch.host_cpu_context;
|
||||
|
@ -454,19 +513,14 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
|
|||
|
||||
__set_host_arch_workaround_state(vcpu);
|
||||
|
||||
fp_enabled = fpsimd_enabled_vhe();
|
||||
|
||||
sysreg_save_guest_state_vhe(guest_ctxt);
|
||||
|
||||
__deactivate_traps(vcpu);
|
||||
|
||||
sysreg_restore_host_state_vhe(host_ctxt);
|
||||
|
||||
if (fp_enabled) {
|
||||
__fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
|
||||
__fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
|
||||
if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
|
||||
__fpsimd_save_fpexc32(vcpu);
|
||||
}
|
||||
|
||||
__debug_switch_to_host(vcpu);
|
||||
|
||||
|
@ -478,7 +532,6 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
|
|||
{
|
||||
struct kvm_cpu_context *host_ctxt;
|
||||
struct kvm_cpu_context *guest_ctxt;
|
||||
bool fp_enabled;
|
||||
u64 exit_code;
|
||||
|
||||
vcpu = kern_hyp_va(vcpu);
|
||||
|
@ -514,8 +567,6 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
|
|||
|
||||
__set_host_arch_workaround_state(vcpu);
|
||||
|
||||
fp_enabled = __fpsimd_enabled_nvhe();
|
||||
|
||||
__sysreg_save_state_nvhe(guest_ctxt);
|
||||
__sysreg32_save_state(vcpu);
|
||||
__timer_disable_traps(vcpu);
|
||||
|
@ -526,11 +577,8 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
|
|||
|
||||
__sysreg_restore_state_nvhe(host_ctxt);
|
||||
|
||||
if (fp_enabled) {
|
||||
__fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
|
||||
__fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
|
||||
if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
|
||||
__fpsimd_save_fpexc32(vcpu);
|
||||
}
|
||||
|
||||
/*
|
||||
* This must come after restoring the host sysregs, since a non-VHE
|
||||
|
|
|
@ -196,7 +196,7 @@ void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
|
|||
sysreg[DACR32_EL2] = read_sysreg(dacr32_el2);
|
||||
sysreg[IFSR32_EL2] = read_sysreg(ifsr32_el2);
|
||||
|
||||
if (has_vhe() || vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
|
||||
if (has_vhe() || vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)
|
||||
sysreg[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
|
||||
}
|
||||
|
||||
|
@ -218,7 +218,7 @@ void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
|
|||
write_sysreg(sysreg[DACR32_EL2], dacr32_el2);
|
||||
write_sysreg(sysreg[IFSR32_EL2], ifsr32_el2);
|
||||
|
||||
if (has_vhe() || vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
|
||||
if (has_vhe() || vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)
|
||||
write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
|
||||
}
|
||||
|
||||
|
|
|
@ -31,7 +31,6 @@
|
|||
#include <asm/debug-monitors.h>
|
||||
#include <asm/esr.h>
|
||||
#include <asm/kvm_arm.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
#include <asm/kvm_coproc.h>
|
||||
#include <asm/kvm_emulate.h>
|
||||
#include <asm/kvm_host.h>
|
||||
|
@ -338,7 +337,7 @@ static bool trap_debug_regs(struct kvm_vcpu *vcpu,
|
|||
{
|
||||
if (p->is_write) {
|
||||
vcpu_write_sys_reg(vcpu, p->regval, r->reg);
|
||||
vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
} else {
|
||||
p->regval = vcpu_read_sys_reg(vcpu, r->reg);
|
||||
}
|
||||
|
@ -369,7 +368,7 @@ static void reg_to_dbg(struct kvm_vcpu *vcpu,
|
|||
}
|
||||
|
||||
*dbg_reg = val;
|
||||
vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
}
|
||||
|
||||
static void dbg_to_reg(struct kvm_vcpu *vcpu,
|
||||
|
@ -1441,7 +1440,7 @@ static bool trap_debug32(struct kvm_vcpu *vcpu,
|
|||
{
|
||||
if (p->is_write) {
|
||||
vcpu_cp14(vcpu, r->reg) = p->regval;
|
||||
vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
} else {
|
||||
p->regval = vcpu_cp14(vcpu, r->reg);
|
||||
}
|
||||
|
@ -1473,7 +1472,7 @@ static bool trap_xvr(struct kvm_vcpu *vcpu,
|
|||
val |= p->regval << 32;
|
||||
*dbg_reg = val;
|
||||
|
||||
vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
|
||||
} else {
|
||||
p->regval = *dbg_reg >> 32;
|
||||
}
|
||||
|
|
|
@ -1076,7 +1076,7 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
|
|||
return -ENOIOCTLCMD;
|
||||
}
|
||||
|
||||
int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
|
||||
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
|
||||
{
|
||||
return VM_FAULT_SIGBUS;
|
||||
}
|
||||
|
|
|
@ -3955,8 +3955,7 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm)
|
|||
*/
|
||||
snprintf(buf, sizeof(buf), "vm%d", current->pid);
|
||||
kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);
|
||||
if (!IS_ERR_OR_NULL(kvm->arch.debugfs_dir))
|
||||
kvmppc_mmu_debugfs_init(kvm);
|
||||
kvmppc_mmu_debugfs_init(kvm);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -1830,7 +1830,7 @@ out:
|
|||
return r;
|
||||
}
|
||||
|
||||
int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
|
||||
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
|
||||
{
|
||||
return VM_FAULT_SIGBUS;
|
||||
}
|
||||
|
|
|
@ -10,8 +10,20 @@
|
|||
|
||||
#include <linux/const.h>
|
||||
|
||||
#define CR0_CLOCK_COMPARATOR_SIGN _BITUL(63 - 10)
|
||||
#define CR0_EMERGENCY_SIGNAL_SUBMASK _BITUL(63 - 49)
|
||||
#define CR0_EXTERNAL_CALL_SUBMASK _BITUL(63 - 50)
|
||||
#define CR0_CLOCK_COMPARATOR_SUBMASK _BITUL(63 - 52)
|
||||
#define CR0_CPU_TIMER_SUBMASK _BITUL(63 - 53)
|
||||
#define CR0_SERVICE_SIGNAL_SUBMASK _BITUL(63 - 54)
|
||||
#define CR0_UNUSED_56 _BITUL(63 - 56)
|
||||
#define CR0_INTERRUPT_KEY_SUBMASK _BITUL(63 - 57)
|
||||
#define CR0_MEASUREMENT_ALERT_SUBMASK _BITUL(63 - 58)
|
||||
|
||||
#define CR2_GUARDED_STORAGE _BITUL(63 - 59)
|
||||
|
||||
#define CR14_UNUSED_32 _BITUL(63 - 32)
|
||||
#define CR14_UNUSED_33 _BITUL(63 - 33)
|
||||
#define CR14_CHANNEL_REPORT_SUBMASK _BITUL(63 - 35)
|
||||
#define CR14_RECOVERY_SUBMASK _BITUL(63 - 36)
|
||||
#define CR14_DEGRADATION_SUBMASK _BITUL(63 - 37)
|
||||
|
|
|
@ -812,6 +812,7 @@ struct kvm_arch{
|
|||
int use_irqchip;
|
||||
int use_cmma;
|
||||
int use_pfmfi;
|
||||
int use_skf;
|
||||
int user_cpu_state_ctrl;
|
||||
int user_sigp;
|
||||
int user_stsi;
|
||||
|
|
|
@ -21,7 +21,7 @@ typedef struct {
|
|||
/* The mmu context uses extended page tables. */
|
||||
unsigned int has_pgste:1;
|
||||
/* The mmu context uses storage keys. */
|
||||
unsigned int use_skey:1;
|
||||
unsigned int uses_skeys:1;
|
||||
/* The mmu context uses CMM. */
|
||||
unsigned int uses_cmm:1;
|
||||
} mm_context_t;
|
||||
|
|
|
@ -30,7 +30,7 @@ static inline int init_new_context(struct task_struct *tsk,
|
|||
test_thread_flag(TIF_PGSTE) ||
|
||||
(current->mm && current->mm->context.alloc_pgste);
|
||||
mm->context.has_pgste = 0;
|
||||
mm->context.use_skey = 0;
|
||||
mm->context.uses_skeys = 0;
|
||||
mm->context.uses_cmm = 0;
|
||||
#endif
|
||||
switch (mm->context.asce_limit) {
|
||||
|
|
|
@ -506,10 +506,10 @@ static inline int mm_alloc_pgste(struct mm_struct *mm)
|
|||
* faults should no longer be backed by zero pages
|
||||
*/
|
||||
#define mm_forbids_zeropage mm_has_pgste
|
||||
static inline int mm_use_skey(struct mm_struct *mm)
|
||||
static inline int mm_uses_skeys(struct mm_struct *mm)
|
||||
{
|
||||
#ifdef CONFIG_PGSTE
|
||||
if (mm->context.use_skey)
|
||||
if (mm->context.uses_skeys)
|
||||
return 1;
|
||||
#endif
|
||||
return 0;
|
||||
|
|
|
@ -153,7 +153,7 @@ void kvm_s390_patch_guest_per_regs(struct kvm_vcpu *vcpu)
|
|||
|
||||
if (guestdbg_sstep_enabled(vcpu)) {
|
||||
/* disable timer (clock-comparator) interrupts */
|
||||
vcpu->arch.sie_block->gcr[0] &= ~0x800ul;
|
||||
vcpu->arch.sie_block->gcr[0] &= ~CR0_CLOCK_COMPARATOR_SUBMASK;
|
||||
vcpu->arch.sie_block->gcr[9] |= PER_EVENT_IFETCH;
|
||||
vcpu->arch.sie_block->gcr[10] = 0;
|
||||
vcpu->arch.sie_block->gcr[11] = -1UL;
|
||||
|
|
|
@ -159,7 +159,7 @@ static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
|
|||
static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (psw_extint_disabled(vcpu) ||
|
||||
!(vcpu->arch.sie_block->gcr[0] & 0x800ul))
|
||||
!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
|
||||
return 0;
|
||||
if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
|
||||
/* No timer interrupts when single stepping */
|
||||
|
@ -172,7 +172,7 @@ static int ckc_irq_pending(struct kvm_vcpu *vcpu)
|
|||
const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
|
||||
const u64 ckc = vcpu->arch.sie_block->ckc;
|
||||
|
||||
if (vcpu->arch.sie_block->gcr[0] & 0x0020000000000000ul) {
|
||||
if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
|
||||
if ((s64)ckc >= (s64)now)
|
||||
return 0;
|
||||
} else if (ckc >= now) {
|
||||
|
@ -184,7 +184,7 @@ static int ckc_irq_pending(struct kvm_vcpu *vcpu)
|
|||
static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return !psw_extint_disabled(vcpu) &&
|
||||
(vcpu->arch.sie_block->gcr[0] & 0x400ul);
|
||||
(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
|
||||
}
|
||||
|
||||
static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
|
||||
|
@ -285,15 +285,15 @@ static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
|
|||
active_mask &= ~IRQ_PEND_IO_MASK;
|
||||
else
|
||||
active_mask = disable_iscs(vcpu, active_mask);
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul))
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
|
||||
__clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul))
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
|
||||
__clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul))
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
|
||||
__clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul))
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
|
||||
__clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
|
||||
__clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
|
||||
if (psw_mchk_disabled(vcpu))
|
||||
active_mask &= ~IRQ_PEND_MCHK_MASK;
|
||||
|
@ -1042,7 +1042,7 @@ int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
|
|||
/* external call pending and deliverable */
|
||||
if (kvm_s390_ext_call_pending(vcpu) &&
|
||||
!psw_extint_disabled(vcpu) &&
|
||||
(vcpu->arch.sie_block->gcr[0] & 0x2000ul))
|
||||
(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
|
||||
return 1;
|
||||
|
||||
if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
|
||||
|
@ -1062,7 +1062,7 @@ static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
|
|||
u64 cputm, sltime = 0;
|
||||
|
||||
if (ckc_interrupts_enabled(vcpu)) {
|
||||
if (vcpu->arch.sie_block->gcr[0] & 0x0020000000000000ul) {
|
||||
if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
|
||||
if ((s64)now < (s64)ckc)
|
||||
sltime = tod_to_ns((s64)ckc - (s64)now);
|
||||
} else if (now < ckc) {
|
||||
|
|
|
@ -791,11 +791,21 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att
|
|||
|
||||
static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
|
||||
|
||||
static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
|
||||
void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
|
||||
{
|
||||
struct kvm_vcpu *vcpu;
|
||||
int i;
|
||||
|
||||
kvm_s390_vcpu_block_all(kvm);
|
||||
|
||||
kvm_for_each_vcpu(i, vcpu, kvm)
|
||||
kvm_s390_vcpu_crypto_setup(vcpu);
|
||||
|
||||
kvm_s390_vcpu_unblock_all(kvm);
|
||||
}
|
||||
|
||||
static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
|
||||
{
|
||||
if (!test_kvm_facility(kvm, 76))
|
||||
return -EINVAL;
|
||||
|
||||
|
@ -832,10 +842,7 @@ static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
|
|||
return -ENXIO;
|
||||
}
|
||||
|
||||
kvm_for_each_vcpu(i, vcpu, kvm) {
|
||||
kvm_s390_vcpu_crypto_setup(vcpu);
|
||||
exit_sie(vcpu);
|
||||
}
|
||||
kvm_s390_vcpu_crypto_reset_all(kvm);
|
||||
mutex_unlock(&kvm->lock);
|
||||
return 0;
|
||||
}
|
||||
|
@ -1033,8 +1040,8 @@ static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
|
|||
return ret;
|
||||
}
|
||||
|
||||
static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
|
||||
struct kvm_s390_vm_tod_clock *gtod)
|
||||
static void kvm_s390_get_tod_clock(struct kvm *kvm,
|
||||
struct kvm_s390_vm_tod_clock *gtod)
|
||||
{
|
||||
struct kvm_s390_tod_clock_ext htod;
|
||||
|
||||
|
@ -1043,10 +1050,12 @@ static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
|
|||
get_tod_clock_ext((char *)&htod);
|
||||
|
||||
gtod->tod = htod.tod + kvm->arch.epoch;
|
||||
gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
|
||||
|
||||
if (gtod->tod < htod.tod)
|
||||
gtod->epoch_idx += 1;
|
||||
gtod->epoch_idx = 0;
|
||||
if (test_kvm_facility(kvm, 139)) {
|
||||
gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
|
||||
if (gtod->tod < htod.tod)
|
||||
gtod->epoch_idx += 1;
|
||||
}
|
||||
|
||||
preempt_enable();
|
||||
}
|
||||
|
@ -1056,12 +1065,7 @@ static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
|
|||
struct kvm_s390_vm_tod_clock gtod;
|
||||
|
||||
memset(>od, 0, sizeof(gtod));
|
||||
|
||||
if (test_kvm_facility(kvm, 139))
|
||||
kvm_s390_get_tod_clock_ext(kvm, >od);
|
||||
else
|
||||
gtod.tod = kvm_s390_get_tod_clock_fast(kvm);
|
||||
|
||||
kvm_s390_get_tod_clock(kvm, >od);
|
||||
if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
|
||||
return -EFAULT;
|
||||
|
||||
|
@ -1493,7 +1497,7 @@ static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
|
|||
return -EINVAL;
|
||||
|
||||
/* Is this guest using storage keys? */
|
||||
if (!mm_use_skey(current->mm))
|
||||
if (!mm_uses_skeys(current->mm))
|
||||
return KVM_S390_GET_SKEYS_NONE;
|
||||
|
||||
/* Enforce sane limit on memory allocation */
|
||||
|
@ -1982,10 +1986,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
|
|||
|
||||
rc = -ENOMEM;
|
||||
|
||||
kvm->arch.use_esca = 0; /* start with basic SCA */
|
||||
if (!sclp.has_64bscao)
|
||||
alloc_flags |= GFP_DMA;
|
||||
rwlock_init(&kvm->arch.sca_lock);
|
||||
/* start with basic SCA */
|
||||
kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
|
||||
if (!kvm->arch.sca)
|
||||
goto out_err;
|
||||
|
@ -2036,8 +2040,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
|
|||
kvm_s390_crypto_init(kvm);
|
||||
|
||||
mutex_init(&kvm->arch.float_int.ais_lock);
|
||||
kvm->arch.float_int.simm = 0;
|
||||
kvm->arch.float_int.nimm = 0;
|
||||
spin_lock_init(&kvm->arch.float_int.lock);
|
||||
for (i = 0; i < FIRQ_LIST_COUNT; i++)
|
||||
INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
|
||||
|
@ -2063,11 +2065,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
|
|||
kvm->arch.gmap->pfault_enabled = 0;
|
||||
}
|
||||
|
||||
kvm->arch.css_support = 0;
|
||||
kvm->arch.use_irqchip = 0;
|
||||
kvm->arch.use_pfmfi = sclp.has_pfmfi;
|
||||
kvm->arch.epoch = 0;
|
||||
|
||||
kvm->arch.use_skf = sclp.has_skey;
|
||||
spin_lock_init(&kvm->arch.start_stop_lock);
|
||||
kvm_s390_vsie_init(kvm);
|
||||
kvm_s390_gisa_init(kvm);
|
||||
|
@ -2433,8 +2432,12 @@ static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
|
|||
vcpu->arch.sie_block->ckc = 0UL;
|
||||
vcpu->arch.sie_block->todpr = 0;
|
||||
memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
|
||||
vcpu->arch.sie_block->gcr[0] = 0xE0UL;
|
||||
vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
|
||||
vcpu->arch.sie_block->gcr[0] = CR0_UNUSED_56 |
|
||||
CR0_INTERRUPT_KEY_SUBMASK |
|
||||
CR0_MEASUREMENT_ALERT_SUBMASK;
|
||||
vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 |
|
||||
CR14_UNUSED_33 |
|
||||
CR14_EXTERNAL_DAMAGE_SUBMASK;
|
||||
/* make sure the new fpc will be lazily loaded */
|
||||
save_fpu_regs();
|
||||
current->thread.fpu.fpc = 0;
|
||||
|
@ -3192,7 +3195,7 @@ static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
|
|||
return 0;
|
||||
if (kvm_s390_vcpu_has_irq(vcpu, 0))
|
||||
return 0;
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
|
||||
if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
|
||||
return 0;
|
||||
if (!vcpu->arch.gmap->pfault_enabled)
|
||||
return 0;
|
||||
|
@ -3990,7 +3993,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
|
|||
return r;
|
||||
}
|
||||
|
||||
int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
|
||||
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
|
||||
{
|
||||
#ifdef CONFIG_KVM_S390_UCONTROL
|
||||
if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
|
||||
|
|
|
@ -410,4 +410,17 @@ static inline int kvm_s390_use_sca_entries(void)
|
|||
}
|
||||
void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
|
||||
struct mcck_volatile_info *mcck_info);
|
||||
|
||||
/**
|
||||
* kvm_s390_vcpu_crypto_reset_all
|
||||
*
|
||||
* Reset the crypto attributes for each vcpu. This can be done while the vcpus
|
||||
* are running as each vcpu will be removed from SIE before resetting the crypt
|
||||
* attributes and restored to SIE afterward.
|
||||
*
|
||||
* Note: The kvm->lock must be held while calling this function
|
||||
*
|
||||
* @kvm: the KVM guest
|
||||
*/
|
||||
void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm);
|
||||
#endif
|
||||
|
|
|
@ -204,24 +204,28 @@ static int handle_store_cpu_address(struct kvm_vcpu *vcpu)
|
|||
|
||||
int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
int rc = 0;
|
||||
int rc;
|
||||
struct kvm_s390_sie_block *sie_block = vcpu->arch.sie_block;
|
||||
|
||||
trace_kvm_s390_skey_related_inst(vcpu);
|
||||
if (!(sie_block->ictl & (ICTL_ISKE | ICTL_SSKE | ICTL_RRBE)) &&
|
||||
/* Already enabled? */
|
||||
if (vcpu->kvm->arch.use_skf &&
|
||||
!(sie_block->ictl & (ICTL_ISKE | ICTL_SSKE | ICTL_RRBE)) &&
|
||||
!kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
|
||||
return rc;
|
||||
return 0;
|
||||
|
||||
rc = s390_enable_skey();
|
||||
VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc);
|
||||
if (!rc) {
|
||||
if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
|
||||
kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);
|
||||
else
|
||||
sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE |
|
||||
ICTL_RRBE);
|
||||
}
|
||||
return rc;
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
|
||||
kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);
|
||||
if (!vcpu->kvm->arch.use_skf)
|
||||
sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
|
||||
else
|
||||
sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE | ICTL_RRBE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int try_handle_skey(struct kvm_vcpu *vcpu)
|
||||
|
@ -231,7 +235,7 @@ static int try_handle_skey(struct kvm_vcpu *vcpu)
|
|||
rc = kvm_s390_skey_check_enable(vcpu);
|
||||
if (rc)
|
||||
return rc;
|
||||
if (sclp.has_skey) {
|
||||
if (vcpu->kvm->arch.use_skf) {
|
||||
/* with storage-key facility, SIE interprets it for us */
|
||||
kvm_s390_retry_instr(vcpu);
|
||||
VCPU_EVENT(vcpu, 4, "%s", "retrying storage key operation");
|
||||
|
|
|
@ -557,7 +557,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
|
|||
if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_64BSCAO))
|
||||
gpa |= (u64) READ_ONCE(scb_o->scaoh) << 32;
|
||||
if (gpa) {
|
||||
if (!(gpa & ~0x1fffUL))
|
||||
if (gpa < 2 * PAGE_SIZE)
|
||||
rc = set_validity_icpt(scb_s, 0x0038U);
|
||||
else if ((gpa & ~0x1fffUL) == kvm_s390_get_prefix(vcpu))
|
||||
rc = set_validity_icpt(scb_s, 0x0011U);
|
||||
|
@ -578,7 +578,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
|
|||
|
||||
gpa = READ_ONCE(scb_o->itdba) & ~0xffUL;
|
||||
if (gpa && (scb_s->ecb & ECB_TE)) {
|
||||
if (!(gpa & ~0x1fffUL)) {
|
||||
if (gpa < 2 * PAGE_SIZE) {
|
||||
rc = set_validity_icpt(scb_s, 0x0080U);
|
||||
goto unpin;
|
||||
}
|
||||
|
@ -594,7 +594,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
|
|||
|
||||
gpa = READ_ONCE(scb_o->gvrd) & ~0x1ffUL;
|
||||
if (gpa && (scb_s->eca & ECA_VX) && !(scb_s->ecd & ECD_HOSTREGMGMT)) {
|
||||
if (!(gpa & ~0x1fffUL)) {
|
||||
if (gpa < 2 * PAGE_SIZE) {
|
||||
rc = set_validity_icpt(scb_s, 0x1310U);
|
||||
goto unpin;
|
||||
}
|
||||
|
@ -613,7 +613,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
|
|||
|
||||
gpa = READ_ONCE(scb_o->riccbd) & ~0x3fUL;
|
||||
if (gpa && (scb_s->ecb3 & ECB3_RI)) {
|
||||
if (!(gpa & ~0x1fffUL)) {
|
||||
if (gpa < 2 * PAGE_SIZE) {
|
||||
rc = set_validity_icpt(scb_s, 0x0043U);
|
||||
goto unpin;
|
||||
}
|
||||
|
@ -632,7 +632,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
|
|||
|
||||
gpa = READ_ONCE(scb_o->sdnxo) & ~0xfUL;
|
||||
sdnxc = READ_ONCE(scb_o->sdnxo) & 0xfUL;
|
||||
if (!gpa || !(gpa & ~0x1fffUL)) {
|
||||
if (!gpa || gpa < 2 * PAGE_SIZE) {
|
||||
rc = set_validity_icpt(scb_s, 0x10b0U);
|
||||
goto unpin;
|
||||
}
|
||||
|
|
|
@ -2184,14 +2184,14 @@ int s390_enable_skey(void)
|
|||
int rc = 0;
|
||||
|
||||
down_write(&mm->mmap_sem);
|
||||
if (mm_use_skey(mm))
|
||||
if (mm_uses_skeys(mm))
|
||||
goto out_up;
|
||||
|
||||
mm->context.use_skey = 1;
|
||||
mm->context.uses_skeys = 1;
|
||||
for (vma = mm->mmap; vma; vma = vma->vm_next) {
|
||||
if (ksm_madvise(vma, vma->vm_start, vma->vm_end,
|
||||
MADV_UNMERGEABLE, &vma->vm_flags)) {
|
||||
mm->context.use_skey = 0;
|
||||
mm->context.uses_skeys = 0;
|
||||
rc = -ENOMEM;
|
||||
goto out_up;
|
||||
}
|
||||
|
|
|
@ -158,7 +158,7 @@ static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
|
|||
#ifdef CONFIG_PGSTE
|
||||
unsigned long address, bits, skey;
|
||||
|
||||
if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID)
|
||||
if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID)
|
||||
return pgste;
|
||||
address = pte_val(pte) & PAGE_MASK;
|
||||
skey = (unsigned long) page_get_storage_key(address);
|
||||
|
@ -180,7 +180,7 @@ static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
|
|||
unsigned long address;
|
||||
unsigned long nkey;
|
||||
|
||||
if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID)
|
||||
if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID)
|
||||
return;
|
||||
VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
|
||||
address = pte_val(entry) & PAGE_MASK;
|
||||
|
|
|
@ -13,22 +13,6 @@
|
|||
#define CREATE_TRACE_POINTS
|
||||
#include <asm/trace/hyperv.h>
|
||||
|
||||
/* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */
|
||||
struct hv_flush_pcpu {
|
||||
u64 address_space;
|
||||
u64 flags;
|
||||
u64 processor_mask;
|
||||
u64 gva_list[];
|
||||
};
|
||||
|
||||
/* HvFlushVirtualAddressSpaceEx, HvFlushVirtualAddressListEx hypercalls */
|
||||
struct hv_flush_pcpu_ex {
|
||||
u64 address_space;
|
||||
u64 flags;
|
||||
struct hv_vpset hv_vp_set;
|
||||
u64 gva_list[];
|
||||
};
|
||||
|
||||
/* Each gva in gva_list encodes up to 4096 pages to flush */
|
||||
#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
|
||||
|
||||
|
@ -67,8 +51,8 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus,
|
|||
const struct flush_tlb_info *info)
|
||||
{
|
||||
int cpu, vcpu, gva_n, max_gvas;
|
||||
struct hv_flush_pcpu **flush_pcpu;
|
||||
struct hv_flush_pcpu *flush;
|
||||
struct hv_tlb_flush **flush_pcpu;
|
||||
struct hv_tlb_flush *flush;
|
||||
u64 status = U64_MAX;
|
||||
unsigned long flags;
|
||||
|
||||
|
@ -82,7 +66,7 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus,
|
|||
|
||||
local_irq_save(flags);
|
||||
|
||||
flush_pcpu = (struct hv_flush_pcpu **)
|
||||
flush_pcpu = (struct hv_tlb_flush **)
|
||||
this_cpu_ptr(hyperv_pcpu_input_arg);
|
||||
|
||||
flush = *flush_pcpu;
|
||||
|
@ -152,8 +136,8 @@ static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
|
|||
const struct flush_tlb_info *info)
|
||||
{
|
||||
int nr_bank = 0, max_gvas, gva_n;
|
||||
struct hv_flush_pcpu_ex **flush_pcpu;
|
||||
struct hv_flush_pcpu_ex *flush;
|
||||
struct hv_tlb_flush_ex **flush_pcpu;
|
||||
struct hv_tlb_flush_ex *flush;
|
||||
u64 status = U64_MAX;
|
||||
unsigned long flags;
|
||||
|
||||
|
@ -167,7 +151,7 @@ static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
|
|||
|
||||
local_irq_save(flags);
|
||||
|
||||
flush_pcpu = (struct hv_flush_pcpu_ex **)
|
||||
flush_pcpu = (struct hv_tlb_flush_ex **)
|
||||
this_cpu_ptr(hyperv_pcpu_input_arg);
|
||||
|
||||
flush = *flush_pcpu;
|
||||
|
|
|
@ -308,6 +308,9 @@ struct ms_hyperv_tsc_page {
|
|||
/* TSC emulation after migration */
|
||||
#define HV_X64_MSR_REENLIGHTENMENT_CONTROL 0x40000106
|
||||
|
||||
/* Nested features (CPUID 0x4000000A) EAX */
|
||||
#define HV_X64_NESTED_MSR_BITMAP BIT(19)
|
||||
|
||||
struct hv_reenlightenment_control {
|
||||
__u64 vector:8;
|
||||
__u64 reserved1:8;
|
||||
|
@ -678,7 +681,11 @@ struct hv_enlightened_vmcs {
|
|||
u32 hv_clean_fields;
|
||||
u32 hv_padding_32;
|
||||
u32 hv_synthetic_controls;
|
||||
u32 hv_enlightenments_control;
|
||||
struct {
|
||||
u32 nested_flush_hypercall:1;
|
||||
u32 msr_bitmap:1;
|
||||
u32 reserved:30;
|
||||
} hv_enlightenments_control;
|
||||
u32 hv_vp_id;
|
||||
|
||||
u64 hv_vm_id;
|
||||
|
@ -734,4 +741,20 @@ struct ipi_arg_ex {
|
|||
struct hv_vpset vp_set;
|
||||
};
|
||||
|
||||
/* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */
|
||||
struct hv_tlb_flush {
|
||||
u64 address_space;
|
||||
u64 flags;
|
||||
u64 processor_mask;
|
||||
u64 gva_list[];
|
||||
};
|
||||
|
||||
/* HvFlushVirtualAddressSpaceEx, HvFlushVirtualAddressListEx hypercalls */
|
||||
struct hv_tlb_flush_ex {
|
||||
u64 address_space;
|
||||
u64 flags;
|
||||
struct hv_vpset hv_vp_set;
|
||||
u64 gva_list[];
|
||||
};
|
||||
|
||||
#endif
|
||||
|
|
|
@ -107,11 +107,12 @@ struct x86_emulate_ops {
|
|||
* @addr: [IN ] Linear address from which to read.
|
||||
* @val: [OUT] Value read from memory, zero-extended to 'u_long'.
|
||||
* @bytes: [IN ] Number of bytes to read from memory.
|
||||
* @system:[IN ] Whether the access is forced to be at CPL0.
|
||||
*/
|
||||
int (*read_std)(struct x86_emulate_ctxt *ctxt,
|
||||
unsigned long addr, void *val,
|
||||
unsigned int bytes,
|
||||
struct x86_exception *fault);
|
||||
struct x86_exception *fault, bool system);
|
||||
|
||||
/*
|
||||
* read_phys: Read bytes of standard (non-emulated/special) memory.
|
||||
|
@ -129,10 +130,11 @@ struct x86_emulate_ops {
|
|||
* @addr: [IN ] Linear address to which to write.
|
||||
* @val: [OUT] Value write to memory, zero-extended to 'u_long'.
|
||||
* @bytes: [IN ] Number of bytes to write to memory.
|
||||
* @system:[IN ] Whether the access is forced to be at CPL0.
|
||||
*/
|
||||
int (*write_std)(struct x86_emulate_ctxt *ctxt,
|
||||
unsigned long addr, void *val, unsigned int bytes,
|
||||
struct x86_exception *fault);
|
||||
struct x86_exception *fault, bool system);
|
||||
/*
|
||||
* fetch: Read bytes of standard (non-emulated/special) memory.
|
||||
* Used for instruction fetch.
|
||||
|
|
|
@ -258,7 +258,8 @@ union kvm_mmu_page_role {
|
|||
unsigned smep_andnot_wp:1;
|
||||
unsigned smap_andnot_wp:1;
|
||||
unsigned ad_disabled:1;
|
||||
unsigned :7;
|
||||
unsigned guest_mode:1;
|
||||
unsigned :6;
|
||||
|
||||
/*
|
||||
* This is left at the top of the word so that
|
||||
|
@ -476,6 +477,7 @@ struct kvm_vcpu_hv {
|
|||
struct kvm_hyperv_exit exit;
|
||||
struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT];
|
||||
DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
|
||||
cpumask_t tlb_lush;
|
||||
};
|
||||
|
||||
struct kvm_vcpu_arch {
|
||||
|
@ -995,7 +997,7 @@ struct kvm_x86_ops {
|
|||
void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr);
|
||||
void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr);
|
||||
void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
|
||||
void (*set_virtual_x2apic_mode)(struct kvm_vcpu *vcpu, bool set);
|
||||
void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu);
|
||||
void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa);
|
||||
void (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector);
|
||||
int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu);
|
||||
|
@ -1277,6 +1279,7 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
|
|||
int kvm_mmu_load(struct kvm_vcpu *vcpu);
|
||||
void kvm_mmu_unload(struct kvm_vcpu *vcpu);
|
||||
void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
|
||||
void kvm_mmu_free_roots(struct kvm_vcpu *vcpu);
|
||||
gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
|
||||
struct x86_exception *exception);
|
||||
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
|
||||
|
|
|
@ -269,7 +269,7 @@ static inline int cpumask_to_vpset(struct hv_vpset *vpset,
|
|||
return 0;
|
||||
|
||||
/*
|
||||
* Clear all banks up to the maximum possible bank as hv_flush_pcpu_ex
|
||||
* Clear all banks up to the maximum possible bank as hv_tlb_flush_ex
|
||||
* structs are not cleared between calls, we risk flushing unneeded
|
||||
* vCPUs otherwise.
|
||||
*/
|
||||
|
|
|
@ -207,7 +207,9 @@ enum vmcs_field {
|
|||
EPTP_LIST_ADDRESS = 0x00002024,
|
||||
EPTP_LIST_ADDRESS_HIGH = 0x00002025,
|
||||
VMREAD_BITMAP = 0x00002026,
|
||||
VMREAD_BITMAP_HIGH = 0x00002027,
|
||||
VMWRITE_BITMAP = 0x00002028,
|
||||
VMWRITE_BITMAP_HIGH = 0x00002029,
|
||||
XSS_EXIT_BITMAP = 0x0000202C,
|
||||
XSS_EXIT_BITMAP_HIGH = 0x0000202D,
|
||||
TSC_MULTIPLIER = 0x00002032,
|
||||
|
|
|
@ -404,7 +404,8 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
|
|||
const u32 kvm_cpuid_7_0_ecx_x86_features =
|
||||
F(AVX512VBMI) | F(LA57) | F(PKU) | 0 /*OSPKE*/ |
|
||||
F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) |
|
||||
F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG);
|
||||
F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) |
|
||||
F(CLDEMOTE);
|
||||
|
||||
/* cpuid 7.0.edx*/
|
||||
const u32 kvm_cpuid_7_0_edx_x86_features =
|
||||
|
|
|
@ -812,6 +812,19 @@ static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
|
|||
return assign_eip_near(ctxt, ctxt->_eip + rel);
|
||||
}
|
||||
|
||||
static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
|
||||
void *data, unsigned size)
|
||||
{
|
||||
return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
|
||||
}
|
||||
|
||||
static int linear_write_system(struct x86_emulate_ctxt *ctxt,
|
||||
ulong linear, void *data,
|
||||
unsigned int size)
|
||||
{
|
||||
return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
|
||||
}
|
||||
|
||||
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
|
||||
struct segmented_address addr,
|
||||
void *data,
|
||||
|
@ -823,7 +836,7 @@ static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
|
|||
rc = linearize(ctxt, addr, size, false, &linear);
|
||||
if (rc != X86EMUL_CONTINUE)
|
||||
return rc;
|
||||
return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
|
||||
return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
|
||||
}
|
||||
|
||||
static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
|
||||
|
@ -837,7 +850,7 @@ static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
|
|||
rc = linearize(ctxt, addr, size, true, &linear);
|
||||
if (rc != X86EMUL_CONTINUE)
|
||||
return rc;
|
||||
return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception);
|
||||
return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -1496,8 +1509,7 @@ static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
|
|||
return emulate_gp(ctxt, index << 3 | 0x2);
|
||||
|
||||
addr = dt.address + index * 8;
|
||||
return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
|
||||
&ctxt->exception);
|
||||
return linear_read_system(ctxt, addr, desc, sizeof *desc);
|
||||
}
|
||||
|
||||
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
|
||||
|
@ -1560,8 +1572,7 @@ static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
|
|||
if (rc != X86EMUL_CONTINUE)
|
||||
return rc;
|
||||
|
||||
return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
|
||||
&ctxt->exception);
|
||||
return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
|
||||
}
|
||||
|
||||
/* allowed just for 8 bytes segments */
|
||||
|
@ -1575,8 +1586,7 @@ static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
|
|||
if (rc != X86EMUL_CONTINUE)
|
||||
return rc;
|
||||
|
||||
return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
|
||||
&ctxt->exception);
|
||||
return linear_write_system(ctxt, addr, desc, sizeof *desc);
|
||||
}
|
||||
|
||||
static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
|
||||
|
@ -1737,8 +1747,7 @@ static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
|
|||
return ret;
|
||||
}
|
||||
} else if (ctxt->mode == X86EMUL_MODE_PROT64) {
|
||||
ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
|
||||
sizeof(base3), &ctxt->exception);
|
||||
ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
|
||||
if (ret != X86EMUL_CONTINUE)
|
||||
return ret;
|
||||
if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
|
||||
|
@ -2051,11 +2060,11 @@ static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
|
|||
eip_addr = dt.address + (irq << 2);
|
||||
cs_addr = dt.address + (irq << 2) + 2;
|
||||
|
||||
rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
|
||||
rc = linear_read_system(ctxt, cs_addr, &cs, 2);
|
||||
if (rc != X86EMUL_CONTINUE)
|
||||
return rc;
|
||||
|
||||
rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
|
||||
rc = linear_read_system(ctxt, eip_addr, &eip, 2);
|
||||
if (rc != X86EMUL_CONTINUE)
|
||||
return rc;
|
||||
|
||||
|
@ -2919,12 +2928,12 @@ static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
|
|||
#ifdef CONFIG_X86_64
|
||||
base |= ((u64)base3) << 32;
|
||||
#endif
|
||||
r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
|
||||
r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
|
||||
if (r != X86EMUL_CONTINUE)
|
||||
return false;
|
||||
if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
|
||||
return false;
|
||||
r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
|
||||
r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
|
||||
if (r != X86EMUL_CONTINUE)
|
||||
return false;
|
||||
if ((perm >> bit_idx) & mask)
|
||||
|
@ -3053,35 +3062,30 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
|
|||
u16 tss_selector, u16 old_tss_sel,
|
||||
ulong old_tss_base, struct desc_struct *new_desc)
|
||||
{
|
||||
const struct x86_emulate_ops *ops = ctxt->ops;
|
||||
struct tss_segment_16 tss_seg;
|
||||
int ret;
|
||||
u32 new_tss_base = get_desc_base(new_desc);
|
||||
|
||||
ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
|
||||
&ctxt->exception);
|
||||
ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
|
||||
if (ret != X86EMUL_CONTINUE)
|
||||
return ret;
|
||||
|
||||
save_state_to_tss16(ctxt, &tss_seg);
|
||||
|
||||
ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
|
||||
&ctxt->exception);
|
||||
ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
|
||||
if (ret != X86EMUL_CONTINUE)
|
||||
return ret;
|
||||
|
||||
ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
|
||||
&ctxt->exception);
|
||||
ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
|
||||
if (ret != X86EMUL_CONTINUE)
|
||||
return ret;
|
||||
|
||||
if (old_tss_sel != 0xffff) {
|
||||
tss_seg.prev_task_link = old_tss_sel;
|
||||
|
||||
ret = ops->write_std(ctxt, new_tss_base,
|
||||
&tss_seg.prev_task_link,
|
||||
sizeof tss_seg.prev_task_link,
|
||||
&ctxt->exception);
|
||||
ret = linear_write_system(ctxt, new_tss_base,
|
||||
&tss_seg.prev_task_link,
|
||||
sizeof tss_seg.prev_task_link);
|
||||
if (ret != X86EMUL_CONTINUE)
|
||||
return ret;
|
||||
}
|
||||
|
@ -3197,38 +3201,34 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
|
|||
u16 tss_selector, u16 old_tss_sel,
|
||||
ulong old_tss_base, struct desc_struct *new_desc)
|
||||
{
|
||||
const struct x86_emulate_ops *ops = ctxt->ops;
|
||||
struct tss_segment_32 tss_seg;
|
||||
int ret;
|
||||
u32 new_tss_base = get_desc_base(new_desc);
|
||||
u32 eip_offset = offsetof(struct tss_segment_32, eip);
|
||||
u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
|
||||
|
||||
ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
|
||||
&ctxt->exception);
|
||||
ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
|
||||
if (ret != X86EMUL_CONTINUE)
|
||||
return ret;
|
||||
|
||||
save_state_to_tss32(ctxt, &tss_seg);
|
||||
|
||||
/* Only GP registers and segment selectors are saved */
|
||||
ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
|
||||
ldt_sel_offset - eip_offset, &ctxt->exception);
|
||||
ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
|
||||
ldt_sel_offset - eip_offset);
|
||||
if (ret != X86EMUL_CONTINUE)
|
||||
return ret;
|
||||
|
||||
ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
|
||||
&ctxt->exception);
|
||||
ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
|
||||
if (ret != X86EMUL_CONTINUE)
|
||||
return ret;
|
||||
|
||||
if (old_tss_sel != 0xffff) {
|
||||
tss_seg.prev_task_link = old_tss_sel;
|
||||
|
||||
ret = ops->write_std(ctxt, new_tss_base,
|
||||
&tss_seg.prev_task_link,
|
||||
sizeof tss_seg.prev_task_link,
|
||||
&ctxt->exception);
|
||||
ret = linear_write_system(ctxt, new_tss_base,
|
||||
&tss_seg.prev_task_link,
|
||||
sizeof tss_seg.prev_task_link);
|
||||
if (ret != X86EMUL_CONTINUE)
|
||||
return ret;
|
||||
}
|
||||
|
@ -4189,7 +4189,9 @@ static int check_cr_write(struct x86_emulate_ctxt *ctxt)
|
|||
maxphyaddr = eax & 0xff;
|
||||
else
|
||||
maxphyaddr = 36;
|
||||
rsvd = rsvd_bits(maxphyaddr, 62);
|
||||
rsvd = rsvd_bits(maxphyaddr, 63);
|
||||
if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
|
||||
rsvd &= ~CR3_PCID_INVD;
|
||||
}
|
||||
|
||||
if (new_val & rsvd)
|
||||
|
|
|
@ -1242,6 +1242,121 @@ int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
|
|||
return kvm_hv_get_msr(vcpu, msr, pdata);
|
||||
}
|
||||
|
||||
static __always_inline int get_sparse_bank_no(u64 valid_bank_mask, int bank_no)
|
||||
{
|
||||
int i = 0, j;
|
||||
|
||||
if (!(valid_bank_mask & BIT_ULL(bank_no)))
|
||||
return -1;
|
||||
|
||||
for (j = 0; j < bank_no; j++)
|
||||
if (valid_bank_mask & BIT_ULL(j))
|
||||
i++;
|
||||
|
||||
return i;
|
||||
}
|
||||
|
||||
static u64 kvm_hv_flush_tlb(struct kvm_vcpu *current_vcpu, u64 ingpa,
|
||||
u16 rep_cnt, bool ex)
|
||||
{
|
||||
struct kvm *kvm = current_vcpu->kvm;
|
||||
struct kvm_vcpu_hv *hv_current = ¤t_vcpu->arch.hyperv;
|
||||
struct hv_tlb_flush_ex flush_ex;
|
||||
struct hv_tlb_flush flush;
|
||||
struct kvm_vcpu *vcpu;
|
||||
unsigned long vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)] = {0};
|
||||
unsigned long valid_bank_mask = 0;
|
||||
u64 sparse_banks[64];
|
||||
int sparse_banks_len, i;
|
||||
bool all_cpus;
|
||||
|
||||
if (!ex) {
|
||||
if (unlikely(kvm_read_guest(kvm, ingpa, &flush, sizeof(flush))))
|
||||
return HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
|
||||
trace_kvm_hv_flush_tlb(flush.processor_mask,
|
||||
flush.address_space, flush.flags);
|
||||
|
||||
sparse_banks[0] = flush.processor_mask;
|
||||
all_cpus = flush.flags & HV_FLUSH_ALL_PROCESSORS;
|
||||
} else {
|
||||
if (unlikely(kvm_read_guest(kvm, ingpa, &flush_ex,
|
||||
sizeof(flush_ex))))
|
||||
return HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
|
||||
trace_kvm_hv_flush_tlb_ex(flush_ex.hv_vp_set.valid_bank_mask,
|
||||
flush_ex.hv_vp_set.format,
|
||||
flush_ex.address_space,
|
||||
flush_ex.flags);
|
||||
|
||||
valid_bank_mask = flush_ex.hv_vp_set.valid_bank_mask;
|
||||
all_cpus = flush_ex.hv_vp_set.format !=
|
||||
HV_GENERIC_SET_SPARSE_4K;
|
||||
|
||||
sparse_banks_len = bitmap_weight(&valid_bank_mask, 64) *
|
||||
sizeof(sparse_banks[0]);
|
||||
|
||||
if (!sparse_banks_len && !all_cpus)
|
||||
goto ret_success;
|
||||
|
||||
if (!all_cpus &&
|
||||
kvm_read_guest(kvm,
|
||||
ingpa + offsetof(struct hv_tlb_flush_ex,
|
||||
hv_vp_set.bank_contents),
|
||||
sparse_banks,
|
||||
sparse_banks_len))
|
||||
return HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
}
|
||||
|
||||
cpumask_clear(&hv_current->tlb_lush);
|
||||
|
||||
kvm_for_each_vcpu(i, vcpu, kvm) {
|
||||
struct kvm_vcpu_hv *hv = &vcpu->arch.hyperv;
|
||||
int bank = hv->vp_index / 64, sbank = 0;
|
||||
|
||||
if (!all_cpus) {
|
||||
/* Banks >64 can't be represented */
|
||||
if (bank >= 64)
|
||||
continue;
|
||||
|
||||
/* Non-ex hypercalls can only address first 64 vCPUs */
|
||||
if (!ex && bank)
|
||||
continue;
|
||||
|
||||
if (ex) {
|
||||
/*
|
||||
* Check is the bank of this vCPU is in sparse
|
||||
* set and get the sparse bank number.
|
||||
*/
|
||||
sbank = get_sparse_bank_no(valid_bank_mask,
|
||||
bank);
|
||||
|
||||
if (sbank < 0)
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!(sparse_banks[sbank] & BIT_ULL(hv->vp_index % 64)))
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* vcpu->arch.cr3 may not be up-to-date for running vCPUs so we
|
||||
* can't analyze it here, flush TLB regardless of the specified
|
||||
* address space.
|
||||
*/
|
||||
__set_bit(i, vcpu_bitmap);
|
||||
}
|
||||
|
||||
kvm_make_vcpus_request_mask(kvm,
|
||||
KVM_REQ_TLB_FLUSH | KVM_REQUEST_NO_WAKEUP,
|
||||
vcpu_bitmap, &hv_current->tlb_lush);
|
||||
|
||||
ret_success:
|
||||
/* We always do full TLB flush, set rep_done = rep_cnt. */
|
||||
return (u64)HV_STATUS_SUCCESS |
|
||||
((u64)rep_cnt << HV_HYPERCALL_REP_COMP_OFFSET);
|
||||
}
|
||||
|
||||
bool kvm_hv_hypercall_enabled(struct kvm *kvm)
|
||||
{
|
||||
return READ_ONCE(kvm->arch.hyperv.hv_hypercall) & HV_X64_MSR_HYPERCALL_ENABLE;
|
||||
|
@ -1315,7 +1430,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
|
|||
{
|
||||
u64 param, ingpa, outgpa, ret = HV_STATUS_SUCCESS;
|
||||
uint16_t code, rep_idx, rep_cnt;
|
||||
bool fast, longmode;
|
||||
bool fast, longmode, rep;
|
||||
|
||||
/*
|
||||
* hypercall generates UD from non zero cpl and real mode
|
||||
|
@ -1345,31 +1460,34 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
|
|||
#endif
|
||||
|
||||
code = param & 0xffff;
|
||||
fast = (param >> 16) & 0x1;
|
||||
rep_cnt = (param >> 32) & 0xfff;
|
||||
rep_idx = (param >> 48) & 0xfff;
|
||||
fast = !!(param & HV_HYPERCALL_FAST_BIT);
|
||||
rep_cnt = (param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff;
|
||||
rep_idx = (param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff;
|
||||
rep = !!(rep_cnt || rep_idx);
|
||||
|
||||
trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa);
|
||||
|
||||
/* Hypercall continuation is not supported yet */
|
||||
if (rep_cnt || rep_idx) {
|
||||
ret = HV_STATUS_INVALID_HYPERCALL_CODE;
|
||||
goto out;
|
||||
}
|
||||
|
||||
switch (code) {
|
||||
case HVCALL_NOTIFY_LONG_SPIN_WAIT:
|
||||
if (unlikely(rep)) {
|
||||
ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
break;
|
||||
}
|
||||
kvm_vcpu_on_spin(vcpu, true);
|
||||
break;
|
||||
case HVCALL_SIGNAL_EVENT:
|
||||
if (unlikely(rep)) {
|
||||
ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
break;
|
||||
}
|
||||
ret = kvm_hvcall_signal_event(vcpu, fast, ingpa);
|
||||
if (ret != HV_STATUS_INVALID_PORT_ID)
|
||||
break;
|
||||
/* maybe userspace knows this conn_id: fall through */
|
||||
case HVCALL_POST_MESSAGE:
|
||||
/* don't bother userspace if it has no way to handle it */
|
||||
if (!vcpu_to_synic(vcpu)->active) {
|
||||
ret = HV_STATUS_INVALID_HYPERCALL_CODE;
|
||||
if (unlikely(rep || !vcpu_to_synic(vcpu)->active)) {
|
||||
ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
break;
|
||||
}
|
||||
vcpu->run->exit_reason = KVM_EXIT_HYPERV;
|
||||
|
@ -1380,12 +1498,39 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
|
|||
vcpu->arch.complete_userspace_io =
|
||||
kvm_hv_hypercall_complete_userspace;
|
||||
return 0;
|
||||
case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST:
|
||||
if (unlikely(fast || !rep_cnt || rep_idx)) {
|
||||
ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
break;
|
||||
}
|
||||
ret = kvm_hv_flush_tlb(vcpu, ingpa, rep_cnt, false);
|
||||
break;
|
||||
case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE:
|
||||
if (unlikely(fast || rep)) {
|
||||
ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
break;
|
||||
}
|
||||
ret = kvm_hv_flush_tlb(vcpu, ingpa, rep_cnt, false);
|
||||
break;
|
||||
case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX:
|
||||
if (unlikely(fast || !rep_cnt || rep_idx)) {
|
||||
ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
break;
|
||||
}
|
||||
ret = kvm_hv_flush_tlb(vcpu, ingpa, rep_cnt, true);
|
||||
break;
|
||||
case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX:
|
||||
if (unlikely(fast || rep)) {
|
||||
ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
break;
|
||||
}
|
||||
ret = kvm_hv_flush_tlb(vcpu, ingpa, rep_cnt, true);
|
||||
break;
|
||||
default:
|
||||
ret = HV_STATUS_INVALID_HYPERCALL_CODE;
|
||||
break;
|
||||
}
|
||||
|
||||
out:
|
||||
return kvm_hv_hypercall_complete(vcpu, ret);
|
||||
}
|
||||
|
||||
|
|
|
@ -2002,13 +2002,11 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
|
|||
}
|
||||
}
|
||||
|
||||
if ((old_value ^ value) & X2APIC_ENABLE) {
|
||||
if (value & X2APIC_ENABLE) {
|
||||
kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
|
||||
kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true);
|
||||
} else
|
||||
kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false);
|
||||
}
|
||||
if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE))
|
||||
kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
|
||||
|
||||
if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
|
||||
kvm_x86_ops->set_virtual_apic_mode(vcpu);
|
||||
|
||||
apic->base_address = apic->vcpu->arch.apic_base &
|
||||
MSR_IA32_APICBASE_BASE;
|
||||
|
|
|
@ -16,6 +16,13 @@
|
|||
#define APIC_BUS_CYCLE_NS 1
|
||||
#define APIC_BUS_FREQUENCY (1000000000ULL / APIC_BUS_CYCLE_NS)
|
||||
|
||||
enum lapic_mode {
|
||||
LAPIC_MODE_DISABLED = 0,
|
||||
LAPIC_MODE_INVALID = X2APIC_ENABLE,
|
||||
LAPIC_MODE_XAPIC = MSR_IA32_APICBASE_ENABLE,
|
||||
LAPIC_MODE_X2APIC = MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE,
|
||||
};
|
||||
|
||||
struct kvm_timer {
|
||||
struct hrtimer timer;
|
||||
s64 period; /* unit: ns */
|
||||
|
@ -89,6 +96,7 @@ u64 kvm_get_apic_base(struct kvm_vcpu *vcpu);
|
|||
int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info);
|
||||
int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s);
|
||||
int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s);
|
||||
enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu);
|
||||
int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu);
|
||||
|
||||
u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu);
|
||||
|
@ -220,4 +228,10 @@ void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu);
|
|||
void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu);
|
||||
bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu);
|
||||
void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu);
|
||||
|
||||
static inline enum lapic_mode kvm_apic_mode(u64 apic_base)
|
||||
{
|
||||
return apic_base & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
@ -222,7 +222,6 @@ static const u64 shadow_acc_track_saved_bits_mask = PT64_EPT_READABLE_MASK |
|
|||
static const u64 shadow_acc_track_saved_bits_shift = PT64_SECOND_AVAIL_BITS_SHIFT;
|
||||
|
||||
static void mmu_spte_set(u64 *sptep, u64 spte);
|
||||
static void mmu_free_roots(struct kvm_vcpu *vcpu);
|
||||
|
||||
void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value)
|
||||
{
|
||||
|
@ -3343,51 +3342,48 @@ out_unlock:
|
|||
return RET_PF_RETRY;
|
||||
}
|
||||
|
||||
static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa,
|
||||
struct list_head *invalid_list)
|
||||
{
|
||||
struct kvm_mmu_page *sp;
|
||||
|
||||
static void mmu_free_roots(struct kvm_vcpu *vcpu)
|
||||
if (!VALID_PAGE(*root_hpa))
|
||||
return;
|
||||
|
||||
sp = page_header(*root_hpa & PT64_BASE_ADDR_MASK);
|
||||
--sp->root_count;
|
||||
if (!sp->root_count && sp->role.invalid)
|
||||
kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
|
||||
|
||||
*root_hpa = INVALID_PAGE;
|
||||
}
|
||||
|
||||
void kvm_mmu_free_roots(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
int i;
|
||||
struct kvm_mmu_page *sp;
|
||||
LIST_HEAD(invalid_list);
|
||||
struct kvm_mmu *mmu = &vcpu->arch.mmu;
|
||||
|
||||
if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
|
||||
if (!VALID_PAGE(mmu->root_hpa))
|
||||
return;
|
||||
|
||||
if (vcpu->arch.mmu.shadow_root_level >= PT64_ROOT_4LEVEL &&
|
||||
(vcpu->arch.mmu.root_level >= PT64_ROOT_4LEVEL ||
|
||||
vcpu->arch.mmu.direct_map)) {
|
||||
hpa_t root = vcpu->arch.mmu.root_hpa;
|
||||
|
||||
spin_lock(&vcpu->kvm->mmu_lock);
|
||||
sp = page_header(root);
|
||||
--sp->root_count;
|
||||
if (!sp->root_count && sp->role.invalid) {
|
||||
kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
|
||||
kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
|
||||
}
|
||||
spin_unlock(&vcpu->kvm->mmu_lock);
|
||||
vcpu->arch.mmu.root_hpa = INVALID_PAGE;
|
||||
return;
|
||||
}
|
||||
|
||||
spin_lock(&vcpu->kvm->mmu_lock);
|
||||
for (i = 0; i < 4; ++i) {
|
||||
hpa_t root = vcpu->arch.mmu.pae_root[i];
|
||||
|
||||
if (root) {
|
||||
root &= PT64_BASE_ADDR_MASK;
|
||||
sp = page_header(root);
|
||||
--sp->root_count;
|
||||
if (!sp->root_count && sp->role.invalid)
|
||||
kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
|
||||
&invalid_list);
|
||||
}
|
||||
vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
|
||||
if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL &&
|
||||
(mmu->root_level >= PT64_ROOT_4LEVEL || mmu->direct_map)) {
|
||||
mmu_free_root_page(vcpu->kvm, &mmu->root_hpa, &invalid_list);
|
||||
} else {
|
||||
for (i = 0; i < 4; ++i)
|
||||
if (mmu->pae_root[i] != 0)
|
||||
mmu_free_root_page(vcpu->kvm, &mmu->pae_root[i],
|
||||
&invalid_list);
|
||||
mmu->root_hpa = INVALID_PAGE;
|
||||
}
|
||||
|
||||
kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
|
||||
spin_unlock(&vcpu->kvm->mmu_lock);
|
||||
vcpu->arch.mmu.root_hpa = INVALID_PAGE;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);
|
||||
|
||||
static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
|
||||
{
|
||||
|
@ -3720,7 +3716,6 @@ static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
|
|||
*/
|
||||
return RET_PF_RETRY;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(handle_mmio_page_fault);
|
||||
|
||||
static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
|
||||
u32 error_code, gfn_t gfn)
|
||||
|
@ -3812,6 +3807,14 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
|
|||
struct kvm_memory_slot *slot;
|
||||
bool async;
|
||||
|
||||
/*
|
||||
* Don't expose private memslots to L2.
|
||||
*/
|
||||
if (is_guest_mode(vcpu) && !kvm_is_visible_gfn(vcpu->kvm, gfn)) {
|
||||
*pfn = KVM_PFN_NOSLOT;
|
||||
return false;
|
||||
}
|
||||
|
||||
slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
|
||||
async = false;
|
||||
*pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable);
|
||||
|
@ -3951,7 +3954,7 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu,
|
|||
|
||||
void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
mmu_free_roots(vcpu);
|
||||
kvm_mmu_free_roots(vcpu);
|
||||
}
|
||||
|
||||
static unsigned long get_cr3(struct kvm_vcpu *vcpu)
|
||||
|
@ -4473,6 +4476,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
|
|||
struct kvm_mmu *context = &vcpu->arch.mmu;
|
||||
|
||||
context->base_role.word = 0;
|
||||
context->base_role.guest_mode = is_guest_mode(vcpu);
|
||||
context->base_role.smm = is_smm(vcpu);
|
||||
context->base_role.ad_disabled = (shadow_accessed_mask == 0);
|
||||
context->page_fault = tdp_page_fault;
|
||||
|
@ -4539,6 +4543,7 @@ void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu)
|
|||
= smep && !is_write_protection(vcpu);
|
||||
context->base_role.smap_andnot_wp
|
||||
= smap && !is_write_protection(vcpu);
|
||||
context->base_role.guest_mode = is_guest_mode(vcpu);
|
||||
context->base_role.smm = is_smm(vcpu);
|
||||
reset_shadow_zero_bits_mask(vcpu, context);
|
||||
}
|
||||
|
@ -4564,7 +4569,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
|
|||
context->root_hpa = INVALID_PAGE;
|
||||
context->direct_map = false;
|
||||
context->base_role.ad_disabled = !accessed_dirty;
|
||||
|
||||
context->base_role.guest_mode = 1;
|
||||
update_permission_bitmask(vcpu, context, true);
|
||||
update_pkru_bitmask(vcpu, context, true);
|
||||
update_last_nonleaf_level(vcpu, context);
|
||||
|
@ -4664,7 +4669,7 @@ EXPORT_SYMBOL_GPL(kvm_mmu_load);
|
|||
|
||||
void kvm_mmu_unload(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
mmu_free_roots(vcpu);
|
||||
kvm_mmu_free_roots(vcpu);
|
||||
WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kvm_mmu_unload);
|
||||
|
@ -4825,6 +4830,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
|
|||
mask.smep_andnot_wp = 1;
|
||||
mask.smap_andnot_wp = 1;
|
||||
mask.smm = 1;
|
||||
mask.guest_mode = 1;
|
||||
mask.ad_disabled = 1;
|
||||
|
||||
/*
|
||||
|
|
|
@ -1768,7 +1768,10 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
|
|||
unsigned long npages, npinned, size;
|
||||
unsigned long locked, lock_limit;
|
||||
struct page **pages;
|
||||
int first, last;
|
||||
unsigned long first, last;
|
||||
|
||||
if (ulen == 0 || uaddr + ulen < uaddr)
|
||||
return NULL;
|
||||
|
||||
/* Calculate number of pages. */
|
||||
first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
|
||||
|
@ -1855,13 +1858,13 @@ static void __unregister_enc_region_locked(struct kvm *kvm,
|
|||
|
||||
static struct kvm *svm_vm_alloc(void)
|
||||
{
|
||||
struct kvm_svm *kvm_svm = kzalloc(sizeof(struct kvm_svm), GFP_KERNEL);
|
||||
struct kvm_svm *kvm_svm = vzalloc(sizeof(struct kvm_svm));
|
||||
return &kvm_svm->kvm;
|
||||
}
|
||||
|
||||
static void svm_vm_free(struct kvm *kvm)
|
||||
{
|
||||
kfree(to_kvm_svm(kvm));
|
||||
vfree(to_kvm_svm(kvm));
|
||||
}
|
||||
|
||||
static void sev_vm_destroy(struct kvm *kvm)
|
||||
|
@ -5062,7 +5065,7 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
|
|||
set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
|
||||
}
|
||||
|
||||
static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
|
||||
static void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
@ -6949,6 +6952,9 @@ static int svm_register_enc_region(struct kvm *kvm,
|
|||
if (!sev_guest(kvm))
|
||||
return -ENOTTY;
|
||||
|
||||
if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
|
||||
return -EINVAL;
|
||||
|
||||
region = kzalloc(sizeof(*region), GFP_KERNEL);
|
||||
if (!region)
|
||||
return -ENOMEM;
|
||||
|
@ -7100,7 +7106,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
|
|||
.enable_nmi_window = enable_nmi_window,
|
||||
.enable_irq_window = enable_irq_window,
|
||||
.update_cr8_intercept = update_cr8_intercept,
|
||||
.set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
|
||||
.set_virtual_apic_mode = svm_set_virtual_apic_mode,
|
||||
.get_enable_apicv = svm_get_enable_apicv,
|
||||
.refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
|
||||
.load_eoi_exitmap = svm_load_eoi_exitmap,
|
||||
|
|
|
@ -1367,6 +1367,57 @@ TRACE_EVENT(kvm_hv_timer_state,
|
|||
__entry->vcpu_id,
|
||||
__entry->hv_timer_in_use)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for kvm_hv_flush_tlb.
|
||||
*/
|
||||
TRACE_EVENT(kvm_hv_flush_tlb,
|
||||
TP_PROTO(u64 processor_mask, u64 address_space, u64 flags),
|
||||
TP_ARGS(processor_mask, address_space, flags),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(u64, processor_mask)
|
||||
__field(u64, address_space)
|
||||
__field(u64, flags)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->processor_mask = processor_mask;
|
||||
__entry->address_space = address_space;
|
||||
__entry->flags = flags;
|
||||
),
|
||||
|
||||
TP_printk("processor_mask 0x%llx address_space 0x%llx flags 0x%llx",
|
||||
__entry->processor_mask, __entry->address_space,
|
||||
__entry->flags)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for kvm_hv_flush_tlb_ex.
|
||||
*/
|
||||
TRACE_EVENT(kvm_hv_flush_tlb_ex,
|
||||
TP_PROTO(u64 valid_bank_mask, u64 format, u64 address_space, u64 flags),
|
||||
TP_ARGS(valid_bank_mask, format, address_space, flags),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(u64, valid_bank_mask)
|
||||
__field(u64, format)
|
||||
__field(u64, address_space)
|
||||
__field(u64, flags)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->valid_bank_mask = valid_bank_mask;
|
||||
__entry->format = format;
|
||||
__entry->address_space = address_space;
|
||||
__entry->flags = flags;
|
||||
),
|
||||
|
||||
TP_printk("valid_bank_mask 0x%llx format 0x%llx "
|
||||
"address_space 0x%llx flags 0x%llx",
|
||||
__entry->valid_bank_mask, __entry->format,
|
||||
__entry->address_space, __entry->flags)
|
||||
);
|
||||
#endif /* _TRACE_KVM_H */
|
||||
|
||||
#undef TRACE_INCLUDE_PATH
|
||||
|
|
|
@ -242,7 +242,11 @@ struct shared_msr_entry {
|
|||
* underlying hardware which will be used to run L2.
|
||||
* This structure is packed to ensure that its layout is identical across
|
||||
* machines (necessary for live migration).
|
||||
* If there are changes in this struct, VMCS12_REVISION must be changed.
|
||||
*
|
||||
* IMPORTANT: Changing the layout of existing fields in this structure
|
||||
* will break save/restore compatibility with older kvm releases. When
|
||||
* adding new fields, either use space in the reserved padding* arrays
|
||||
* or add the new fields to the end of the structure.
|
||||
*/
|
||||
typedef u64 natural_width;
|
||||
struct __packed vmcs12 {
|
||||
|
@ -265,17 +269,14 @@ struct __packed vmcs12 {
|
|||
u64 virtual_apic_page_addr;
|
||||
u64 apic_access_addr;
|
||||
u64 posted_intr_desc_addr;
|
||||
u64 vm_function_control;
|
||||
u64 ept_pointer;
|
||||
u64 eoi_exit_bitmap0;
|
||||
u64 eoi_exit_bitmap1;
|
||||
u64 eoi_exit_bitmap2;
|
||||
u64 eoi_exit_bitmap3;
|
||||
u64 eptp_list_address;
|
||||
u64 xss_exit_bitmap;
|
||||
u64 guest_physical_address;
|
||||
u64 vmcs_link_pointer;
|
||||
u64 pml_address;
|
||||
u64 guest_ia32_debugctl;
|
||||
u64 guest_ia32_pat;
|
||||
u64 guest_ia32_efer;
|
||||
|
@ -288,7 +289,12 @@ struct __packed vmcs12 {
|
|||
u64 host_ia32_pat;
|
||||
u64 host_ia32_efer;
|
||||
u64 host_ia32_perf_global_ctrl;
|
||||
u64 padding64[8]; /* room for future expansion */
|
||||
u64 vmread_bitmap;
|
||||
u64 vmwrite_bitmap;
|
||||
u64 vm_function_control;
|
||||
u64 eptp_list_address;
|
||||
u64 pml_address;
|
||||
u64 padding64[3]; /* room for future expansion */
|
||||
/*
|
||||
* To allow migration of L1 (complete with its L2 guests) between
|
||||
* machines of different natural widths (32 or 64 bit), we cannot have
|
||||
|
@ -397,7 +403,6 @@ struct __packed vmcs12 {
|
|||
u16 guest_ldtr_selector;
|
||||
u16 guest_tr_selector;
|
||||
u16 guest_intr_status;
|
||||
u16 guest_pml_index;
|
||||
u16 host_es_selector;
|
||||
u16 host_cs_selector;
|
||||
u16 host_ss_selector;
|
||||
|
@ -405,12 +410,172 @@ struct __packed vmcs12 {
|
|||
u16 host_fs_selector;
|
||||
u16 host_gs_selector;
|
||||
u16 host_tr_selector;
|
||||
u16 guest_pml_index;
|
||||
};
|
||||
|
||||
/*
|
||||
* For save/restore compatibility, the vmcs12 field offsets must not change.
|
||||
*/
|
||||
#define CHECK_OFFSET(field, loc) \
|
||||
BUILD_BUG_ON_MSG(offsetof(struct vmcs12, field) != (loc), \
|
||||
"Offset of " #field " in struct vmcs12 has changed.")
|
||||
|
||||
static inline void vmx_check_vmcs12_offsets(void) {
|
||||
CHECK_OFFSET(revision_id, 0);
|
||||
CHECK_OFFSET(abort, 4);
|
||||
CHECK_OFFSET(launch_state, 8);
|
||||
CHECK_OFFSET(io_bitmap_a, 40);
|
||||
CHECK_OFFSET(io_bitmap_b, 48);
|
||||
CHECK_OFFSET(msr_bitmap, 56);
|
||||
CHECK_OFFSET(vm_exit_msr_store_addr, 64);
|
||||
CHECK_OFFSET(vm_exit_msr_load_addr, 72);
|
||||
CHECK_OFFSET(vm_entry_msr_load_addr, 80);
|
||||
CHECK_OFFSET(tsc_offset, 88);
|
||||
CHECK_OFFSET(virtual_apic_page_addr, 96);
|
||||
CHECK_OFFSET(apic_access_addr, 104);
|
||||
CHECK_OFFSET(posted_intr_desc_addr, 112);
|
||||
CHECK_OFFSET(ept_pointer, 120);
|
||||
CHECK_OFFSET(eoi_exit_bitmap0, 128);
|
||||
CHECK_OFFSET(eoi_exit_bitmap1, 136);
|
||||
CHECK_OFFSET(eoi_exit_bitmap2, 144);
|
||||
CHECK_OFFSET(eoi_exit_bitmap3, 152);
|
||||
CHECK_OFFSET(xss_exit_bitmap, 160);
|
||||
CHECK_OFFSET(guest_physical_address, 168);
|
||||
CHECK_OFFSET(vmcs_link_pointer, 176);
|
||||
CHECK_OFFSET(guest_ia32_debugctl, 184);
|
||||
CHECK_OFFSET(guest_ia32_pat, 192);
|
||||
CHECK_OFFSET(guest_ia32_efer, 200);
|
||||
CHECK_OFFSET(guest_ia32_perf_global_ctrl, 208);
|
||||
CHECK_OFFSET(guest_pdptr0, 216);
|
||||
CHECK_OFFSET(guest_pdptr1, 224);
|
||||
CHECK_OFFSET(guest_pdptr2, 232);
|
||||
CHECK_OFFSET(guest_pdptr3, 240);
|
||||
CHECK_OFFSET(guest_bndcfgs, 248);
|
||||
CHECK_OFFSET(host_ia32_pat, 256);
|
||||
CHECK_OFFSET(host_ia32_efer, 264);
|
||||
CHECK_OFFSET(host_ia32_perf_global_ctrl, 272);
|
||||
CHECK_OFFSET(vmread_bitmap, 280);
|
||||
CHECK_OFFSET(vmwrite_bitmap, 288);
|
||||
CHECK_OFFSET(vm_function_control, 296);
|
||||
CHECK_OFFSET(eptp_list_address, 304);
|
||||
CHECK_OFFSET(pml_address, 312);
|
||||
CHECK_OFFSET(cr0_guest_host_mask, 344);
|
||||
CHECK_OFFSET(cr4_guest_host_mask, 352);
|
||||
CHECK_OFFSET(cr0_read_shadow, 360);
|
||||
CHECK_OFFSET(cr4_read_shadow, 368);
|
||||
CHECK_OFFSET(cr3_target_value0, 376);
|
||||
CHECK_OFFSET(cr3_target_value1, 384);
|
||||
CHECK_OFFSET(cr3_target_value2, 392);
|
||||
CHECK_OFFSET(cr3_target_value3, 400);
|
||||
CHECK_OFFSET(exit_qualification, 408);
|
||||
CHECK_OFFSET(guest_linear_address, 416);
|
||||
CHECK_OFFSET(guest_cr0, 424);
|
||||
CHECK_OFFSET(guest_cr3, 432);
|
||||
CHECK_OFFSET(guest_cr4, 440);
|
||||
CHECK_OFFSET(guest_es_base, 448);
|
||||
CHECK_OFFSET(guest_cs_base, 456);
|
||||
CHECK_OFFSET(guest_ss_base, 464);
|
||||
CHECK_OFFSET(guest_ds_base, 472);
|
||||
CHECK_OFFSET(guest_fs_base, 480);
|
||||
CHECK_OFFSET(guest_gs_base, 488);
|
||||
CHECK_OFFSET(guest_ldtr_base, 496);
|
||||
CHECK_OFFSET(guest_tr_base, 504);
|
||||
CHECK_OFFSET(guest_gdtr_base, 512);
|
||||
CHECK_OFFSET(guest_idtr_base, 520);
|
||||
CHECK_OFFSET(guest_dr7, 528);
|
||||
CHECK_OFFSET(guest_rsp, 536);
|
||||
CHECK_OFFSET(guest_rip, 544);
|
||||
CHECK_OFFSET(guest_rflags, 552);
|
||||
CHECK_OFFSET(guest_pending_dbg_exceptions, 560);
|
||||
CHECK_OFFSET(guest_sysenter_esp, 568);
|
||||
CHECK_OFFSET(guest_sysenter_eip, 576);
|
||||
CHECK_OFFSET(host_cr0, 584);
|
||||
CHECK_OFFSET(host_cr3, 592);
|
||||
CHECK_OFFSET(host_cr4, 600);
|
||||
CHECK_OFFSET(host_fs_base, 608);
|
||||
CHECK_OFFSET(host_gs_base, 616);
|
||||
CHECK_OFFSET(host_tr_base, 624);
|
||||
CHECK_OFFSET(host_gdtr_base, 632);
|
||||
CHECK_OFFSET(host_idtr_base, 640);
|
||||
CHECK_OFFSET(host_ia32_sysenter_esp, 648);
|
||||
CHECK_OFFSET(host_ia32_sysenter_eip, 656);
|
||||
CHECK_OFFSET(host_rsp, 664);
|
||||
CHECK_OFFSET(host_rip, 672);
|
||||
CHECK_OFFSET(pin_based_vm_exec_control, 744);
|
||||
CHECK_OFFSET(cpu_based_vm_exec_control, 748);
|
||||
CHECK_OFFSET(exception_bitmap, 752);
|
||||
CHECK_OFFSET(page_fault_error_code_mask, 756);
|
||||
CHECK_OFFSET(page_fault_error_code_match, 760);
|
||||
CHECK_OFFSET(cr3_target_count, 764);
|
||||
CHECK_OFFSET(vm_exit_controls, 768);
|
||||
CHECK_OFFSET(vm_exit_msr_store_count, 772);
|
||||
CHECK_OFFSET(vm_exit_msr_load_count, 776);
|
||||
CHECK_OFFSET(vm_entry_controls, 780);
|
||||
CHECK_OFFSET(vm_entry_msr_load_count, 784);
|
||||
CHECK_OFFSET(vm_entry_intr_info_field, 788);
|
||||
CHECK_OFFSET(vm_entry_exception_error_code, 792);
|
||||
CHECK_OFFSET(vm_entry_instruction_len, 796);
|
||||
CHECK_OFFSET(tpr_threshold, 800);
|
||||
CHECK_OFFSET(secondary_vm_exec_control, 804);
|
||||
CHECK_OFFSET(vm_instruction_error, 808);
|
||||
CHECK_OFFSET(vm_exit_reason, 812);
|
||||
CHECK_OFFSET(vm_exit_intr_info, 816);
|
||||
CHECK_OFFSET(vm_exit_intr_error_code, 820);
|
||||
CHECK_OFFSET(idt_vectoring_info_field, 824);
|
||||
CHECK_OFFSET(idt_vectoring_error_code, 828);
|
||||
CHECK_OFFSET(vm_exit_instruction_len, 832);
|
||||
CHECK_OFFSET(vmx_instruction_info, 836);
|
||||
CHECK_OFFSET(guest_es_limit, 840);
|
||||
CHECK_OFFSET(guest_cs_limit, 844);
|
||||
CHECK_OFFSET(guest_ss_limit, 848);
|
||||
CHECK_OFFSET(guest_ds_limit, 852);
|
||||
CHECK_OFFSET(guest_fs_limit, 856);
|
||||
CHECK_OFFSET(guest_gs_limit, 860);
|
||||
CHECK_OFFSET(guest_ldtr_limit, 864);
|
||||
CHECK_OFFSET(guest_tr_limit, 868);
|
||||
CHECK_OFFSET(guest_gdtr_limit, 872);
|
||||
CHECK_OFFSET(guest_idtr_limit, 876);
|
||||
CHECK_OFFSET(guest_es_ar_bytes, 880);
|
||||
CHECK_OFFSET(guest_cs_ar_bytes, 884);
|
||||
CHECK_OFFSET(guest_ss_ar_bytes, 888);
|
||||
CHECK_OFFSET(guest_ds_ar_bytes, 892);
|
||||
CHECK_OFFSET(guest_fs_ar_bytes, 896);
|
||||
CHECK_OFFSET(guest_gs_ar_bytes, 900);
|
||||
CHECK_OFFSET(guest_ldtr_ar_bytes, 904);
|
||||
CHECK_OFFSET(guest_tr_ar_bytes, 908);
|
||||
CHECK_OFFSET(guest_interruptibility_info, 912);
|
||||
CHECK_OFFSET(guest_activity_state, 916);
|
||||
CHECK_OFFSET(guest_sysenter_cs, 920);
|
||||
CHECK_OFFSET(host_ia32_sysenter_cs, 924);
|
||||
CHECK_OFFSET(vmx_preemption_timer_value, 928);
|
||||
CHECK_OFFSET(virtual_processor_id, 960);
|
||||
CHECK_OFFSET(posted_intr_nv, 962);
|
||||
CHECK_OFFSET(guest_es_selector, 964);
|
||||
CHECK_OFFSET(guest_cs_selector, 966);
|
||||
CHECK_OFFSET(guest_ss_selector, 968);
|
||||
CHECK_OFFSET(guest_ds_selector, 970);
|
||||
CHECK_OFFSET(guest_fs_selector, 972);
|
||||
CHECK_OFFSET(guest_gs_selector, 974);
|
||||
CHECK_OFFSET(guest_ldtr_selector, 976);
|
||||
CHECK_OFFSET(guest_tr_selector, 978);
|
||||
CHECK_OFFSET(guest_intr_status, 980);
|
||||
CHECK_OFFSET(host_es_selector, 982);
|
||||
CHECK_OFFSET(host_cs_selector, 984);
|
||||
CHECK_OFFSET(host_ss_selector, 986);
|
||||
CHECK_OFFSET(host_ds_selector, 988);
|
||||
CHECK_OFFSET(host_fs_selector, 990);
|
||||
CHECK_OFFSET(host_gs_selector, 992);
|
||||
CHECK_OFFSET(host_tr_selector, 994);
|
||||
CHECK_OFFSET(guest_pml_index, 996);
|
||||
}
|
||||
|
||||
/*
|
||||
* VMCS12_REVISION is an arbitrary id that should be changed if the content or
|
||||
* layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
|
||||
* VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
|
||||
*
|
||||
* IMPORTANT: Changing this value will break save/restore compatibility with
|
||||
* older kvm releases.
|
||||
*/
|
||||
#define VMCS12_REVISION 0x11e57ed0
|
||||
|
||||
|
@ -481,7 +646,8 @@ struct nested_vmx {
|
|||
bool sync_shadow_vmcs;
|
||||
bool dirty_vmcs12;
|
||||
|
||||
bool change_vmcs01_virtual_x2apic_mode;
|
||||
bool change_vmcs01_virtual_apic_mode;
|
||||
|
||||
/* L2 must run next, and mustn't decide to exit to L1. */
|
||||
bool nested_run_pending;
|
||||
|
||||
|
@ -761,6 +927,7 @@ static const unsigned short vmcs_field_to_offset_table[] = {
|
|||
FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
|
||||
FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
|
||||
FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
|
||||
FIELD64(PML_ADDRESS, pml_address),
|
||||
FIELD64(TSC_OFFSET, tsc_offset),
|
||||
FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
|
||||
FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
|
||||
|
@ -772,10 +939,11 @@ static const unsigned short vmcs_field_to_offset_table[] = {
|
|||
FIELD64(EOI_EXIT_BITMAP2, eoi_exit_bitmap2),
|
||||
FIELD64(EOI_EXIT_BITMAP3, eoi_exit_bitmap3),
|
||||
FIELD64(EPTP_LIST_ADDRESS, eptp_list_address),
|
||||
FIELD64(VMREAD_BITMAP, vmread_bitmap),
|
||||
FIELD64(VMWRITE_BITMAP, vmwrite_bitmap),
|
||||
FIELD64(XSS_EXIT_BITMAP, xss_exit_bitmap),
|
||||
FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
|
||||
FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
|
||||
FIELD64(PML_ADDRESS, pml_address),
|
||||
FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
|
||||
FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
|
||||
FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
|
||||
|
@ -1089,6 +1257,16 @@ static inline u16 evmcs_read16(unsigned long field)
|
|||
return *(u16 *)((char *)current_evmcs + offset);
|
||||
}
|
||||
|
||||
static inline void evmcs_touch_msr_bitmap(void)
|
||||
{
|
||||
if (unlikely(!current_evmcs))
|
||||
return;
|
||||
|
||||
if (current_evmcs->hv_enlightenments_control.msr_bitmap)
|
||||
current_evmcs->hv_clean_fields &=
|
||||
~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP;
|
||||
}
|
||||
|
||||
static void evmcs_load(u64 phys_addr)
|
||||
{
|
||||
struct hv_vp_assist_page *vp_ap =
|
||||
|
@ -1173,6 +1351,7 @@ static inline u32 evmcs_read32(unsigned long field) { return 0; }
|
|||
static inline u16 evmcs_read16(unsigned long field) { return 0; }
|
||||
static inline void evmcs_load(u64 phys_addr) {}
|
||||
static inline void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf) {}
|
||||
static inline void evmcs_touch_msr_bitmap(void) {}
|
||||
#endif /* IS_ENABLED(CONFIG_HYPERV) */
|
||||
|
||||
static inline bool is_exception_n(u32 intr_info, u8 vector)
|
||||
|
@ -1393,6 +1572,11 @@ static inline bool cpu_has_vmx_invept_global(void)
|
|||
return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
|
||||
}
|
||||
|
||||
static inline bool cpu_has_vmx_invvpid_individual_addr(void)
|
||||
{
|
||||
return vmx_capability.vpid & VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT;
|
||||
}
|
||||
|
||||
static inline bool cpu_has_vmx_invvpid_single(void)
|
||||
{
|
||||
return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
|
||||
|
@ -1510,6 +1694,17 @@ static inline unsigned nested_cpu_vmx_misc_cr3_count(struct kvm_vcpu *vcpu)
|
|||
return vmx_misc_cr3_count(to_vmx(vcpu)->nested.msrs.misc_low);
|
||||
}
|
||||
|
||||
/*
|
||||
* Do the virtual VMX capability MSRs specify that L1 can use VMWRITE
|
||||
* to modify any valid field of the VMCS, or are the VM-exit
|
||||
* information fields read-only?
|
||||
*/
|
||||
static inline bool nested_cpu_has_vmwrite_any_field(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return to_vmx(vcpu)->nested.msrs.misc_low &
|
||||
MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS;
|
||||
}
|
||||
|
||||
static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit)
|
||||
{
|
||||
return vmcs12->cpu_based_vm_exec_control & bit;
|
||||
|
@ -3127,6 +3322,7 @@ static void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, bool apicv)
|
|||
msrs->misc_high);
|
||||
msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA;
|
||||
msrs->misc_low |=
|
||||
MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS |
|
||||
VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
|
||||
VMX_MISC_ACTIVITY_HLT;
|
||||
msrs->misc_high = 0;
|
||||
|
@ -3300,6 +3496,15 @@ static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
|
|||
|
||||
vmx->nested.msrs.misc_low = data;
|
||||
vmx->nested.msrs.misc_high = data >> 32;
|
||||
|
||||
/*
|
||||
* If L1 has read-only VM-exit information fields, use the
|
||||
* less permissive vmx_vmwrite_bitmap to specify write
|
||||
* permissions for the shadow VMCS.
|
||||
*/
|
||||
if (enable_shadow_vmcs && !nested_cpu_has_vmwrite_any_field(&vmx->vcpu))
|
||||
vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -3354,6 +3559,13 @@ static int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
|
|||
{
|
||||
struct vcpu_vmx *vmx = to_vmx(vcpu);
|
||||
|
||||
/*
|
||||
* Don't allow changes to the VMX capability MSRs while the vCPU
|
||||
* is in VMX operation.
|
||||
*/
|
||||
if (vmx->nested.vmxon)
|
||||
return -EBUSY;
|
||||
|
||||
switch (msr_index) {
|
||||
case MSR_IA32_VMX_BASIC:
|
||||
return vmx_restore_vmx_basic(vmx, data);
|
||||
|
@ -4216,6 +4428,14 @@ static int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
|
|||
if (!loaded_vmcs->msr_bitmap)
|
||||
goto out_vmcs;
|
||||
memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
|
||||
|
||||
if (static_branch_unlikely(&enable_evmcs) &&
|
||||
(ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) {
|
||||
struct hv_enlightened_vmcs *evmcs =
|
||||
(struct hv_enlightened_vmcs *)loaded_vmcs->vmcs;
|
||||
|
||||
evmcs->hv_enlightenments_control.msr_bitmap = 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
|
||||
|
@ -5329,6 +5549,9 @@ static void __always_inline vmx_disable_intercept_for_msr(unsigned long *msr_bit
|
|||
if (!cpu_has_vmx_msr_bitmap())
|
||||
return;
|
||||
|
||||
if (static_branch_unlikely(&enable_evmcs))
|
||||
evmcs_touch_msr_bitmap();
|
||||
|
||||
/*
|
||||
* See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
|
||||
* have the write-low and read-high bitmap offsets the wrong way round.
|
||||
|
@ -5364,6 +5587,9 @@ static void __always_inline vmx_enable_intercept_for_msr(unsigned long *msr_bitm
|
|||
if (!cpu_has_vmx_msr_bitmap())
|
||||
return;
|
||||
|
||||
if (static_branch_unlikely(&enable_evmcs))
|
||||
evmcs_touch_msr_bitmap();
|
||||
|
||||
/*
|
||||
* See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
|
||||
* have the write-low and read-high bitmap offsets the wrong way round.
|
||||
|
@ -5946,8 +6172,14 @@ static void vmx_vcpu_setup(struct vcpu_vmx *vmx)
|
|||
int i;
|
||||
|
||||
if (enable_shadow_vmcs) {
|
||||
/*
|
||||
* At vCPU creation, "VMWRITE to any supported field
|
||||
* in the VMCS" is supported, so use the more
|
||||
* permissive vmx_vmread_bitmap to specify both read
|
||||
* and write permissions for the shadow VMCS.
|
||||
*/
|
||||
vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
|
||||
vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
|
||||
vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmread_bitmap));
|
||||
}
|
||||
if (cpu_has_vmx_msr_bitmap())
|
||||
vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap));
|
||||
|
@ -7588,8 +7820,7 @@ static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer)
|
|||
vmcs_read32(VMX_INSTRUCTION_INFO), false, &gva))
|
||||
return 1;
|
||||
|
||||
if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, vmpointer,
|
||||
sizeof(*vmpointer), &e)) {
|
||||
if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) {
|
||||
kvm_inject_page_fault(vcpu, &e);
|
||||
return 1;
|
||||
}
|
||||
|
@ -7670,6 +7901,12 @@ static int handle_vmon(struct kvm_vcpu *vcpu)
|
|||
return 1;
|
||||
}
|
||||
|
||||
/* CPL=0 must be checked manually. */
|
||||
if (vmx_get_cpl(vcpu)) {
|
||||
kvm_queue_exception(vcpu, UD_VECTOR);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (vmx->nested.vmxon) {
|
||||
nested_vmx_failValid(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
|
||||
return kvm_skip_emulated_instruction(vcpu);
|
||||
|
@ -7729,6 +7966,11 @@ static int handle_vmon(struct kvm_vcpu *vcpu)
|
|||
*/
|
||||
static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (vmx_get_cpl(vcpu)) {
|
||||
kvm_queue_exception(vcpu, UD_VECTOR);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!to_vmx(vcpu)->nested.vmxon) {
|
||||
kvm_queue_exception(vcpu, UD_VECTOR);
|
||||
return 0;
|
||||
|
@ -7928,23 +8170,42 @@ static inline int vmcs12_write_any(struct kvm_vcpu *vcpu,
|
|||
|
||||
}
|
||||
|
||||
/*
|
||||
* Copy the writable VMCS shadow fields back to the VMCS12, in case
|
||||
* they have been modified by the L1 guest. Note that the "read-only"
|
||||
* VM-exit information fields are actually writable if the vCPU is
|
||||
* configured to support "VMWRITE to any supported field in the VMCS."
|
||||
*/
|
||||
static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
|
||||
{
|
||||
int i;
|
||||
const u16 *fields[] = {
|
||||
shadow_read_write_fields,
|
||||
shadow_read_only_fields
|
||||
};
|
||||
const int max_fields[] = {
|
||||
max_shadow_read_write_fields,
|
||||
max_shadow_read_only_fields
|
||||
};
|
||||
int i, q;
|
||||
unsigned long field;
|
||||
u64 field_value;
|
||||
struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
|
||||
const u16 *fields = shadow_read_write_fields;
|
||||
const int num_fields = max_shadow_read_write_fields;
|
||||
|
||||
preempt_disable();
|
||||
|
||||
vmcs_load(shadow_vmcs);
|
||||
|
||||
for (i = 0; i < num_fields; i++) {
|
||||
field = fields[i];
|
||||
field_value = __vmcs_readl(field);
|
||||
vmcs12_write_any(&vmx->vcpu, field, field_value);
|
||||
for (q = 0; q < ARRAY_SIZE(fields); q++) {
|
||||
for (i = 0; i < max_fields[q]; i++) {
|
||||
field = fields[q][i];
|
||||
field_value = __vmcs_readl(field);
|
||||
vmcs12_write_any(&vmx->vcpu, field, field_value);
|
||||
}
|
||||
/*
|
||||
* Skip the VM-exit information fields if they are read-only.
|
||||
*/
|
||||
if (!nested_cpu_has_vmwrite_any_field(&vmx->vcpu))
|
||||
break;
|
||||
}
|
||||
|
||||
vmcs_clear(shadow_vmcs);
|
||||
|
@ -8029,9 +8290,9 @@ static int handle_vmread(struct kvm_vcpu *vcpu)
|
|||
if (get_vmx_mem_address(vcpu, exit_qualification,
|
||||
vmx_instruction_info, true, &gva))
|
||||
return 1;
|
||||
/* _system ok, as hardware has verified cpl=0 */
|
||||
kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, gva,
|
||||
&field_value, (is_long_mode(vcpu) ? 8 : 4), NULL);
|
||||
/* _system ok, nested_vmx_check_permission has verified cpl=0 */
|
||||
kvm_write_guest_virt_system(vcpu, gva, &field_value,
|
||||
(is_long_mode(vcpu) ? 8 : 4), NULL);
|
||||
}
|
||||
|
||||
nested_vmx_succeed(vcpu);
|
||||
|
@ -8069,8 +8330,8 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu)
|
|||
if (get_vmx_mem_address(vcpu, exit_qualification,
|
||||
vmx_instruction_info, false, &gva))
|
||||
return 1;
|
||||
if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva,
|
||||
&field_value, (is_64_bit_mode(vcpu) ? 8 : 4), &e)) {
|
||||
if (kvm_read_guest_virt(vcpu, gva, &field_value,
|
||||
(is_64_bit_mode(vcpu) ? 8 : 4), &e)) {
|
||||
kvm_inject_page_fault(vcpu, &e);
|
||||
return 1;
|
||||
}
|
||||
|
@ -8078,7 +8339,12 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu)
|
|||
|
||||
|
||||
field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
|
||||
if (vmcs_field_readonly(field)) {
|
||||
/*
|
||||
* If the vCPU supports "VMWRITE to any supported field in the
|
||||
* VMCS," then the "read-only" fields are actually read/write.
|
||||
*/
|
||||
if (vmcs_field_readonly(field) &&
|
||||
!nested_cpu_has_vmwrite_any_field(vcpu)) {
|
||||
nested_vmx_failValid(vcpu,
|
||||
VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
|
||||
return kvm_skip_emulated_instruction(vcpu);
|
||||
|
@ -8189,10 +8455,10 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu)
|
|||
if (get_vmx_mem_address(vcpu, exit_qualification,
|
||||
vmx_instruction_info, true, &vmcs_gva))
|
||||
return 1;
|
||||
/* ok to use *_system, as hardware has verified cpl=0 */
|
||||
if (kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, vmcs_gva,
|
||||
(void *)&to_vmx(vcpu)->nested.current_vmptr,
|
||||
sizeof(u64), &e)) {
|
||||
/* *_system ok, nested_vmx_check_permission has verified cpl=0 */
|
||||
if (kvm_write_guest_virt_system(vcpu, vmcs_gva,
|
||||
(void *)&to_vmx(vcpu)->nested.current_vmptr,
|
||||
sizeof(u64), &e)) {
|
||||
kvm_inject_page_fault(vcpu, &e);
|
||||
return 1;
|
||||
}
|
||||
|
@ -8239,8 +8505,7 @@ static int handle_invept(struct kvm_vcpu *vcpu)
|
|||
if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
|
||||
vmx_instruction_info, false, &gva))
|
||||
return 1;
|
||||
if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &operand,
|
||||
sizeof(operand), &e)) {
|
||||
if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
|
||||
kvm_inject_page_fault(vcpu, &e);
|
||||
return 1;
|
||||
}
|
||||
|
@ -8304,8 +8569,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
|
|||
if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
|
||||
vmx_instruction_info, false, &gva))
|
||||
return 1;
|
||||
if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &operand,
|
||||
sizeof(operand), &e)) {
|
||||
if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
|
||||
kvm_inject_page_fault(vcpu, &e);
|
||||
return 1;
|
||||
}
|
||||
|
@ -8317,12 +8581,19 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
|
|||
|
||||
switch (type) {
|
||||
case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
|
||||
if (is_noncanonical_address(operand.gla, vcpu)) {
|
||||
if (!operand.vpid ||
|
||||
is_noncanonical_address(operand.gla, vcpu)) {
|
||||
nested_vmx_failValid(vcpu,
|
||||
VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
|
||||
return kvm_skip_emulated_instruction(vcpu);
|
||||
}
|
||||
/* fall through */
|
||||
if (cpu_has_vmx_invvpid_individual_addr() &&
|
||||
vmx->nested.vpid02) {
|
||||
__invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR,
|
||||
vmx->nested.vpid02, operand.gla);
|
||||
} else
|
||||
__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
|
||||
break;
|
||||
case VMX_VPID_EXTENT_SINGLE_CONTEXT:
|
||||
case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
|
||||
if (!operand.vpid) {
|
||||
|
@ -8330,15 +8601,16 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
|
|||
VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
|
||||
return kvm_skip_emulated_instruction(vcpu);
|
||||
}
|
||||
__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
|
||||
break;
|
||||
case VMX_VPID_EXTENT_ALL_CONTEXT:
|
||||
__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
|
||||
break;
|
||||
default:
|
||||
WARN_ON_ONCE(1);
|
||||
return kvm_skip_emulated_instruction(vcpu);
|
||||
}
|
||||
|
||||
__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
|
||||
nested_vmx_succeed(vcpu);
|
||||
|
||||
return kvm_skip_emulated_instruction(vcpu);
|
||||
|
@ -8842,11 +9114,13 @@ static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason)
|
|||
case EXIT_REASON_TPR_BELOW_THRESHOLD:
|
||||
return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
|
||||
case EXIT_REASON_APIC_ACCESS:
|
||||
return nested_cpu_has2(vmcs12,
|
||||
SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
|
||||
case EXIT_REASON_APIC_WRITE:
|
||||
case EXIT_REASON_EOI_INDUCED:
|
||||
/* apic_write and eoi_induced should exit unconditionally. */
|
||||
/*
|
||||
* The controls for "virtualize APIC accesses," "APIC-
|
||||
* register virtualization," and "virtual-interrupt
|
||||
* delivery" only come from vmcs12.
|
||||
*/
|
||||
return true;
|
||||
case EXIT_REASON_EPT_VIOLATION:
|
||||
/*
|
||||
|
@ -9253,31 +9527,43 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
|
|||
vmcs_write32(TPR_THRESHOLD, irr);
|
||||
}
|
||||
|
||||
static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
|
||||
static void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u32 sec_exec_control;
|
||||
|
||||
if (!lapic_in_kernel(vcpu))
|
||||
return;
|
||||
|
||||
/* Postpone execution until vmcs01 is the current VMCS. */
|
||||
if (is_guest_mode(vcpu)) {
|
||||
to_vmx(vcpu)->nested.change_vmcs01_virtual_x2apic_mode = true;
|
||||
to_vmx(vcpu)->nested.change_vmcs01_virtual_apic_mode = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!cpu_has_vmx_virtualize_x2apic_mode())
|
||||
return;
|
||||
|
||||
if (!cpu_need_tpr_shadow(vcpu))
|
||||
return;
|
||||
|
||||
sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
|
||||
sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
|
||||
SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
|
||||
|
||||
if (set) {
|
||||
sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
|
||||
sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
|
||||
} else {
|
||||
sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
|
||||
sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
|
||||
vmx_flush_tlb(vcpu, true);
|
||||
switch (kvm_get_apic_mode(vcpu)) {
|
||||
case LAPIC_MODE_INVALID:
|
||||
WARN_ONCE(true, "Invalid local APIC state");
|
||||
case LAPIC_MODE_DISABLED:
|
||||
break;
|
||||
case LAPIC_MODE_XAPIC:
|
||||
if (flexpriority_enabled) {
|
||||
sec_exec_control |=
|
||||
SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
|
||||
vmx_flush_tlb(vcpu, true);
|
||||
}
|
||||
break;
|
||||
case LAPIC_MODE_X2APIC:
|
||||
if (cpu_has_vmx_virtualize_x2apic_mode())
|
||||
sec_exec_control |=
|
||||
SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
|
||||
break;
|
||||
}
|
||||
vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);
|
||||
|
||||
|
@ -9286,24 +9572,7 @@ static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
|
|||
|
||||
static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa)
|
||||
{
|
||||
struct vcpu_vmx *vmx = to_vmx(vcpu);
|
||||
|
||||
/*
|
||||
* Currently we do not handle the nested case where L2 has an
|
||||
* APIC access page of its own; that page is still pinned.
|
||||
* Hence, we skip the case where the VCPU is in guest mode _and_
|
||||
* L1 prepared an APIC access page for L2.
|
||||
*
|
||||
* For the case where L1 and L2 share the same APIC access page
|
||||
* (flexpriority=Y but SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES clear
|
||||
* in the vmcs12), this function will only update either the vmcs01
|
||||
* or the vmcs02. If the former, the vmcs02 will be updated by
|
||||
* prepare_vmcs02. If the latter, the vmcs01 will be updated in
|
||||
* the next L2->L1 exit.
|
||||
*/
|
||||
if (!is_guest_mode(vcpu) ||
|
||||
!nested_cpu_has2(get_vmcs12(&vmx->vcpu),
|
||||
SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
|
||||
if (!is_guest_mode(vcpu)) {
|
||||
vmcs_write64(APIC_ACCESS_ADDR, hpa);
|
||||
vmx_flush_tlb(vcpu, true);
|
||||
}
|
||||
|
@ -9943,13 +10212,13 @@ STACK_FRAME_NON_STANDARD(vmx_vcpu_run);
|
|||
|
||||
static struct kvm *vmx_vm_alloc(void)
|
||||
{
|
||||
struct kvm_vmx *kvm_vmx = kzalloc(sizeof(struct kvm_vmx), GFP_KERNEL);
|
||||
struct kvm_vmx *kvm_vmx = vzalloc(sizeof(struct kvm_vmx));
|
||||
return &kvm_vmx->kvm;
|
||||
}
|
||||
|
||||
static void vmx_vm_free(struct kvm *kvm)
|
||||
{
|
||||
kfree(to_kvm_vmx(kvm));
|
||||
vfree(to_kvm_vmx(kvm));
|
||||
}
|
||||
|
||||
static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
|
||||
|
@ -10387,11 +10656,6 @@ static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
|
|||
vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
|
||||
SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
|
||||
}
|
||||
} else if (!(nested_cpu_has_virt_x2apic_mode(vmcs12)) &&
|
||||
cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
|
||||
vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
|
||||
SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
|
||||
kvm_vcpu_reload_apic_access_page(vcpu);
|
||||
}
|
||||
|
||||
if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
|
||||
|
@ -10871,8 +11135,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void prepare_vmcs02_full(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
|
||||
bool from_vmentry)
|
||||
static void prepare_vmcs02_full(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
|
||||
{
|
||||
struct vcpu_vmx *vmx = to_vmx(vcpu);
|
||||
|
||||
|
@ -11006,13 +11269,13 @@ static void prepare_vmcs02_full(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
|
|||
* is assigned to entry_failure_code on failure.
|
||||
*/
|
||||
static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
|
||||
bool from_vmentry, u32 *entry_failure_code)
|
||||
u32 *entry_failure_code)
|
||||
{
|
||||
struct vcpu_vmx *vmx = to_vmx(vcpu);
|
||||
u32 exec_control, vmcs12_exec_ctrl;
|
||||
|
||||
if (vmx->nested.dirty_vmcs12) {
|
||||
prepare_vmcs02_full(vcpu, vmcs12, from_vmentry);
|
||||
prepare_vmcs02_full(vcpu, vmcs12);
|
||||
vmx->nested.dirty_vmcs12 = false;
|
||||
}
|
||||
|
||||
|
@ -11032,7 +11295,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
|
|||
* HOST_FS_BASE, HOST_GS_BASE.
|
||||
*/
|
||||
|
||||
if (from_vmentry &&
|
||||
if (vmx->nested.nested_run_pending &&
|
||||
(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
|
||||
kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
|
||||
vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
|
||||
|
@ -11040,7 +11303,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
|
|||
kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
|
||||
vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
|
||||
}
|
||||
if (from_vmentry) {
|
||||
if (vmx->nested.nested_run_pending) {
|
||||
vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
|
||||
vmcs12->vm_entry_intr_info_field);
|
||||
vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
|
||||
|
@ -11172,7 +11435,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
|
|||
~VM_ENTRY_IA32E_MODE) |
|
||||
(vmcs_config.vmentry_ctrl & ~VM_ENTRY_IA32E_MODE));
|
||||
|
||||
if (from_vmentry &&
|
||||
if (vmx->nested.nested_run_pending &&
|
||||
(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) {
|
||||
vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
|
||||
vcpu->arch.pat = vmcs12->guest_ia32_pat;
|
||||
|
@ -11197,7 +11460,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
|
|||
if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) {
|
||||
if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
|
||||
vmx->nested.last_vpid = vmcs12->virtual_processor_id;
|
||||
__vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02, true);
|
||||
__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
|
||||
}
|
||||
} else {
|
||||
vmx_flush_tlb(vcpu, true);
|
||||
|
@ -11240,7 +11503,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
|
|||
vmx_set_cr4(vcpu, vmcs12->guest_cr4);
|
||||
vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
|
||||
|
||||
if (from_vmentry &&
|
||||
if (vmx->nested.nested_run_pending &&
|
||||
(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER))
|
||||
vcpu->arch.efer = vmcs12->guest_ia32_efer;
|
||||
else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
|
||||
|
@ -11418,7 +11681,7 @@ static int check_vmentry_postreqs(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry)
|
||||
static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
struct vcpu_vmx *vmx = to_vmx(vcpu);
|
||||
struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
|
||||
|
@ -11438,7 +11701,7 @@ static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry)
|
|||
vcpu->arch.tsc_offset += vmcs12->tsc_offset;
|
||||
|
||||
r = EXIT_REASON_INVALID_STATE;
|
||||
if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &exit_qual))
|
||||
if (prepare_vmcs02(vcpu, vmcs12, &exit_qual))
|
||||
goto fail;
|
||||
|
||||
nested_get_vmcs12_pages(vcpu, vmcs12);
|
||||
|
@ -11540,20 +11803,22 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
|
|||
* the nested entry.
|
||||
*/
|
||||
|
||||
ret = enter_vmx_non_root_mode(vcpu, true);
|
||||
if (ret)
|
||||
vmx->nested.nested_run_pending = 1;
|
||||
ret = enter_vmx_non_root_mode(vcpu);
|
||||
if (ret) {
|
||||
vmx->nested.nested_run_pending = 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* If we're entering a halted L2 vcpu and the L2 vcpu won't be woken
|
||||
* by event injection, halt vcpu.
|
||||
*/
|
||||
if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) &&
|
||||
!(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK))
|
||||
!(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK)) {
|
||||
vmx->nested.nested_run_pending = 0;
|
||||
return kvm_vcpu_halt(vcpu);
|
||||
|
||||
vmx->nested.nested_run_pending = 1;
|
||||
|
||||
}
|
||||
return 1;
|
||||
|
||||
out:
|
||||
|
@ -11925,12 +12190,20 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
|
|||
|
||||
load_vmcs12_mmu_host_state(vcpu, vmcs12);
|
||||
|
||||
if (enable_vpid) {
|
||||
/*
|
||||
* Trivially support vpid by letting L2s share their parent
|
||||
* L1's vpid. TODO: move to a more elaborate solution, giving
|
||||
* each L2 its own vpid and exposing the vpid feature to L1.
|
||||
*/
|
||||
/*
|
||||
* If vmcs01 don't use VPID, CPU flushes TLB on every
|
||||
* VMEntry/VMExit. Thus, no need to flush TLB.
|
||||
*
|
||||
* If vmcs12 uses VPID, TLB entries populated by L2 are
|
||||
* tagged with vmx->nested.vpid02 while L1 entries are tagged
|
||||
* with vmx->vpid. Thus, no need to flush TLB.
|
||||
*
|
||||
* Therefore, flush TLB only in case vmcs01 uses VPID and
|
||||
* vmcs12 don't use VPID as in this case L1 & L2 TLB entries
|
||||
* are both tagged with vmx->vpid.
|
||||
*/
|
||||
if (enable_vpid &&
|
||||
!(nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02)) {
|
||||
vmx_flush_tlb(vcpu, true);
|
||||
}
|
||||
|
||||
|
@ -12069,10 +12342,9 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
|
|||
if (kvm_has_tsc_control)
|
||||
decache_tsc_multiplier(vmx);
|
||||
|
||||
if (vmx->nested.change_vmcs01_virtual_x2apic_mode) {
|
||||
vmx->nested.change_vmcs01_virtual_x2apic_mode = false;
|
||||
vmx_set_virtual_x2apic_mode(vcpu,
|
||||
vcpu->arch.apic_base & X2APIC_ENABLE);
|
||||
if (vmx->nested.change_vmcs01_virtual_apic_mode) {
|
||||
vmx->nested.change_vmcs01_virtual_apic_mode = false;
|
||||
vmx_set_virtual_apic_mode(vcpu);
|
||||
} else if (!nested_cpu_has_ept(vmcs12) &&
|
||||
nested_cpu_has2(vmcs12,
|
||||
SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
|
||||
|
@ -12236,7 +12508,7 @@ static inline int u64_shl_div_u64(u64 a, unsigned int shift,
|
|||
static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc)
|
||||
{
|
||||
struct vcpu_vmx *vmx;
|
||||
u64 tscl, guest_tscl, delta_tsc;
|
||||
u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles;
|
||||
|
||||
if (kvm_mwait_in_guest(vcpu->kvm))
|
||||
return -EOPNOTSUPP;
|
||||
|
@ -12245,6 +12517,12 @@ static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc)
|
|||
tscl = rdtsc();
|
||||
guest_tscl = kvm_read_l1_tsc(vcpu, tscl);
|
||||
delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl;
|
||||
lapic_timer_advance_cycles = nsec_to_cycles(vcpu, lapic_timer_advance_ns);
|
||||
|
||||
if (delta_tsc > lapic_timer_advance_cycles)
|
||||
delta_tsc -= lapic_timer_advance_cycles;
|
||||
else
|
||||
delta_tsc = 0;
|
||||
|
||||
/* Convert to host delta tsc if tsc scaling is enabled */
|
||||
if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio &&
|
||||
|
@ -12615,7 +12893,7 @@ static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, u64 smbase)
|
|||
|
||||
if (vmx->nested.smm.guest_mode) {
|
||||
vcpu->arch.hflags &= ~HF_SMM_MASK;
|
||||
ret = enter_vmx_non_root_mode(vcpu, false);
|
||||
ret = enter_vmx_non_root_mode(vcpu);
|
||||
vcpu->arch.hflags |= HF_SMM_MASK;
|
||||
if (ret)
|
||||
return ret;
|
||||
|
@ -12700,7 +12978,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
|
|||
.enable_nmi_window = enable_nmi_window,
|
||||
.enable_irq_window = enable_irq_window,
|
||||
.update_cr8_intercept = update_cr8_intercept,
|
||||
.set_virtual_x2apic_mode = vmx_set_virtual_x2apic_mode,
|
||||
.set_virtual_apic_mode = vmx_set_virtual_apic_mode,
|
||||
.set_apic_access_page_addr = vmx_set_apic_access_page_addr,
|
||||
.get_enable_apicv = vmx_get_enable_apicv,
|
||||
.refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
|
||||
|
@ -12812,6 +13090,7 @@ static int __init vmx_init(void)
|
|||
rcu_assign_pointer(crash_vmclear_loaded_vmcss,
|
||||
crash_vmclear_local_loaded_vmcss);
|
||||
#endif
|
||||
vmx_check_vmcs12_offsets();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -138,6 +138,7 @@ module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
|
|||
/* lapic timer advance (tscdeadline mode only) in nanoseconds */
|
||||
unsigned int __read_mostly lapic_timer_advance_ns = 0;
|
||||
module_param(lapic_timer_advance_ns, uint, S_IRUGO | S_IWUSR);
|
||||
EXPORT_SYMBOL_GPL(lapic_timer_advance_ns);
|
||||
|
||||
static bool __read_mostly vector_hashing = true;
|
||||
module_param(vector_hashing, bool, S_IRUGO);
|
||||
|
@ -318,23 +319,27 @@ u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(kvm_get_apic_base);
|
||||
|
||||
enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return kvm_apic_mode(kvm_get_apic_base(vcpu));
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kvm_get_apic_mode);
|
||||
|
||||
int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
|
||||
{
|
||||
u64 old_state = vcpu->arch.apic_base &
|
||||
(MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
|
||||
u64 new_state = msr_info->data &
|
||||
(MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
|
||||
enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
|
||||
enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
|
||||
u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |
|
||||
(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
|
||||
|
||||
if ((msr_info->data & reserved_bits) || new_state == X2APIC_ENABLE)
|
||||
return 1;
|
||||
if (!msr_info->host_initiated &&
|
||||
((new_state == MSR_IA32_APICBASE_ENABLE &&
|
||||
old_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) ||
|
||||
(new_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE) &&
|
||||
old_state == 0)))
|
||||
if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
|
||||
return 1;
|
||||
if (!msr_info->host_initiated) {
|
||||
if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
|
||||
return 1;
|
||||
if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
|
||||
return 1;
|
||||
}
|
||||
|
||||
kvm_lapic_set_base(vcpu, msr_info->data);
|
||||
return 0;
|
||||
|
@ -856,7 +861,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
|
|||
}
|
||||
|
||||
if (is_long_mode(vcpu) &&
|
||||
(cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 62)))
|
||||
(cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63)))
|
||||
return 1;
|
||||
else if (is_pae(vcpu) && is_paging(vcpu) &&
|
||||
!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
|
||||
|
@ -1761,7 +1766,7 @@ static int do_monotonic_boot(s64 *t, u64 *tsc_timestamp)
|
|||
return mode;
|
||||
}
|
||||
|
||||
static int do_realtime(struct timespec *ts, u64 *tsc_timestamp)
|
||||
static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
|
||||
{
|
||||
struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
|
||||
unsigned long seq;
|
||||
|
@ -1794,7 +1799,7 @@ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
|
|||
}
|
||||
|
||||
/* returns true if host is using TSC based clocksource */
|
||||
static bool kvm_get_walltime_and_clockread(struct timespec *ts,
|
||||
static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
|
||||
u64 *tsc_timestamp)
|
||||
{
|
||||
/* checked again under seqlock below */
|
||||
|
@ -2868,6 +2873,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
|
|||
case KVM_CAP_HYPERV_SYNIC2:
|
||||
case KVM_CAP_HYPERV_VP_INDEX:
|
||||
case KVM_CAP_HYPERV_EVENTFD:
|
||||
case KVM_CAP_HYPERV_TLBFLUSH:
|
||||
case KVM_CAP_PCI_SEGMENT:
|
||||
case KVM_CAP_DEBUGREGS:
|
||||
case KVM_CAP_X86_ROBUST_SINGLESTEP:
|
||||
|
@ -2894,7 +2900,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
|
|||
r = KVM_CLOCK_TSC_STABLE;
|
||||
break;
|
||||
case KVM_CAP_X86_DISABLE_EXITS:
|
||||
r |= KVM_X86_DISABLE_EXITS_HTL | KVM_X86_DISABLE_EXITS_PAUSE;
|
||||
r |= KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE;
|
||||
if(kvm_can_mwait_in_guest())
|
||||
r |= KVM_X86_DISABLE_EXITS_MWAIT;
|
||||
break;
|
||||
|
@ -3962,7 +3968,7 @@ out_nofree:
|
|||
return r;
|
||||
}
|
||||
|
||||
int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
|
||||
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
|
||||
{
|
||||
return VM_FAULT_SIGBUS;
|
||||
}
|
||||
|
@ -4248,7 +4254,7 @@ split_irqchip_unlock:
|
|||
if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
|
||||
kvm_can_mwait_in_guest())
|
||||
kvm->arch.mwait_in_guest = true;
|
||||
if (cap->args[0] & KVM_X86_DISABLE_EXITS_HTL)
|
||||
if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
|
||||
kvm->arch.hlt_in_guest = true;
|
||||
if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
|
||||
kvm->arch.pause_in_guest = true;
|
||||
|
@ -4787,11 +4793,10 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
|
|||
return X86EMUL_CONTINUE;
|
||||
}
|
||||
|
||||
int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
|
||||
int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
|
||||
gva_t addr, void *val, unsigned int bytes,
|
||||
struct x86_exception *exception)
|
||||
{
|
||||
struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
|
||||
u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
|
||||
|
||||
return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
|
||||
|
@ -4799,12 +4804,17 @@ int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
|
||||
|
||||
static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt,
|
||||
gva_t addr, void *val, unsigned int bytes,
|
||||
struct x86_exception *exception)
|
||||
static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
|
||||
gva_t addr, void *val, unsigned int bytes,
|
||||
struct x86_exception *exception, bool system)
|
||||
{
|
||||
struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
|
||||
return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception);
|
||||
u32 access = 0;
|
||||
|
||||
if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
|
||||
access |= PFERR_USER_MASK;
|
||||
|
||||
return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
|
||||
}
|
||||
|
||||
static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
|
||||
|
@ -4816,18 +4826,16 @@ static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
|
|||
return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
|
||||
}
|
||||
|
||||
int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
|
||||
gva_t addr, void *val,
|
||||
unsigned int bytes,
|
||||
struct x86_exception *exception)
|
||||
static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
|
||||
struct kvm_vcpu *vcpu, u32 access,
|
||||
struct x86_exception *exception)
|
||||
{
|
||||
struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
|
||||
void *data = val;
|
||||
int r = X86EMUL_CONTINUE;
|
||||
|
||||
while (bytes) {
|
||||
gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
|
||||
PFERR_WRITE_MASK,
|
||||
access,
|
||||
exception);
|
||||
unsigned offset = addr & (PAGE_SIZE-1);
|
||||
unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
|
||||
|
@ -4848,6 +4856,27 @@ int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
|
|||
out:
|
||||
return r;
|
||||
}
|
||||
|
||||
static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
|
||||
unsigned int bytes, struct x86_exception *exception,
|
||||
bool system)
|
||||
{
|
||||
struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
|
||||
u32 access = PFERR_WRITE_MASK;
|
||||
|
||||
if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
|
||||
access |= PFERR_USER_MASK;
|
||||
|
||||
return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
|
||||
access, exception);
|
||||
}
|
||||
|
||||
int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
|
||||
unsigned int bytes, struct x86_exception *exception)
|
||||
{
|
||||
return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
|
||||
PFERR_WRITE_MASK, exception);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
|
||||
|
||||
int handle_ud(struct kvm_vcpu *vcpu)
|
||||
|
@ -4858,8 +4887,8 @@ int handle_ud(struct kvm_vcpu *vcpu)
|
|||
struct x86_exception e;
|
||||
|
||||
if (force_emulation_prefix &&
|
||||
kvm_read_guest_virt(&vcpu->arch.emulate_ctxt,
|
||||
kvm_get_linear_rip(vcpu), sig, sizeof(sig), &e) == 0 &&
|
||||
kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
|
||||
sig, sizeof(sig), &e) == 0 &&
|
||||
memcmp(sig, "\xf\xbkvm", sizeof(sig)) == 0) {
|
||||
kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
|
||||
emul_type = 0;
|
||||
|
@ -5600,8 +5629,8 @@ static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt, u64 smbase)
|
|||
static const struct x86_emulate_ops emulate_ops = {
|
||||
.read_gpr = emulator_read_gpr,
|
||||
.write_gpr = emulator_write_gpr,
|
||||
.read_std = kvm_read_guest_virt_system,
|
||||
.write_std = kvm_write_guest_virt_system,
|
||||
.read_std = emulator_read_std,
|
||||
.write_std = emulator_write_std,
|
||||
.read_phys = kvm_read_guest_phys_system,
|
||||
.fetch = kvm_fetch_guest_virt,
|
||||
.read_emulated = emulator_read_emulated,
|
||||
|
@ -6617,7 +6646,7 @@ static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
|
|||
unsigned long clock_type)
|
||||
{
|
||||
struct kvm_clock_pairing clock_pairing;
|
||||
struct timespec ts;
|
||||
struct timespec64 ts;
|
||||
u64 cycle;
|
||||
int ret;
|
||||
|
||||
|
|
|
@ -247,11 +247,11 @@ int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip);
|
|||
void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr);
|
||||
u64 get_kvmclock_ns(struct kvm *kvm);
|
||||
|
||||
int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
|
||||
int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
|
||||
gva_t addr, void *val, unsigned int bytes,
|
||||
struct x86_exception *exception);
|
||||
|
||||
int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
|
||||
int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu,
|
||||
gva_t addr, void *val, unsigned int bytes,
|
||||
struct x86_exception *exception);
|
||||
|
||||
|
|
|
@ -28,7 +28,7 @@
|
|||
|
||||
#include <linux/irqchip/arm-gic-v4.h>
|
||||
|
||||
#define VGIC_V3_MAX_CPUS 255
|
||||
#define VGIC_V3_MAX_CPUS 512
|
||||
#define VGIC_V2_MAX_CPUS 8
|
||||
#define VGIC_NR_IRQS_LEGACY 256
|
||||
#define VGIC_NR_SGIS 16
|
||||
|
@ -201,6 +201,14 @@ struct vgic_its {
|
|||
|
||||
struct vgic_state_iter;
|
||||
|
||||
struct vgic_redist_region {
|
||||
u32 index;
|
||||
gpa_t base;
|
||||
u32 count; /* number of redistributors or 0 if single region */
|
||||
u32 free_index; /* index of the next free redistributor */
|
||||
struct list_head list;
|
||||
};
|
||||
|
||||
struct vgic_dist {
|
||||
bool in_kernel;
|
||||
bool ready;
|
||||
|
@ -220,10 +228,7 @@ struct vgic_dist {
|
|||
/* either a GICv2 CPU interface */
|
||||
gpa_t vgic_cpu_base;
|
||||
/* or a number of GICv3 redistributor regions */
|
||||
struct {
|
||||
gpa_t vgic_redist_base;
|
||||
gpa_t vgic_redist_free_offset;
|
||||
};
|
||||
struct list_head rd_regions;
|
||||
};
|
||||
|
||||
/* distributor enabled */
|
||||
|
@ -311,6 +316,7 @@ struct vgic_cpu {
|
|||
*/
|
||||
struct vgic_io_device rd_iodev;
|
||||
struct vgic_io_device sgi_iodev;
|
||||
struct vgic_redist_region *rdreg;
|
||||
|
||||
/* Contains the attributes and gpa of the LPI pending tables. */
|
||||
u64 pendbaser;
|
||||
|
@ -332,7 +338,6 @@ void kvm_vgic_early_init(struct kvm *kvm);
|
|||
int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu);
|
||||
int kvm_vgic_create(struct kvm *kvm, u32 type);
|
||||
void kvm_vgic_destroy(struct kvm *kvm);
|
||||
void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu);
|
||||
void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu);
|
||||
int kvm_vgic_map_resources(struct kvm *kvm);
|
||||
int kvm_vgic_hyp_init(void);
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
#include <linux/preempt.h>
|
||||
#include <linux/msi.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/rcupdate.h>
|
||||
#include <linux/ratelimit.h>
|
||||
#include <linux/err.h>
|
||||
|
@ -730,13 +731,16 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu);
|
|||
|
||||
void kvm_flush_remote_tlbs(struct kvm *kvm);
|
||||
void kvm_reload_remote_mmus(struct kvm *kvm);
|
||||
|
||||
bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
|
||||
unsigned long *vcpu_bitmap, cpumask_var_t tmp);
|
||||
bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req);
|
||||
|
||||
long kvm_arch_dev_ioctl(struct file *filp,
|
||||
unsigned int ioctl, unsigned long arg);
|
||||
long kvm_arch_vcpu_ioctl(struct file *filp,
|
||||
unsigned int ioctl, unsigned long arg);
|
||||
int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf);
|
||||
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf);
|
||||
|
||||
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext);
|
||||
|
||||
|
@ -808,6 +812,10 @@ bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu);
|
|||
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
|
||||
|
||||
#ifndef __KVM_HAVE_ARCH_VM_ALLOC
|
||||
/*
|
||||
* All architectures that want to use vzalloc currently also
|
||||
* need their own kvm_arch_alloc_vm implementation.
|
||||
*/
|
||||
static inline struct kvm *kvm_arch_alloc_vm(void)
|
||||
{
|
||||
return kzalloc(sizeof(struct kvm), GFP_KERNEL);
|
||||
|
@ -1270,4 +1278,13 @@ static inline long kvm_arch_vcpu_async_ioctl(struct file *filp,
|
|||
void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
|
||||
unsigned long start, unsigned long end);
|
||||
|
||||
#ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE
|
||||
int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu);
|
||||
#else
|
||||
static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif /* CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE */
|
||||
|
||||
#endif
|
||||
|
|
|
@ -1639,6 +1639,12 @@ static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
|
|||
clear_ti_thread_flag(task_thread_info(tsk), flag);
|
||||
}
|
||||
|
||||
static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
|
||||
bool value)
|
||||
{
|
||||
update_ti_thread_flag(task_thread_info(tsk), flag, value);
|
||||
}
|
||||
|
||||
static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
|
||||
{
|
||||
return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
|
||||
|
|
|
@ -60,6 +60,15 @@ static inline void clear_ti_thread_flag(struct thread_info *ti, int flag)
|
|||
clear_bit(flag, (unsigned long *)&ti->flags);
|
||||
}
|
||||
|
||||
static inline void update_ti_thread_flag(struct thread_info *ti, int flag,
|
||||
bool value)
|
||||
{
|
||||
if (value)
|
||||
set_ti_thread_flag(ti, flag);
|
||||
else
|
||||
clear_ti_thread_flag(ti, flag);
|
||||
}
|
||||
|
||||
static inline int test_and_set_ti_thread_flag(struct thread_info *ti, int flag)
|
||||
{
|
||||
return test_and_set_bit(flag, (unsigned long *)&ti->flags);
|
||||
|
@ -79,6 +88,8 @@ static inline int test_ti_thread_flag(struct thread_info *ti, int flag)
|
|||
set_ti_thread_flag(current_thread_info(), flag)
|
||||
#define clear_thread_flag(flag) \
|
||||
clear_ti_thread_flag(current_thread_info(), flag)
|
||||
#define update_thread_flag(flag, value) \
|
||||
update_ti_thread_flag(current_thread_info(), flag, value)
|
||||
#define test_and_set_thread_flag(flag) \
|
||||
test_and_set_ti_thread_flag(current_thread_info(), flag)
|
||||
#define test_and_clear_thread_flag(flag) \
|
||||
|
|
|
@ -677,10 +677,10 @@ struct kvm_ioeventfd {
|
|||
};
|
||||
|
||||
#define KVM_X86_DISABLE_EXITS_MWAIT (1 << 0)
|
||||
#define KVM_X86_DISABLE_EXITS_HTL (1 << 1)
|
||||
#define KVM_X86_DISABLE_EXITS_HLT (1 << 1)
|
||||
#define KVM_X86_DISABLE_EXITS_PAUSE (1 << 2)
|
||||
#define KVM_X86_DISABLE_VALID_EXITS (KVM_X86_DISABLE_EXITS_MWAIT | \
|
||||
KVM_X86_DISABLE_EXITS_HTL | \
|
||||
KVM_X86_DISABLE_EXITS_HLT | \
|
||||
KVM_X86_DISABLE_EXITS_PAUSE)
|
||||
|
||||
/* for KVM_ENABLE_CAP */
|
||||
|
@ -948,6 +948,7 @@ struct kvm_ppc_resize_hpt {
|
|||
#define KVM_CAP_S390_BPB 152
|
||||
#define KVM_CAP_GET_MSR_FEATURES 153
|
||||
#define KVM_CAP_HYPERV_EVENTFD 154
|
||||
#define KVM_CAP_HYPERV_TLBFLUSH 155
|
||||
|
||||
#ifdef KVM_CAP_IRQ_ROUTING
|
||||
|
||||
|
|
|
@ -677,10 +677,10 @@ struct kvm_ioeventfd {
|
|||
};
|
||||
|
||||
#define KVM_X86_DISABLE_EXITS_MWAIT (1 << 0)
|
||||
#define KVM_X86_DISABLE_EXITS_HTL (1 << 1)
|
||||
#define KVM_X86_DISABLE_EXITS_HLT (1 << 1)
|
||||
#define KVM_X86_DISABLE_EXITS_PAUSE (1 << 2)
|
||||
#define KVM_X86_DISABLE_VALID_EXITS (KVM_X86_DISABLE_EXITS_MWAIT | \
|
||||
KVM_X86_DISABLE_EXITS_HTL | \
|
||||
KVM_X86_DISABLE_EXITS_HLT | \
|
||||
KVM_X86_DISABLE_EXITS_PAUSE)
|
||||
|
||||
/* for KVM_ENABLE_CAP */
|
||||
|
|
|
@ -54,3 +54,6 @@ config HAVE_KVM_IRQ_BYPASS
|
|||
|
||||
config HAVE_KVM_VCPU_ASYNC_IOCTL
|
||||
bool
|
||||
|
||||
config HAVE_KVM_VCPU_RUN_PID_CHANGE
|
||||
bool
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include <linux/bug.h>
|
||||
#include <linux/cpu_pm.h>
|
||||
#include <linux/errno.h>
|
||||
#include <linux/err.h>
|
||||
|
@ -41,6 +42,7 @@
|
|||
#include <asm/mman.h>
|
||||
#include <asm/tlbflush.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/virt.h>
|
||||
#include <asm/kvm_arm.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
|
@ -163,7 +165,7 @@ int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
|
|||
return 0;
|
||||
}
|
||||
|
||||
int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
|
||||
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
|
||||
{
|
||||
return VM_FAULT_SIGBUS;
|
||||
}
|
||||
|
@ -249,6 +251,21 @@ long kvm_arch_dev_ioctl(struct file *filp,
|
|||
return -EINVAL;
|
||||
}
|
||||
|
||||
struct kvm *kvm_arch_alloc_vm(void)
|
||||
{
|
||||
if (!has_vhe())
|
||||
return kzalloc(sizeof(struct kvm), GFP_KERNEL);
|
||||
|
||||
return vzalloc(sizeof(struct kvm));
|
||||
}
|
||||
|
||||
void kvm_arch_free_vm(struct kvm *kvm)
|
||||
{
|
||||
if (!has_vhe())
|
||||
kfree(kvm);
|
||||
else
|
||||
vfree(kvm);
|
||||
}
|
||||
|
||||
struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
|
||||
{
|
||||
|
@ -290,7 +307,6 @@ out:
|
|||
|
||||
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
kvm_vgic_vcpu_early_init(vcpu);
|
||||
}
|
||||
|
||||
void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
|
||||
|
@ -363,10 +379,12 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
|
|||
kvm_vgic_load(vcpu);
|
||||
kvm_timer_vcpu_load(vcpu);
|
||||
kvm_vcpu_load_sysregs(vcpu);
|
||||
kvm_arch_vcpu_load_fp(vcpu);
|
||||
}
|
||||
|
||||
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
kvm_arch_vcpu_put_fp(vcpu);
|
||||
kvm_vcpu_put_sysregs(vcpu);
|
||||
kvm_timer_vcpu_put(vcpu);
|
||||
kvm_vgic_put(vcpu);
|
||||
|
@ -678,9 +696,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
|
|||
*/
|
||||
preempt_disable();
|
||||
|
||||
/* Flush FP/SIMD state that can't survive guest entry/exit */
|
||||
kvm_fpsimd_flush_cpu_state();
|
||||
|
||||
kvm_pmu_flush_hwstate(vcpu);
|
||||
|
||||
local_irq_disable();
|
||||
|
@ -778,6 +793,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
|
|||
if (static_branch_unlikely(&userspace_irqchip_in_use))
|
||||
kvm_timer_sync_hwstate(vcpu);
|
||||
|
||||
kvm_arch_vcpu_ctxsync_fp(vcpu);
|
||||
|
||||
/*
|
||||
* We may have taken a host interrupt in HYP mode (ie
|
||||
* while executing the guest). This interrupt is still
|
||||
|
@ -1574,6 +1591,11 @@ int kvm_arch_init(void *opaque)
|
|||
return -ENODEV;
|
||||
}
|
||||
|
||||
if (!kvm_arch_check_sve_has_vhe()) {
|
||||
kvm_pr_unimpl("SVE system without VHE unsupported. Broken cpu?");
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
for_each_online_cpu(cpu) {
|
||||
smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1);
|
||||
if (ret < 0) {
|
||||
|
|
|
@ -264,21 +264,12 @@ static const struct file_operations vgic_debug_fops = {
|
|||
.release = seq_release
|
||||
};
|
||||
|
||||
int vgic_debug_init(struct kvm *kvm)
|
||||
void vgic_debug_init(struct kvm *kvm)
|
||||
{
|
||||
if (!kvm->debugfs_dentry)
|
||||
return -ENOENT;
|
||||
|
||||
if (!debugfs_create_file("vgic-state", 0444,
|
||||
kvm->debugfs_dentry,
|
||||
kvm,
|
||||
&vgic_debug_fops))
|
||||
return -ENOMEM;
|
||||
|
||||
return 0;
|
||||
debugfs_create_file("vgic-state", 0444, kvm->debugfs_dentry, kvm,
|
||||
&vgic_debug_fops);
|
||||
}
|
||||
|
||||
int vgic_debug_destroy(struct kvm *kvm)
|
||||
void vgic_debug_destroy(struct kvm *kvm)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -44,7 +44,7 @@
|
|||
*
|
||||
* CPU Interface:
|
||||
*
|
||||
* - kvm_vgic_vcpu_early_init(): initialization of static data that
|
||||
* - kvm_vgic_vcpu_init(): initialization of static data that
|
||||
* doesn't depend on any sizing information or emulation type. No
|
||||
* allocation is allowed there.
|
||||
*/
|
||||
|
@ -67,46 +67,6 @@ void kvm_vgic_early_init(struct kvm *kvm)
|
|||
spin_lock_init(&dist->lpi_list_lock);
|
||||
}
|
||||
|
||||
/**
|
||||
* kvm_vgic_vcpu_early_init() - Initialize static VGIC VCPU data structures
|
||||
* @vcpu: The VCPU whose VGIC data structures whould be initialized
|
||||
*
|
||||
* Only do initialization, but do not actually enable the VGIC CPU interface
|
||||
* yet.
|
||||
*/
|
||||
void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
|
||||
int i;
|
||||
|
||||
INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
|
||||
spin_lock_init(&vgic_cpu->ap_list_lock);
|
||||
|
||||
/*
|
||||
* Enable and configure all SGIs to be edge-triggered and
|
||||
* configure all PPIs as level-triggered.
|
||||
*/
|
||||
for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
|
||||
struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
|
||||
|
||||
INIT_LIST_HEAD(&irq->ap_list);
|
||||
spin_lock_init(&irq->irq_lock);
|
||||
irq->intid = i;
|
||||
irq->vcpu = NULL;
|
||||
irq->target_vcpu = vcpu;
|
||||
irq->targets = 1U << vcpu->vcpu_id;
|
||||
kref_init(&irq->refcount);
|
||||
if (vgic_irq_is_sgi(i)) {
|
||||
/* SGIs */
|
||||
irq->enabled = 1;
|
||||
irq->config = VGIC_CONFIG_EDGE;
|
||||
} else {
|
||||
/* PPIs */
|
||||
irq->config = VGIC_CONFIG_LEVEL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* CREATION */
|
||||
|
||||
/**
|
||||
|
@ -167,8 +127,11 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
|
|||
kvm->arch.vgic.vgic_model = type;
|
||||
|
||||
kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
|
||||
kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
|
||||
kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;
|
||||
|
||||
if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
|
||||
kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
|
||||
else
|
||||
INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
|
||||
|
||||
out_unlock:
|
||||
for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
|
||||
|
@ -221,13 +184,50 @@ static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
|
|||
}
|
||||
|
||||
/**
|
||||
* kvm_vgic_vcpu_init() - Register VCPU-specific KVM iodevs
|
||||
* kvm_vgic_vcpu_init() - Initialize static VGIC VCPU data
|
||||
* structures and register VCPU-specific KVM iodevs
|
||||
*
|
||||
* @vcpu: pointer to the VCPU being created and initialized
|
||||
*
|
||||
* Only do initialization, but do not actually enable the
|
||||
* VGIC CPU interface
|
||||
*/
|
||||
int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
int ret = 0;
|
||||
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
|
||||
struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
|
||||
int ret = 0;
|
||||
int i;
|
||||
|
||||
vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
|
||||
vgic_cpu->sgi_iodev.base_addr = VGIC_ADDR_UNDEF;
|
||||
|
||||
INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
|
||||
spin_lock_init(&vgic_cpu->ap_list_lock);
|
||||
|
||||
/*
|
||||
* Enable and configure all SGIs to be edge-triggered and
|
||||
* configure all PPIs as level-triggered.
|
||||
*/
|
||||
for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
|
||||
struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
|
||||
|
||||
INIT_LIST_HEAD(&irq->ap_list);
|
||||
spin_lock_init(&irq->irq_lock);
|
||||
irq->intid = i;
|
||||
irq->vcpu = NULL;
|
||||
irq->target_vcpu = vcpu;
|
||||
irq->targets = 1U << vcpu->vcpu_id;
|
||||
kref_init(&irq->refcount);
|
||||
if (vgic_irq_is_sgi(i)) {
|
||||
/* SGIs */
|
||||
irq->enabled = 1;
|
||||
irq->config = VGIC_CONFIG_EDGE;
|
||||
} else {
|
||||
/* PPIs */
|
||||
irq->config = VGIC_CONFIG_LEVEL;
|
||||
}
|
||||
}
|
||||
|
||||
if (!irqchip_in_kernel(vcpu->kvm))
|
||||
return 0;
|
||||
|
@ -303,13 +303,23 @@ out:
|
|||
static void kvm_vgic_dist_destroy(struct kvm *kvm)
|
||||
{
|
||||
struct vgic_dist *dist = &kvm->arch.vgic;
|
||||
struct vgic_redist_region *rdreg, *next;
|
||||
|
||||
dist->ready = false;
|
||||
dist->initialized = false;
|
||||
|
||||
kfree(dist->spis);
|
||||
dist->spis = NULL;
|
||||
dist->nr_spis = 0;
|
||||
|
||||
if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
|
||||
list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list) {
|
||||
list_del(&rdreg->list);
|
||||
kfree(rdreg);
|
||||
}
|
||||
INIT_LIST_HEAD(&dist->rd_regions);
|
||||
}
|
||||
|
||||
if (vgic_supports_direct_msis(kvm))
|
||||
vgic_v4_teardown(kvm);
|
||||
}
|
||||
|
|
|
@ -66,6 +66,7 @@ int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)
|
|||
int r = 0;
|
||||
struct vgic_dist *vgic = &kvm->arch.vgic;
|
||||
phys_addr_t *addr_ptr, alignment;
|
||||
u64 undef_value = VGIC_ADDR_UNDEF;
|
||||
|
||||
mutex_lock(&kvm->lock);
|
||||
switch (type) {
|
||||
|
@ -84,16 +85,61 @@ int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)
|
|||
addr_ptr = &vgic->vgic_dist_base;
|
||||
alignment = SZ_64K;
|
||||
break;
|
||||
case KVM_VGIC_V3_ADDR_TYPE_REDIST:
|
||||
case KVM_VGIC_V3_ADDR_TYPE_REDIST: {
|
||||
struct vgic_redist_region *rdreg;
|
||||
|
||||
r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V3);
|
||||
if (r)
|
||||
break;
|
||||
if (write) {
|
||||
r = vgic_v3_set_redist_base(kvm, *addr);
|
||||
r = vgic_v3_set_redist_base(kvm, 0, *addr, 0);
|
||||
goto out;
|
||||
}
|
||||
addr_ptr = &vgic->vgic_redist_base;
|
||||
rdreg = list_first_entry(&vgic->rd_regions,
|
||||
struct vgic_redist_region, list);
|
||||
if (!rdreg)
|
||||
addr_ptr = &undef_value;
|
||||
else
|
||||
addr_ptr = &rdreg->base;
|
||||
break;
|
||||
}
|
||||
case KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION:
|
||||
{
|
||||
struct vgic_redist_region *rdreg;
|
||||
u8 index;
|
||||
|
||||
r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V3);
|
||||
if (r)
|
||||
break;
|
||||
|
||||
index = *addr & KVM_VGIC_V3_RDIST_INDEX_MASK;
|
||||
|
||||
if (write) {
|
||||
gpa_t base = *addr & KVM_VGIC_V3_RDIST_BASE_MASK;
|
||||
u32 count = (*addr & KVM_VGIC_V3_RDIST_COUNT_MASK)
|
||||
>> KVM_VGIC_V3_RDIST_COUNT_SHIFT;
|
||||
u8 flags = (*addr & KVM_VGIC_V3_RDIST_FLAGS_MASK)
|
||||
>> KVM_VGIC_V3_RDIST_FLAGS_SHIFT;
|
||||
|
||||
if (!count || flags)
|
||||
r = -EINVAL;
|
||||
else
|
||||
r = vgic_v3_set_redist_base(kvm, index,
|
||||
base, count);
|
||||
goto out;
|
||||
}
|
||||
|
||||
rdreg = vgic_v3_rdist_region_from_index(kvm, index);
|
||||
if (!rdreg) {
|
||||
r = -ENOENT;
|
||||
goto out;
|
||||
}
|
||||
|
||||
*addr = index;
|
||||
*addr |= rdreg->base;
|
||||
*addr |= (u64)rdreg->count << KVM_VGIC_V3_RDIST_COUNT_SHIFT;
|
||||
goto out;
|
||||
}
|
||||
default:
|
||||
r = -ENODEV;
|
||||
}
|
||||
|
@ -665,6 +711,7 @@ static int vgic_v3_has_attr(struct kvm_device *dev,
|
|||
switch (attr->attr) {
|
||||
case KVM_VGIC_V3_ADDR_TYPE_DIST:
|
||||
case KVM_VGIC_V3_ADDR_TYPE_REDIST:
|
||||
case KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION:
|
||||
return 0;
|
||||
}
|
||||
break;
|
||||
|
|
|
@ -184,12 +184,17 @@ static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu,
|
|||
gpa_t addr, unsigned int len)
|
||||
{
|
||||
unsigned long mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
|
||||
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
|
||||
struct vgic_redist_region *rdreg = vgic_cpu->rdreg;
|
||||
int target_vcpu_id = vcpu->vcpu_id;
|
||||
gpa_t last_rdist_typer = rdreg->base + GICR_TYPER +
|
||||
(rdreg->free_index - 1) * KVM_VGIC_V3_REDIST_SIZE;
|
||||
u64 value;
|
||||
|
||||
value = (u64)(mpidr & GENMASK(23, 0)) << 32;
|
||||
value |= ((target_vcpu_id & 0xffff) << 8);
|
||||
if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1)
|
||||
|
||||
if (addr == last_rdist_typer)
|
||||
value |= GICR_TYPER_LAST;
|
||||
if (vgic_has_its(vcpu->kvm))
|
||||
value |= GICR_TYPER_PLPIS;
|
||||
|
@ -580,24 +585,32 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu)
|
|||
{
|
||||
struct kvm *kvm = vcpu->kvm;
|
||||
struct vgic_dist *vgic = &kvm->arch.vgic;
|
||||
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
|
||||
struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
|
||||
struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev;
|
||||
struct vgic_redist_region *rdreg;
|
||||
gpa_t rd_base, sgi_base;
|
||||
int ret;
|
||||
|
||||
if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr))
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* We may be creating VCPUs before having set the base address for the
|
||||
* redistributor region, in which case we will come back to this
|
||||
* function for all VCPUs when the base address is set. Just return
|
||||
* without doing any work for now.
|
||||
*/
|
||||
if (IS_VGIC_ADDR_UNDEF(vgic->vgic_redist_base))
|
||||
rdreg = vgic_v3_rdist_free_slot(&vgic->rd_regions);
|
||||
if (!rdreg)
|
||||
return 0;
|
||||
|
||||
if (!vgic_v3_check_base(kvm))
|
||||
return -EINVAL;
|
||||
|
||||
rd_base = vgic->vgic_redist_base + vgic->vgic_redist_free_offset;
|
||||
vgic_cpu->rdreg = rdreg;
|
||||
|
||||
rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE;
|
||||
sgi_base = rd_base + SZ_64K;
|
||||
|
||||
kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops);
|
||||
|
@ -631,7 +644,7 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu)
|
|||
goto out;
|
||||
}
|
||||
|
||||
vgic->vgic_redist_free_offset += 2 * SZ_64K;
|
||||
rdreg->free_index++;
|
||||
out:
|
||||
mutex_unlock(&kvm->slots_lock);
|
||||
return ret;
|
||||
|
@ -670,22 +683,95 @@ static int vgic_register_all_redist_iodevs(struct kvm *kvm)
|
|||
return ret;
|
||||
}
|
||||
|
||||
int vgic_v3_set_redist_base(struct kvm *kvm, u64 addr)
|
||||
/**
|
||||
* vgic_v3_insert_redist_region - Insert a new redistributor region
|
||||
*
|
||||
* Performs various checks before inserting the rdist region in the list.
|
||||
* Those tests depend on whether the size of the rdist region is known
|
||||
* (ie. count != 0). The list is sorted by rdist region index.
|
||||
*
|
||||
* @kvm: kvm handle
|
||||
* @index: redist region index
|
||||
* @base: base of the new rdist region
|
||||
* @count: number of redistributors the region is made of (0 in the old style
|
||||
* single region, whose size is induced from the number of vcpus)
|
||||
*
|
||||
* Return 0 on success, < 0 otherwise
|
||||
*/
|
||||
static int vgic_v3_insert_redist_region(struct kvm *kvm, uint32_t index,
|
||||
gpa_t base, uint32_t count)
|
||||
{
|
||||
struct vgic_dist *vgic = &kvm->arch.vgic;
|
||||
struct vgic_dist *d = &kvm->arch.vgic;
|
||||
struct vgic_redist_region *rdreg;
|
||||
struct list_head *rd_regions = &d->rd_regions;
|
||||
size_t size = count * KVM_VGIC_V3_REDIST_SIZE;
|
||||
int ret;
|
||||
|
||||
/* vgic_check_ioaddr makes sure we don't do this twice */
|
||||
ret = vgic_check_ioaddr(kvm, &vgic->vgic_redist_base, addr, SZ_64K);
|
||||
/* single rdist region already set ?*/
|
||||
if (!count && !list_empty(rd_regions))
|
||||
return -EINVAL;
|
||||
|
||||
/* cross the end of memory ? */
|
||||
if (base + size < base)
|
||||
return -EINVAL;
|
||||
|
||||
if (list_empty(rd_regions)) {
|
||||
if (index != 0)
|
||||
return -EINVAL;
|
||||
} else {
|
||||
rdreg = list_last_entry(rd_regions,
|
||||
struct vgic_redist_region, list);
|
||||
if (index != rdreg->index + 1)
|
||||
return -EINVAL;
|
||||
|
||||
/* Cannot add an explicitly sized regions after legacy region */
|
||||
if (!rdreg->count)
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/*
|
||||
* For legacy single-region redistributor regions (!count),
|
||||
* check that the redistributor region does not overlap with the
|
||||
* distributor's address space.
|
||||
*/
|
||||
if (!count && !IS_VGIC_ADDR_UNDEF(d->vgic_dist_base) &&
|
||||
vgic_dist_overlap(kvm, base, size))
|
||||
return -EINVAL;
|
||||
|
||||
/* collision with any other rdist region? */
|
||||
if (vgic_v3_rdist_overlap(kvm, base, size))
|
||||
return -EINVAL;
|
||||
|
||||
rdreg = kzalloc(sizeof(*rdreg), GFP_KERNEL);
|
||||
if (!rdreg)
|
||||
return -ENOMEM;
|
||||
|
||||
rdreg->base = VGIC_ADDR_UNDEF;
|
||||
|
||||
ret = vgic_check_ioaddr(kvm, &rdreg->base, base, SZ_64K);
|
||||
if (ret)
|
||||
goto free;
|
||||
|
||||
rdreg->base = base;
|
||||
rdreg->count = count;
|
||||
rdreg->free_index = 0;
|
||||
rdreg->index = index;
|
||||
|
||||
list_add_tail(&rdreg->list, rd_regions);
|
||||
return 0;
|
||||
free:
|
||||
kfree(rdreg);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = vgic_v3_insert_redist_region(kvm, index, addr, count);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
vgic->vgic_redist_base = addr;
|
||||
if (!vgic_v3_check_base(kvm)) {
|
||||
vgic->vgic_redist_base = VGIC_ADDR_UNDEF;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Register iodevs for each existing VCPU. Adding more VCPUs
|
||||
* afterwards will register the iodevs when needed.
|
||||
|
|
|
@ -419,6 +419,29 @@ int vgic_v3_save_pending_tables(struct kvm *kvm)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* vgic_v3_rdist_overlap - check if a region overlaps with any
|
||||
* existing redistributor region
|
||||
*
|
||||
* @kvm: kvm handle
|
||||
* @base: base of the region
|
||||
* @size: size of region
|
||||
*
|
||||
* Return: true if there is an overlap
|
||||
*/
|
||||
bool vgic_v3_rdist_overlap(struct kvm *kvm, gpa_t base, size_t size)
|
||||
{
|
||||
struct vgic_dist *d = &kvm->arch.vgic;
|
||||
struct vgic_redist_region *rdreg;
|
||||
|
||||
list_for_each_entry(rdreg, &d->rd_regions, list) {
|
||||
if ((base + size > rdreg->base) &&
|
||||
(base < rdreg->base + vgic_v3_rd_region_size(kvm, rdreg)))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check for overlapping regions and for regions crossing the end of memory
|
||||
* for base addresses which have already been set.
|
||||
|
@ -426,41 +449,83 @@ int vgic_v3_save_pending_tables(struct kvm *kvm)
|
|||
bool vgic_v3_check_base(struct kvm *kvm)
|
||||
{
|
||||
struct vgic_dist *d = &kvm->arch.vgic;
|
||||
gpa_t redist_size = KVM_VGIC_V3_REDIST_SIZE;
|
||||
|
||||
redist_size *= atomic_read(&kvm->online_vcpus);
|
||||
struct vgic_redist_region *rdreg;
|
||||
|
||||
if (!IS_VGIC_ADDR_UNDEF(d->vgic_dist_base) &&
|
||||
d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE < d->vgic_dist_base)
|
||||
return false;
|
||||
|
||||
if (!IS_VGIC_ADDR_UNDEF(d->vgic_redist_base) &&
|
||||
d->vgic_redist_base + redist_size < d->vgic_redist_base)
|
||||
return false;
|
||||
list_for_each_entry(rdreg, &d->rd_regions, list) {
|
||||
if (rdreg->base + vgic_v3_rd_region_size(kvm, rdreg) <
|
||||
rdreg->base)
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Both base addresses must be set to check if they overlap */
|
||||
if (IS_VGIC_ADDR_UNDEF(d->vgic_dist_base) ||
|
||||
IS_VGIC_ADDR_UNDEF(d->vgic_redist_base))
|
||||
if (IS_VGIC_ADDR_UNDEF(d->vgic_dist_base))
|
||||
return true;
|
||||
|
||||
if (d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE <= d->vgic_redist_base)
|
||||
return true;
|
||||
if (d->vgic_redist_base + redist_size <= d->vgic_dist_base)
|
||||
return true;
|
||||
|
||||
return false;
|
||||
return !vgic_v3_rdist_overlap(kvm, d->vgic_dist_base,
|
||||
KVM_VGIC_V3_DIST_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* vgic_v3_rdist_free_slot - Look up registered rdist regions and identify one
|
||||
* which has free space to put a new rdist region.
|
||||
*
|
||||
* @rd_regions: redistributor region list head
|
||||
*
|
||||
* A redistributor regions maps n redistributors, n = region size / (2 x 64kB).
|
||||
* Stride between redistributors is 0 and regions are filled in the index order.
|
||||
*
|
||||
* Return: the redist region handle, if any, that has space to map a new rdist
|
||||
* region.
|
||||
*/
|
||||
struct vgic_redist_region *vgic_v3_rdist_free_slot(struct list_head *rd_regions)
|
||||
{
|
||||
struct vgic_redist_region *rdreg;
|
||||
|
||||
list_for_each_entry(rdreg, rd_regions, list) {
|
||||
if (!vgic_v3_redist_region_full(rdreg))
|
||||
return rdreg;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct vgic_redist_region *vgic_v3_rdist_region_from_index(struct kvm *kvm,
|
||||
u32 index)
|
||||
{
|
||||
struct list_head *rd_regions = &kvm->arch.vgic.rd_regions;
|
||||
struct vgic_redist_region *rdreg;
|
||||
|
||||
list_for_each_entry(rdreg, rd_regions, list) {
|
||||
if (rdreg->index == index)
|
||||
return rdreg;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
int vgic_v3_map_resources(struct kvm *kvm)
|
||||
{
|
||||
int ret = 0;
|
||||
struct vgic_dist *dist = &kvm->arch.vgic;
|
||||
struct kvm_vcpu *vcpu;
|
||||
int ret = 0;
|
||||
int c;
|
||||
|
||||
if (vgic_ready(kvm))
|
||||
goto out;
|
||||
|
||||
if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
|
||||
IS_VGIC_ADDR_UNDEF(dist->vgic_redist_base)) {
|
||||
kvm_for_each_vcpu(c, vcpu, kvm) {
|
||||
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
|
||||
|
||||
if (IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr)) {
|
||||
kvm_debug("vcpu %d redistributor base not set\n", c);
|
||||
ret = -ENXIO;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base)) {
|
||||
kvm_err("Need to set vgic distributor addresses first\n");
|
||||
ret = -ENXIO;
|
||||
goto out;
|
||||
|
|
|
@ -96,6 +96,13 @@
|
|||
/* we only support 64 kB translation table page size */
|
||||
#define KVM_ITS_L1E_ADDR_MASK GENMASK_ULL(51, 16)
|
||||
|
||||
#define KVM_VGIC_V3_RDIST_INDEX_MASK GENMASK_ULL(11, 0)
|
||||
#define KVM_VGIC_V3_RDIST_FLAGS_MASK GENMASK_ULL(15, 12)
|
||||
#define KVM_VGIC_V3_RDIST_FLAGS_SHIFT 12
|
||||
#define KVM_VGIC_V3_RDIST_BASE_MASK GENMASK_ULL(51, 16)
|
||||
#define KVM_VGIC_V3_RDIST_COUNT_MASK GENMASK_ULL(63, 52)
|
||||
#define KVM_VGIC_V3_RDIST_COUNT_SHIFT 52
|
||||
|
||||
/* Requires the irq_lock to be held by the caller. */
|
||||
static inline bool irq_is_pending(struct vgic_irq *irq)
|
||||
{
|
||||
|
@ -215,7 +222,7 @@ int vgic_v3_probe(const struct gic_kvm_info *info);
|
|||
int vgic_v3_map_resources(struct kvm *kvm);
|
||||
int vgic_v3_lpi_sync_pending_status(struct kvm *kvm, struct vgic_irq *irq);
|
||||
int vgic_v3_save_pending_tables(struct kvm *kvm);
|
||||
int vgic_v3_set_redist_base(struct kvm *kvm, u64 addr);
|
||||
int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count);
|
||||
int vgic_register_redist_iodev(struct kvm_vcpu *vcpu);
|
||||
bool vgic_v3_check_base(struct kvm *kvm);
|
||||
|
||||
|
@ -243,8 +250,8 @@ void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
|
|||
int vgic_lazy_init(struct kvm *kvm);
|
||||
int vgic_init(struct kvm *kvm);
|
||||
|
||||
int vgic_debug_init(struct kvm *kvm);
|
||||
int vgic_debug_destroy(struct kvm *kvm);
|
||||
void vgic_debug_init(struct kvm *kvm);
|
||||
void vgic_debug_destroy(struct kvm *kvm);
|
||||
|
||||
bool lock_all_vcpus(struct kvm *kvm);
|
||||
void unlock_all_vcpus(struct kvm *kvm);
|
||||
|
@ -265,6 +272,39 @@ static inline int vgic_v3_max_apr_idx(struct kvm_vcpu *vcpu)
|
|||
}
|
||||
}
|
||||
|
||||
static inline bool
|
||||
vgic_v3_redist_region_full(struct vgic_redist_region *region)
|
||||
{
|
||||
if (!region->count)
|
||||
return false;
|
||||
|
||||
return (region->free_index >= region->count);
|
||||
}
|
||||
|
||||
struct vgic_redist_region *vgic_v3_rdist_free_slot(struct list_head *rdregs);
|
||||
|
||||
static inline size_t
|
||||
vgic_v3_rd_region_size(struct kvm *kvm, struct vgic_redist_region *rdreg)
|
||||
{
|
||||
if (!rdreg->count)
|
||||
return atomic_read(&kvm->online_vcpus) * KVM_VGIC_V3_REDIST_SIZE;
|
||||
else
|
||||
return rdreg->count * KVM_VGIC_V3_REDIST_SIZE;
|
||||
}
|
||||
|
||||
struct vgic_redist_region *vgic_v3_rdist_region_from_index(struct kvm *kvm,
|
||||
u32 index);
|
||||
|
||||
bool vgic_v3_rdist_overlap(struct kvm *kvm, gpa_t base, size_t size);
|
||||
|
||||
static inline bool vgic_dist_overlap(struct kvm *kvm, gpa_t base, size_t size)
|
||||
{
|
||||
struct vgic_dist *d = &kvm->arch.vgic;
|
||||
|
||||
return (base + size > d->vgic_dist_base) &&
|
||||
(base < d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE);
|
||||
}
|
||||
|
||||
int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
|
||||
u32 devid, u32 eventid, struct vgic_irq **irq);
|
||||
struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi);
|
||||
|
|
|
@ -203,29 +203,47 @@ static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait)
|
|||
return true;
|
||||
}
|
||||
|
||||
bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
|
||||
bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
|
||||
unsigned long *vcpu_bitmap, cpumask_var_t tmp)
|
||||
{
|
||||
int i, cpu, me;
|
||||
cpumask_var_t cpus;
|
||||
bool called;
|
||||
struct kvm_vcpu *vcpu;
|
||||
|
||||
zalloc_cpumask_var(&cpus, GFP_ATOMIC);
|
||||
bool called;
|
||||
|
||||
me = get_cpu();
|
||||
|
||||
kvm_for_each_vcpu(i, vcpu, kvm) {
|
||||
if (!test_bit(i, vcpu_bitmap))
|
||||
continue;
|
||||
|
||||
kvm_make_request(req, vcpu);
|
||||
cpu = vcpu->cpu;
|
||||
|
||||
if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
|
||||
continue;
|
||||
|
||||
if (cpus != NULL && cpu != -1 && cpu != me &&
|
||||
if (tmp != NULL && cpu != -1 && cpu != me &&
|
||||
kvm_request_needs_ipi(vcpu, req))
|
||||
__cpumask_set_cpu(cpu, cpus);
|
||||
__cpumask_set_cpu(cpu, tmp);
|
||||
}
|
||||
called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT));
|
||||
|
||||
called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT));
|
||||
put_cpu();
|
||||
|
||||
return called;
|
||||
}
|
||||
|
||||
bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
|
||||
{
|
||||
cpumask_var_t cpus;
|
||||
bool called;
|
||||
static unsigned long vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)]
|
||||
= {[0 ... BITS_TO_LONGS(KVM_MAX_VCPUS)-1] = ULONG_MAX};
|
||||
|
||||
zalloc_cpumask_var(&cpus, GFP_ATOMIC);
|
||||
|
||||
called = kvm_make_vcpus_request_mask(kvm, req, vcpu_bitmap, cpus);
|
||||
|
||||
free_cpumask_var(cpus);
|
||||
return called;
|
||||
}
|
||||
|
@ -572,10 +590,7 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
|
|||
return 0;
|
||||
|
||||
snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
|
||||
kvm->debugfs_dentry = debugfs_create_dir(dir_name,
|
||||
kvm_debugfs_dir);
|
||||
if (!kvm->debugfs_dentry)
|
||||
return -ENOMEM;
|
||||
kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir);
|
||||
|
||||
kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
|
||||
sizeof(*kvm->debugfs_stat_data),
|
||||
|
@ -591,11 +606,8 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
|
|||
stat_data->kvm = kvm;
|
||||
stat_data->offset = p->offset;
|
||||
kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
|
||||
if (!debugfs_create_file(p->name, 0644,
|
||||
kvm->debugfs_dentry,
|
||||
stat_data,
|
||||
stat_fops_per_vm[p->kind]))
|
||||
return -ENOMEM;
|
||||
debugfs_create_file(p->name, 0644, kvm->debugfs_dentry,
|
||||
stat_data, stat_fops_per_vm[p->kind]);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
@ -2340,7 +2352,7 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
|
||||
|
||||
static int kvm_vcpu_fault(struct vm_fault *vmf)
|
||||
static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
|
||||
{
|
||||
struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
|
||||
struct page *page;
|
||||
|
@ -2550,8 +2562,13 @@ static long kvm_vcpu_ioctl(struct file *filp,
|
|||
oldpid = rcu_access_pointer(vcpu->pid);
|
||||
if (unlikely(oldpid != current->pids[PIDTYPE_PID].pid)) {
|
||||
/* The thread running this VCPU changed. */
|
||||
struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
|
||||
struct pid *newpid;
|
||||
|
||||
r = kvm_arch_vcpu_run_pid_change(vcpu);
|
||||
if (r)
|
||||
break;
|
||||
|
||||
newpid = get_task_pid(current, PIDTYPE_PID);
|
||||
rcu_assign_pointer(vcpu->pid, newpid);
|
||||
if (oldpid)
|
||||
synchronize_rcu();
|
||||
|
@ -3896,29 +3913,18 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
|
|||
kfree(env);
|
||||
}
|
||||
|
||||
static int kvm_init_debug(void)
|
||||
static void kvm_init_debug(void)
|
||||
{
|
||||
int r = -EEXIST;
|
||||
struct kvm_stats_debugfs_item *p;
|
||||
|
||||
kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
|
||||
if (kvm_debugfs_dir == NULL)
|
||||
goto out;
|
||||
|
||||
kvm_debugfs_num_entries = 0;
|
||||
for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
|
||||
if (!debugfs_create_file(p->name, 0644, kvm_debugfs_dir,
|
||||
(void *)(long)p->offset,
|
||||
stat_fops[p->kind]))
|
||||
goto out_dir;
|
||||
debugfs_create_file(p->name, 0644, kvm_debugfs_dir,
|
||||
(void *)(long)p->offset,
|
||||
stat_fops[p->kind]);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
out_dir:
|
||||
debugfs_remove_recursive(kvm_debugfs_dir);
|
||||
out:
|
||||
return r;
|
||||
}
|
||||
|
||||
static int kvm_suspend(void)
|
||||
|
@ -4046,20 +4052,13 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
|
|||
kvm_preempt_ops.sched_in = kvm_sched_in;
|
||||
kvm_preempt_ops.sched_out = kvm_sched_out;
|
||||
|
||||
r = kvm_init_debug();
|
||||
if (r) {
|
||||
pr_err("kvm: create debugfs files failed\n");
|
||||
goto out_undebugfs;
|
||||
}
|
||||
kvm_init_debug();
|
||||
|
||||
r = kvm_vfio_ops_init();
|
||||
WARN_ON(r);
|
||||
|
||||
return 0;
|
||||
|
||||
out_undebugfs:
|
||||
unregister_syscore_ops(&kvm_syscore_ops);
|
||||
misc_deregister(&kvm_dev);
|
||||
out_unreg:
|
||||
kvm_async_pf_deinit();
|
||||
out_free:
|
||||
|
|
Loading…
Reference in a new issue