From 0b5e3bac30c545720f7e6b026241b5f8dd832df2 Mon Sep 17 00:00:00 2001 From: Jonghwan Choi Date: Tue, 19 Feb 2013 15:19:32 +0900 Subject: [PATCH 01/31] KVM: ARM: Fix wrong address in comment hyp_hvc vector offset is 0x14 and hyp_svc vector offset is 0x8. Signed-off-by: Jonghwan Choi Signed-off-by: Christoffer Dall --- arch/arm/kvm/interrupts.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/kvm/interrupts.S b/arch/arm/kvm/interrupts.S index 8ca87ab0919d..a8e0c2d85cb5 100644 --- a/arch/arm/kvm/interrupts.S +++ b/arch/arm/kvm/interrupts.S @@ -235,9 +235,9 @@ ENTRY(kvm_call_hyp) * instruction is issued since all traps are disabled when running the host * kernel as per the Hyp-mode initialization at boot time. * - * HVC instructions cause a trap to the vector page + offset 0x18 (see hyp_hvc + * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc * below) when the HVC instruction is called from SVC mode (i.e. a guest or the - * host kernel) and they cause a trap to the vector page + offset 0xc when HVC + * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC * instructions are called from within Hyp-mode. * * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode): From db730d8d623a0826f7fb6b74e890d3eb97a1b7a3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 3 Oct 2012 11:17:02 +0100 Subject: [PATCH 02/31] ARM: KVM: convert GP registers from u32 to unsigned long On 32bit ARM, unsigned long is guaranteed to be a 32bit quantity. On 64bit ARM, it is a 64bit quantity. In order to be able to share code between the two architectures, convert the registers to be unsigned long, so the core code can be oblivious of the change. Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_emulate.h | 12 ++++++------ arch/arm/include/uapi/asm/kvm.h | 12 ++++++------ arch/arm/kvm/coproc.c | 4 ++-- arch/arm/kvm/coproc.h | 4 ++-- arch/arm/kvm/emulate.c | 22 +++++++++++----------- arch/arm/kvm/mmio.c | 2 +- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index fd611996bfb5..510488ad30bd 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -23,8 +23,8 @@ #include #include -u32 *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num); -u32 *vcpu_spsr(struct kvm_vcpu *vcpu); +unsigned long *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num); +unsigned long *vcpu_spsr(struct kvm_vcpu *vcpu); int kvm_handle_wfi(struct kvm_vcpu *vcpu, struct kvm_run *run); void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr); @@ -37,14 +37,14 @@ static inline bool vcpu_mode_is_32bit(struct kvm_vcpu *vcpu) return 1; } -static inline u32 *vcpu_pc(struct kvm_vcpu *vcpu) +static inline unsigned long *vcpu_pc(struct kvm_vcpu *vcpu) { - return (u32 *)&vcpu->arch.regs.usr_regs.ARM_pc; + return &vcpu->arch.regs.usr_regs.ARM_pc; } -static inline u32 *vcpu_cpsr(struct kvm_vcpu *vcpu) +static inline unsigned long *vcpu_cpsr(struct kvm_vcpu *vcpu) { - return (u32 *)&vcpu->arch.regs.usr_regs.ARM_cpsr; + return &vcpu->arch.regs.usr_regs.ARM_cpsr; } static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu) diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h index 023bfeb367bf..c1ee007523d7 100644 --- a/arch/arm/include/uapi/asm/kvm.h +++ b/arch/arm/include/uapi/asm/kvm.h @@ -53,12 +53,12 @@ #define KVM_ARM_FIQ_spsr fiq_regs[7] struct kvm_regs { - struct pt_regs usr_regs;/* R0_usr - R14_usr, PC, CPSR */ - __u32 svc_regs[3]; /* SP_svc, LR_svc, SPSR_svc */ - __u32 abt_regs[3]; /* SP_abt, LR_abt, SPSR_abt */ - __u32 und_regs[3]; /* SP_und, LR_und, SPSR_und */ - __u32 irq_regs[3]; /* SP_irq, LR_irq, SPSR_irq */ - __u32 fiq_regs[8]; /* R8_fiq - R14_fiq, SPSR_fiq */ + struct pt_regs usr_regs; /* R0_usr - R14_usr, PC, CPSR */ + unsigned long svc_regs[3]; /* SP_svc, LR_svc, SPSR_svc */ + unsigned long abt_regs[3]; /* SP_abt, LR_abt, SPSR_abt */ + unsigned long und_regs[3]; /* SP_und, LR_und, SPSR_und */ + unsigned long irq_regs[3]; /* SP_irq, LR_irq, SPSR_irq */ + unsigned long fiq_regs[8]; /* R8_fiq - R14_fiq, SPSR_fiq */ }; /* Supported Processor Types */ diff --git a/arch/arm/kvm/coproc.c b/arch/arm/kvm/coproc.c index 4ea9a982269c..38e76bcb52a4 100644 --- a/arch/arm/kvm/coproc.c +++ b/arch/arm/kvm/coproc.c @@ -76,7 +76,7 @@ static bool access_dcsw(struct kvm_vcpu *vcpu, const struct coproc_params *p, const struct coproc_reg *r) { - u32 val; + unsigned long val; int cpu; cpu = get_cpu(); @@ -298,7 +298,7 @@ static int emulate_cp15(struct kvm_vcpu *vcpu, } /* If access function fails, it should complain. */ } else { - kvm_err("Unsupported guest CP15 access at: %08x\n", + kvm_err("Unsupported guest CP15 access at: %08lx\n", *vcpu_pc(vcpu)); print_cp_instr(params); } diff --git a/arch/arm/kvm/coproc.h b/arch/arm/kvm/coproc.h index 992adfafa2ff..b7301d3e4799 100644 --- a/arch/arm/kvm/coproc.h +++ b/arch/arm/kvm/coproc.h @@ -84,7 +84,7 @@ static inline bool read_zero(struct kvm_vcpu *vcpu, static inline bool write_to_read_only(struct kvm_vcpu *vcpu, const struct coproc_params *params) { - kvm_debug("CP15 write to read-only register at: %08x\n", + kvm_debug("CP15 write to read-only register at: %08lx\n", *vcpu_pc(vcpu)); print_cp_instr(params); return false; @@ -93,7 +93,7 @@ static inline bool write_to_read_only(struct kvm_vcpu *vcpu, static inline bool read_from_write_only(struct kvm_vcpu *vcpu, const struct coproc_params *params) { - kvm_debug("CP15 read to write-only register at: %08x\n", + kvm_debug("CP15 read to write-only register at: %08lx\n", *vcpu_pc(vcpu)); print_cp_instr(params); return false; diff --git a/arch/arm/kvm/emulate.c b/arch/arm/kvm/emulate.c index d61450ac6665..d3094eb4ade6 100644 --- a/arch/arm/kvm/emulate.c +++ b/arch/arm/kvm/emulate.c @@ -109,10 +109,10 @@ static const unsigned long vcpu_reg_offsets[VCPU_NR_MODES][15] = { * Return a pointer to the register number valid in the current mode of * the virtual CPU. */ -u32 *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num) +unsigned long *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num) { - u32 *reg_array = (u32 *)&vcpu->arch.regs; - u32 mode = *vcpu_cpsr(vcpu) & MODE_MASK; + unsigned long *reg_array = (unsigned long *)&vcpu->arch.regs; + unsigned long mode = *vcpu_cpsr(vcpu) & MODE_MASK; switch (mode) { case USR_MODE...SVC_MODE: @@ -141,9 +141,9 @@ u32 *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num) /* * Return the SPSR for the current mode of the virtual CPU. */ -u32 *vcpu_spsr(struct kvm_vcpu *vcpu) +unsigned long *vcpu_spsr(struct kvm_vcpu *vcpu) { - u32 mode = *vcpu_cpsr(vcpu) & MODE_MASK; + unsigned long mode = *vcpu_cpsr(vcpu) & MODE_MASK; switch (mode) { case SVC_MODE: return &vcpu->arch.regs.KVM_ARM_SVC_spsr; @@ -257,9 +257,9 @@ static u32 exc_vector_base(struct kvm_vcpu *vcpu) */ void kvm_inject_undefined(struct kvm_vcpu *vcpu) { - u32 new_lr_value; - u32 new_spsr_value; - u32 cpsr = *vcpu_cpsr(vcpu); + unsigned long new_lr_value; + unsigned long new_spsr_value; + unsigned long cpsr = *vcpu_cpsr(vcpu); u32 sctlr = vcpu->arch.cp15[c1_SCTLR]; bool is_thumb = (cpsr & PSR_T_BIT); u32 vect_offset = 4; @@ -291,9 +291,9 @@ void kvm_inject_undefined(struct kvm_vcpu *vcpu) */ static void inject_abt(struct kvm_vcpu *vcpu, bool is_pabt, unsigned long addr) { - u32 new_lr_value; - u32 new_spsr_value; - u32 cpsr = *vcpu_cpsr(vcpu); + unsigned long new_lr_value; + unsigned long new_spsr_value; + unsigned long cpsr = *vcpu_cpsr(vcpu); u32 sctlr = vcpu->arch.cp15[c1_SCTLR]; bool is_thumb = (cpsr & PSR_T_BIT); u32 vect_offset; diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index 98a870ff1a5c..c186bc910715 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -33,7 +33,7 @@ */ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) { - __u32 *dest; + unsigned long *dest; unsigned int len; int mask; From 7393b599177d301d4c9ca2c7f69a6849aba793c7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 17 Sep 2012 19:27:09 +0100 Subject: [PATCH 03/31] ARM: KVM: abstract fault register accesses Instead of directly accessing the fault registers, use proper accessors so the core code can be shared. Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_emulate.h | 21 +++++++++++++++++++++ arch/arm/include/asm/kvm_host.h | 14 ++++++++------ arch/arm/kernel/asm-offsets.c | 8 ++++---- arch/arm/kvm/arm.c | 28 ++++++++++++++-------------- arch/arm/kvm/coproc.c | 24 ++++++++++++------------ arch/arm/kvm/mmio.c | 22 +++++++++++----------- arch/arm/kvm/mmu.c | 16 ++++++++-------- 7 files changed, 78 insertions(+), 55 deletions(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index 510488ad30bd..3c01988b3c88 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -22,6 +22,7 @@ #include #include #include +#include unsigned long *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num); unsigned long *vcpu_spsr(struct kvm_vcpu *vcpu); @@ -69,4 +70,24 @@ static inline bool kvm_vcpu_reg_is_pc(struct kvm_vcpu *vcpu, int reg) return reg == 15; } +static inline u32 kvm_vcpu_get_hsr(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.fault.hsr; +} + +static inline unsigned long kvm_vcpu_get_hfar(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.fault.hxfar; +} + +static inline phys_addr_t kvm_vcpu_get_fault_ipa(struct kvm_vcpu *vcpu) +{ + return ((phys_addr_t)vcpu->arch.fault.hpfar & HPFAR_MASK) << 8; +} + +static inline unsigned long kvm_vcpu_get_hyp_pc(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.fault.hyp_pc; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h index d1736a53b12d..eb836e6d3c59 100644 --- a/arch/arm/include/asm/kvm_host.h +++ b/arch/arm/include/asm/kvm_host.h @@ -80,6 +80,13 @@ struct kvm_mmu_memory_cache { void *objects[KVM_NR_MEM_OBJS]; }; +struct kvm_vcpu_fault_info { + u32 hsr; /* Hyp Syndrome Register */ + u32 hxfar; /* Hyp Data/Inst. Fault Address Register */ + u32 hpfar; /* Hyp IPA Fault Address Register */ + u32 hyp_pc; /* PC when exception was taken from Hyp mode */ +}; + struct kvm_vcpu_arch { struct kvm_regs regs; @@ -93,9 +100,7 @@ struct kvm_vcpu_arch { u32 midr; /* Exception Information */ - u32 hsr; /* Hyp Syndrome Register */ - u32 hxfar; /* Hyp Data/Inst Fault Address Register */ - u32 hpfar; /* Hyp IPA Fault Address Register */ + struct kvm_vcpu_fault_info fault; /* Floating point registers (VFP and Advanced SIMD/NEON) */ struct vfp_hard_struct vfp_guest; @@ -122,9 +127,6 @@ struct kvm_vcpu_arch { /* Interrupt related fields */ u32 irq_lines; /* IRQ and FIQ levels */ - /* Hyp exception information */ - u32 hyp_pc; /* PC when exception was taken from Hyp mode */ - /* Cache some mmu pages needed inside spinlock regions */ struct kvm_mmu_memory_cache mmu_page_cache; diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 5ce738b43508..3c09d8c7798b 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -165,10 +165,10 @@ int main(void) DEFINE(VCPU_PC, offsetof(struct kvm_vcpu, arch.regs.usr_regs.ARM_pc)); DEFINE(VCPU_CPSR, offsetof(struct kvm_vcpu, arch.regs.usr_regs.ARM_cpsr)); DEFINE(VCPU_IRQ_LINES, offsetof(struct kvm_vcpu, arch.irq_lines)); - DEFINE(VCPU_HSR, offsetof(struct kvm_vcpu, arch.hsr)); - DEFINE(VCPU_HxFAR, offsetof(struct kvm_vcpu, arch.hxfar)); - DEFINE(VCPU_HPFAR, offsetof(struct kvm_vcpu, arch.hpfar)); - DEFINE(VCPU_HYP_PC, offsetof(struct kvm_vcpu, arch.hyp_pc)); + DEFINE(VCPU_HSR, offsetof(struct kvm_vcpu, arch.fault.hsr)); + DEFINE(VCPU_HxFAR, offsetof(struct kvm_vcpu, arch.fault.hxfar)); + DEFINE(VCPU_HPFAR, offsetof(struct kvm_vcpu, arch.fault.hpfar)); + DEFINE(VCPU_HYP_PC, offsetof(struct kvm_vcpu, arch.fault.hyp_pc)); #ifdef CONFIG_KVM_ARM_VGIC DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu)); DEFINE(VGIC_CPU_HCR, offsetof(struct vgic_cpu, vgic_hcr)); diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index 5a936988eb24..6626c7e62371 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -492,7 +492,7 @@ static int handle_svc_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) { trace_kvm_hvc(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0), - vcpu->arch.hsr & HSR_HVC_IMM_MASK); + kvm_vcpu_get_hsr(vcpu) & HSR_HVC_IMM_MASK); if (kvm_psci_call(vcpu)) return 1; @@ -513,16 +513,16 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run) static int handle_pabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) { /* The hypervisor should never cause aborts */ - kvm_err("Prefetch Abort taken from Hyp mode at %#08x (HSR: %#08x)\n", - vcpu->arch.hxfar, vcpu->arch.hsr); + kvm_err("Prefetch Abort taken from Hyp mode at %#08lx (HSR: %#08x)\n", + kvm_vcpu_get_hfar(vcpu), kvm_vcpu_get_hsr(vcpu)); return -EFAULT; } static int handle_dabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) { /* This is either an error in the ws. code or an external abort */ - kvm_err("Data Abort taken from Hyp mode at %#08x (HSR: %#08x)\n", - vcpu->arch.hxfar, vcpu->arch.hsr); + kvm_err("Data Abort taken from Hyp mode at %#08lx (HSR: %#08x)\n", + kvm_vcpu_get_hfar(vcpu), kvm_vcpu_get_hsr(vcpu)); return -EFAULT; } @@ -559,17 +559,17 @@ static bool kvm_condition_valid(struct kvm_vcpu *vcpu) * catch undefined instructions, and then we won't get past * the arm_exit_handlers test anyway. */ - BUG_ON(((vcpu->arch.hsr & HSR_EC) >> HSR_EC_SHIFT) == 0); + BUG_ON(((kvm_vcpu_get_hsr(vcpu) & HSR_EC) >> HSR_EC_SHIFT) == 0); /* Top two bits non-zero? Unconditional. */ - if (vcpu->arch.hsr >> 30) + if (kvm_vcpu_get_hsr(vcpu) >> 30) return true; cpsr = *vcpu_cpsr(vcpu); /* Is condition field valid? */ - if ((vcpu->arch.hsr & HSR_CV) >> HSR_CV_SHIFT) - cond = (vcpu->arch.hsr & HSR_COND) >> HSR_COND_SHIFT; + if ((kvm_vcpu_get_hsr(vcpu) & HSR_CV) >> HSR_CV_SHIFT) + cond = (kvm_vcpu_get_hsr(vcpu) & HSR_COND) >> HSR_COND_SHIFT; else { /* This can happen in Thumb mode: examine IT state. */ unsigned long it; @@ -602,20 +602,20 @@ static int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, case ARM_EXCEPTION_IRQ: return 1; case ARM_EXCEPTION_UNDEFINED: - kvm_err("Undefined exception in Hyp mode at: %#08x\n", - vcpu->arch.hyp_pc); + kvm_err("Undefined exception in Hyp mode at: %#08lx\n", + kvm_vcpu_get_hyp_pc(vcpu)); BUG(); panic("KVM: Hypervisor undefined exception!\n"); case ARM_EXCEPTION_DATA_ABORT: case ARM_EXCEPTION_PREF_ABORT: case ARM_EXCEPTION_HVC: - hsr_ec = (vcpu->arch.hsr & HSR_EC) >> HSR_EC_SHIFT; + hsr_ec = (kvm_vcpu_get_hsr(vcpu) & HSR_EC) >> HSR_EC_SHIFT; if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) || !arm_exit_handlers[hsr_ec]) { kvm_err("Unkown exception class: %#08lx, " "hsr: %#08x\n", hsr_ec, - (unsigned int)vcpu->arch.hsr); + (unsigned int)kvm_vcpu_get_hsr(vcpu)); BUG(); } @@ -624,7 +624,7 @@ static int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, * that fail their condition code check" */ if (!kvm_condition_valid(vcpu)) { - bool is_wide = vcpu->arch.hsr & HSR_IL; + bool is_wide = kvm_vcpu_get_hsr(vcpu) & HSR_IL; kvm_skip_instr(vcpu, is_wide); return 1; } diff --git a/arch/arm/kvm/coproc.c b/arch/arm/kvm/coproc.c index 38e76bcb52a4..b30591615da4 100644 --- a/arch/arm/kvm/coproc.c +++ b/arch/arm/kvm/coproc.c @@ -293,7 +293,7 @@ static int emulate_cp15(struct kvm_vcpu *vcpu, if (likely(r->access(vcpu, params, r))) { /* Skip instruction, since it was emulated */ - kvm_skip_instr(vcpu, (vcpu->arch.hsr >> 25) & 1); + kvm_skip_instr(vcpu, (kvm_vcpu_get_hsr(vcpu) >> 25) & 1); return 1; } /* If access function fails, it should complain. */ @@ -315,14 +315,14 @@ int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run) { struct coproc_params params; - params.CRm = (vcpu->arch.hsr >> 1) & 0xf; - params.Rt1 = (vcpu->arch.hsr >> 5) & 0xf; - params.is_write = ((vcpu->arch.hsr & 1) == 0); + params.CRm = (kvm_vcpu_get_hsr(vcpu) >> 1) & 0xf; + params.Rt1 = (kvm_vcpu_get_hsr(vcpu) >> 5) & 0xf; + params.is_write = ((kvm_vcpu_get_hsr(vcpu) & 1) == 0); params.is_64bit = true; - params.Op1 = (vcpu->arch.hsr >> 16) & 0xf; + params.Op1 = (kvm_vcpu_get_hsr(vcpu) >> 16) & 0xf; params.Op2 = 0; - params.Rt2 = (vcpu->arch.hsr >> 10) & 0xf; + params.Rt2 = (kvm_vcpu_get_hsr(vcpu) >> 10) & 0xf; params.CRn = 0; return emulate_cp15(vcpu, ¶ms); @@ -347,14 +347,14 @@ int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run) { struct coproc_params params; - params.CRm = (vcpu->arch.hsr >> 1) & 0xf; - params.Rt1 = (vcpu->arch.hsr >> 5) & 0xf; - params.is_write = ((vcpu->arch.hsr & 1) == 0); + params.CRm = (kvm_vcpu_get_hsr(vcpu) >> 1) & 0xf; + params.Rt1 = (kvm_vcpu_get_hsr(vcpu) >> 5) & 0xf; + params.is_write = ((kvm_vcpu_get_hsr(vcpu) & 1) == 0); params.is_64bit = false; - params.CRn = (vcpu->arch.hsr >> 10) & 0xf; - params.Op1 = (vcpu->arch.hsr >> 14) & 0x7; - params.Op2 = (vcpu->arch.hsr >> 17) & 0x7; + params.CRn = (kvm_vcpu_get_hsr(vcpu) >> 10) & 0xf; + params.Op1 = (kvm_vcpu_get_hsr(vcpu) >> 14) & 0x7; + params.Op2 = (kvm_vcpu_get_hsr(vcpu) >> 17) & 0x7; params.Rt2 = 0; return emulate_cp15(vcpu, ¶ms); diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index c186bc910715..ce63f39071ad 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -65,19 +65,19 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, unsigned long rt, len; bool is_write, sign_extend; - if ((vcpu->arch.hsr >> 8) & 1) { + if ((kvm_vcpu_get_hsr(vcpu) >> 8) & 1) { /* cache operation on I/O addr, tell guest unsupported */ - kvm_inject_dabt(vcpu, vcpu->arch.hxfar); + kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); return 1; } - if ((vcpu->arch.hsr >> 7) & 1) { + if ((kvm_vcpu_get_hsr(vcpu) >> 7) & 1) { /* page table accesses IO mem: tell guest to fix its TTBR */ - kvm_inject_dabt(vcpu, vcpu->arch.hxfar); + kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); return 1; } - switch ((vcpu->arch.hsr >> 22) & 0x3) { + switch ((kvm_vcpu_get_hsr(vcpu) >> 22) & 0x3) { case 0: len = 1; break; @@ -92,13 +92,13 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, return -EFAULT; } - is_write = vcpu->arch.hsr & HSR_WNR; - sign_extend = vcpu->arch.hsr & HSR_SSE; - rt = (vcpu->arch.hsr & HSR_SRT_MASK) >> HSR_SRT_SHIFT; + is_write = kvm_vcpu_get_hsr(vcpu) & HSR_WNR; + sign_extend = kvm_vcpu_get_hsr(vcpu) & HSR_SSE; + rt = (kvm_vcpu_get_hsr(vcpu) & HSR_SRT_MASK) >> HSR_SRT_SHIFT; if (kvm_vcpu_reg_is_pc(vcpu, rt)) { /* IO memory trying to read/write pc */ - kvm_inject_pabt(vcpu, vcpu->arch.hxfar); + kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu)); return 1; } @@ -112,7 +112,7 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, * The MMIO instruction is emulated and should not be re-executed * in the guest. */ - kvm_skip_instr(vcpu, (vcpu->arch.hsr >> 25) & 1); + kvm_skip_instr(vcpu, (kvm_vcpu_get_hsr(vcpu) >> 25) & 1); return 0; } @@ -130,7 +130,7 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, * space do its magic. */ - if (vcpu->arch.hsr & HSR_ISV) { + if (kvm_vcpu_get_hsr(vcpu) & HSR_ISV) { ret = decode_hsr(vcpu, fault_ipa, &mmio); if (ret) return ret; diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index 99e07c7dd745..d0f5f52c179f 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -526,7 +526,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, unsigned long mmu_seq; struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; - write_fault = kvm_is_write_fault(vcpu->arch.hsr); + write_fault = kvm_is_write_fault(kvm_vcpu_get_hsr(vcpu)); if (fault_status == FSC_PERM && !write_fault) { kvm_err("Unexpected L2 read permission error\n"); return -EFAULT; @@ -593,15 +593,15 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) gfn_t gfn; int ret, idx; - hsr_ec = vcpu->arch.hsr >> HSR_EC_SHIFT; + hsr_ec = kvm_vcpu_get_hsr(vcpu) >> HSR_EC_SHIFT; is_iabt = (hsr_ec == HSR_EC_IABT); - fault_ipa = ((phys_addr_t)vcpu->arch.hpfar & HPFAR_MASK) << 8; + fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); - trace_kvm_guest_fault(*vcpu_pc(vcpu), vcpu->arch.hsr, - vcpu->arch.hxfar, fault_ipa); + trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_hsr(vcpu), + kvm_vcpu_get_hfar(vcpu), fault_ipa); /* Check the stage-2 fault is trans. fault or write fault */ - fault_status = (vcpu->arch.hsr & HSR_FSC_TYPE); + fault_status = (kvm_vcpu_get_hsr(vcpu) & HSR_FSC_TYPE); if (fault_status != FSC_FAULT && fault_status != FSC_PERM) { kvm_err("Unsupported fault status: EC=%#lx DFCS=%#lx\n", hsr_ec, fault_status); @@ -614,7 +614,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) if (!kvm_is_visible_gfn(vcpu->kvm, gfn)) { if (is_iabt) { /* Prefetch Abort on I/O address */ - kvm_inject_pabt(vcpu, vcpu->arch.hxfar); + kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu)); ret = 1; goto out_unlock; } @@ -627,7 +627,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) } /* Adjust page offset */ - fault_ipa |= vcpu->arch.hxfar & ~PAGE_MASK; + fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ~PAGE_MASK; ret = io_mem_abort(vcpu, run, fault_ipa); goto out_unlock; } From 4a1df28ac0ce6d76d336210d8c4216087c17de3a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 18 Sep 2012 11:06:23 +0100 Subject: [PATCH 04/31] ARM: KVM: abstract HSR_ISV away Signed-off-by: Marc Zyngier Signed-off-by: Christoffer Dall --- arch/arm/include/asm/kvm_emulate.h | 5 +++++ arch/arm/kvm/mmio.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index 3c01988b3c88..ae9119e53bfc 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -90,4 +90,9 @@ static inline unsigned long kvm_vcpu_get_hyp_pc(struct kvm_vcpu *vcpu) return vcpu->arch.fault.hyp_pc; } +static inline bool kvm_vcpu_dabt_isvalid(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_get_hsr(vcpu) & HSR_ISV; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index ce63f39071ad..41f96e9e2e95 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -130,7 +130,7 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, * space do its magic. */ - if (kvm_vcpu_get_hsr(vcpu) & HSR_ISV) { + if (kvm_vcpu_dabt_isvalid(vcpu)) { ret = decode_hsr(vcpu, fault_ipa, &mmio); if (ret) return ret; From 023cc964063509ece283e0bed227114197ac14e0 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 18 Sep 2012 11:12:26 +0100 Subject: [PATCH 05/31] ARM: KVM: abstract HSR_WNR away Signed-off-by: Marc Zyngier Signed-off-by: Christoffer Dall --- arch/arm/include/asm/kvm_emulate.h | 5 +++++ arch/arm/kvm/mmio.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index ae9119e53bfc..a0d76df470b0 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -95,4 +95,9 @@ static inline bool kvm_vcpu_dabt_isvalid(struct kvm_vcpu *vcpu) return kvm_vcpu_get_hsr(vcpu) & HSR_ISV; } +static inline bool kvm_vcpu_dabt_iswrite(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_get_hsr(vcpu) & HSR_WNR; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index 41f96e9e2e95..7d58cb5b8bde 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -92,7 +92,7 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, return -EFAULT; } - is_write = kvm_vcpu_get_hsr(vcpu) & HSR_WNR; + is_write = kvm_vcpu_dabt_iswrite(vcpu); sign_extend = kvm_vcpu_get_hsr(vcpu) & HSR_SSE; rt = (kvm_vcpu_get_hsr(vcpu) & HSR_SRT_MASK) >> HSR_SRT_SHIFT; From 7c511b881f79a2a390e7c2e9a39b9a881255e614 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 18 Sep 2012 11:23:02 +0100 Subject: [PATCH 06/31] ARM: KVM: abstract HSR_SSE away Signed-off-by: Marc Zyngier Signed-off-by: Christoffer Dall --- arch/arm/include/asm/kvm_emulate.h | 5 +++++ arch/arm/kvm/mmio.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index a0d76df470b0..cd9cb874a7ff 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -100,4 +100,9 @@ static inline bool kvm_vcpu_dabt_iswrite(struct kvm_vcpu *vcpu) return kvm_vcpu_get_hsr(vcpu) & HSR_WNR; } +static inline bool kvm_vcpu_dabt_issext(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_get_hsr(vcpu) & HSR_SSE; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index 7d58cb5b8bde..058029c2d504 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -93,7 +93,7 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, } is_write = kvm_vcpu_dabt_iswrite(vcpu); - sign_extend = kvm_vcpu_get_hsr(vcpu) & HSR_SSE; + sign_extend = kvm_vcpu_dabt_issext(vcpu); rt = (kvm_vcpu_get_hsr(vcpu) & HSR_SRT_MASK) >> HSR_SRT_SHIFT; if (kvm_vcpu_reg_is_pc(vcpu, rt)) { From d0adf747c9caa8b01d0c1f987e306b7c6aaa5a04 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 18 Sep 2012 11:28:57 +0100 Subject: [PATCH 07/31] ARM: KVM: abstract HSR_SRT_{MASK,SHIFT} away Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_emulate.h | 5 +++++ arch/arm/kvm/mmio.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index cd9cb874a7ff..d548078af4d8 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -105,4 +105,9 @@ static inline bool kvm_vcpu_dabt_issext(struct kvm_vcpu *vcpu) return kvm_vcpu_get_hsr(vcpu) & HSR_SSE; } +static inline int kvm_vcpu_dabt_get_rd(struct kvm_vcpu *vcpu) +{ + return (kvm_vcpu_get_hsr(vcpu) & HSR_SRT_MASK) >> HSR_SRT_SHIFT; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index 058029c2d504..586063d0697a 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -94,7 +94,7 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, is_write = kvm_vcpu_dabt_iswrite(vcpu); sign_extend = kvm_vcpu_dabt_issext(vcpu); - rt = (kvm_vcpu_get_hsr(vcpu) & HSR_SRT_MASK) >> HSR_SRT_SHIFT; + rt = kvm_vcpu_dabt_get_rd(vcpu); if (kvm_vcpu_reg_is_pc(vcpu, rt)) { /* IO memory trying to read/write pc */ From 78abfcde49e0e454cabf8e56cd4c1591752e2706 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 18 Sep 2012 11:36:16 +0100 Subject: [PATCH 08/31] ARM: KVM: abstract (and fix) external abort detection away Bit 8 is cache maintenance, bit 9 is external abort. Signed-off-by: Marc Zyngier Signed-off-by: Christoffer Dall --- arch/arm/include/asm/kvm_arm.h | 3 +++ arch/arm/include/asm/kvm_emulate.h | 5 +++++ arch/arm/kvm/mmio.c | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/kvm_arm.h b/arch/arm/include/asm/kvm_arm.h index 7c3d813e15df..990764e89489 100644 --- a/arch/arm/include/asm/kvm_arm.h +++ b/arch/arm/include/asm/kvm_arm.h @@ -211,4 +211,7 @@ #define HSR_HVC_IMM_MASK ((1UL << 16) - 1) +#define HSR_DABT_CM (1U << 8) +#define HSR_DABT_EA (1U << 9) + #endif /* __ARM_KVM_ARM_H__ */ diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index d548078af4d8..2d1c5854243b 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -110,4 +110,9 @@ static inline int kvm_vcpu_dabt_get_rd(struct kvm_vcpu *vcpu) return (kvm_vcpu_get_hsr(vcpu) & HSR_SRT_MASK) >> HSR_SRT_SHIFT; } +static inline bool kvm_vcpu_dabt_isextabt(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_get_hsr(vcpu) & HSR_DABT_EA; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index 586063d0697a..e4682a3af313 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -65,7 +65,7 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, unsigned long rt, len; bool is_write, sign_extend; - if ((kvm_vcpu_get_hsr(vcpu) >> 8) & 1) { + if (kvm_vcpu_dabt_isextabt(vcpu)) { /* cache operation on I/O addr, tell guest unsupported */ kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); return 1; From b37670b0f37d8015d0d428e6a63bd07397430a2f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 18 Sep 2012 11:37:28 +0100 Subject: [PATCH 09/31] ARM: KVM: abstract S1TW abort detection away Signed-off-by: Marc Zyngier Signed-off-by: Christoffer Dall --- arch/arm/include/asm/kvm_arm.h | 1 + arch/arm/include/asm/kvm_emulate.h | 5 +++++ arch/arm/kvm/mmio.c | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/kvm_arm.h b/arch/arm/include/asm/kvm_arm.h index 990764e89489..124623e5ef14 100644 --- a/arch/arm/include/asm/kvm_arm.h +++ b/arch/arm/include/asm/kvm_arm.h @@ -211,6 +211,7 @@ #define HSR_HVC_IMM_MASK ((1UL << 16) - 1) +#define HSR_DABT_S1PTW (1U << 7) #define HSR_DABT_CM (1U << 8) #define HSR_DABT_EA (1U << 9) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index 2d1c5854243b..6b43653865fb 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -115,4 +115,9 @@ static inline bool kvm_vcpu_dabt_isextabt(struct kvm_vcpu *vcpu) return kvm_vcpu_get_hsr(vcpu) & HSR_DABT_EA; } +static inline bool kvm_vcpu_dabt_iss1tw(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_get_hsr(vcpu) & HSR_DABT_S1PTW; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index e4682a3af313..6495c1ca7bfa 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -71,7 +71,7 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, return 1; } - if ((kvm_vcpu_get_hsr(vcpu) >> 7) & 1) { + if (kvm_vcpu_dabt_iss1tw(vcpu)) { /* page table accesses IO mem: tell guest to fix its TTBR */ kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); return 1; From a7123377e7107a5f4f7ae198aa5b5000c9362871 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 18 Sep 2012 11:43:30 +0100 Subject: [PATCH 10/31] ARM: KVM: abstract SAS decoding away Signed-off-by: Marc Zyngier Signed-off-by: Christoffer Dall --- arch/arm/include/asm/kvm_emulate.h | 16 ++++++++++++++++ arch/arm/kvm/mmio.c | 17 +++-------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index 6b43653865fb..e4a4fc2f9156 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -120,4 +120,20 @@ static inline bool kvm_vcpu_dabt_iss1tw(struct kvm_vcpu *vcpu) return kvm_vcpu_get_hsr(vcpu) & HSR_DABT_S1PTW; } +/* Get Access Size from a data abort */ +static inline int kvm_vcpu_dabt_get_as(struct kvm_vcpu *vcpu) +{ + switch ((kvm_vcpu_get_hsr(vcpu) >> 22) & 0x3) { + case 0: + return 1; + case 1: + return 2; + case 2: + return 4; + default: + kvm_err("Hardware is weird: SAS 0b11 is reserved\n"); + return -EFAULT; + } +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index 6495c1ca7bfa..0b8f49985b70 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -77,20 +77,9 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, return 1; } - switch ((kvm_vcpu_get_hsr(vcpu) >> 22) & 0x3) { - case 0: - len = 1; - break; - case 1: - len = 2; - break; - case 2: - len = 4; - break; - default: - kvm_err("Hardware is weird: SAS 0b11 is reserved\n"); - return -EFAULT; - } + len = kvm_vcpu_dabt_get_as(vcpu); + if (unlikely(len < 0)) + return len; is_write = kvm_vcpu_dabt_iswrite(vcpu); sign_extend = kvm_vcpu_dabt_issext(vcpu); From 23b415d61a80c0c63f43dd2b3a08a1fa0b79b8ea Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 18 Sep 2012 12:07:06 +0100 Subject: [PATCH 11/31] ARM: KVM: abstract IL decoding away Signed-off-by: Marc Zyngier Signed-off-by: Christoffer Dall --- arch/arm/include/asm/kvm_emulate.h | 6 ++++++ arch/arm/kvm/arm.c | 3 +-- arch/arm/kvm/coproc.c | 2 +- arch/arm/kvm/mmio.c | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index e4a4fc2f9156..a9b853f35aed 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -136,4 +136,10 @@ static inline int kvm_vcpu_dabt_get_as(struct kvm_vcpu *vcpu) } } +/* This one is not specific to Data Abort */ +static inline bool kvm_vcpu_trap_il_is32bit(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_get_hsr(vcpu) & HSR_IL; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index 6626c7e62371..31616336255c 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -624,8 +624,7 @@ static int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, * that fail their condition code check" */ if (!kvm_condition_valid(vcpu)) { - bool is_wide = kvm_vcpu_get_hsr(vcpu) & HSR_IL; - kvm_skip_instr(vcpu, is_wide); + kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); return 1; } diff --git a/arch/arm/kvm/coproc.c b/arch/arm/kvm/coproc.c index b30591615da4..94eee8befc83 100644 --- a/arch/arm/kvm/coproc.c +++ b/arch/arm/kvm/coproc.c @@ -293,7 +293,7 @@ static int emulate_cp15(struct kvm_vcpu *vcpu, if (likely(r->access(vcpu, params, r))) { /* Skip instruction, since it was emulated */ - kvm_skip_instr(vcpu, (kvm_vcpu_get_hsr(vcpu) >> 25) & 1); + kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); return 1; } /* If access function fails, it should complain. */ diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index 0b8f49985b70..02ca76555bd4 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -101,7 +101,7 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, * The MMIO instruction is emulated and should not be re-executed * in the guest. */ - kvm_skip_instr(vcpu, (kvm_vcpu_get_hsr(vcpu) >> 25) & 1); + kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); return 0; } From 4926d445eb76bec8ebd71f5ed9e9c94fd738014d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 18 Sep 2012 14:09:58 +0100 Subject: [PATCH 12/31] ARM: KVM: abstract exception class decoding away Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_emulate.h | 5 +++++ arch/arm/kvm/arm.c | 4 ++-- arch/arm/kvm/mmu.c | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index a9b853f35aed..539f83ac167f 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -142,4 +142,9 @@ static inline bool kvm_vcpu_trap_il_is32bit(struct kvm_vcpu *vcpu) return kvm_vcpu_get_hsr(vcpu) & HSR_IL; } +static inline u8 kvm_vcpu_trap_get_class(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_get_hsr(vcpu) >> HSR_EC_SHIFT; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index 31616336255c..269900174102 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -559,7 +559,7 @@ static bool kvm_condition_valid(struct kvm_vcpu *vcpu) * catch undefined instructions, and then we won't get past * the arm_exit_handlers test anyway. */ - BUG_ON(((kvm_vcpu_get_hsr(vcpu) & HSR_EC) >> HSR_EC_SHIFT) == 0); + BUG_ON(!kvm_vcpu_trap_get_class(vcpu)); /* Top two bits non-zero? Unconditional. */ if (kvm_vcpu_get_hsr(vcpu) >> 30) @@ -609,7 +609,7 @@ static int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, case ARM_EXCEPTION_DATA_ABORT: case ARM_EXCEPTION_PREF_ABORT: case ARM_EXCEPTION_HVC: - hsr_ec = (kvm_vcpu_get_hsr(vcpu) & HSR_EC) >> HSR_EC_SHIFT; + hsr_ec = kvm_vcpu_trap_get_class(vcpu); if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) || !arm_exit_handlers[hsr_ec]) { diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index d0f5f52c179f..41fa75df107f 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -593,7 +593,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) gfn_t gfn; int ret, idx; - hsr_ec = kvm_vcpu_get_hsr(vcpu) >> HSR_EC_SHIFT; + hsr_ec = kvm_vcpu_trap_get_class(vcpu); is_iabt = (hsr_ec == HSR_EC_IABT); fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); From 1cc287dd081235e02cebd791f1e930ca6f422dcd Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 18 Sep 2012 14:14:35 +0100 Subject: [PATCH 13/31] ARM: KVM: abstract fault decoding away Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_emulate.h | 5 +++++ arch/arm/kvm/mmu.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index 539f83ac167f..021a59c46dda 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -147,4 +147,9 @@ static inline u8 kvm_vcpu_trap_get_class(struct kvm_vcpu *vcpu) return kvm_vcpu_get_hsr(vcpu) >> HSR_EC_SHIFT; } +static inline u8 kvm_vcpu_trap_get_fault(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_get_hsr(vcpu) & HSR_FSC_TYPE; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index 41fa75df107f..e00f28d2670c 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -601,7 +601,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) kvm_vcpu_get_hfar(vcpu), fault_ipa); /* Check the stage-2 fault is trans. fault or write fault */ - fault_status = (kvm_vcpu_get_hsr(vcpu) & HSR_FSC_TYPE); + fault_status = kvm_vcpu_trap_get_fault(vcpu); if (fault_status != FSC_FAULT && fault_status != FSC_PERM) { kvm_err("Unsupported fault status: EC=%#lx DFCS=%#lx\n", hsr_ec, fault_status); From 52d1dba933f601d8d9e6373377377b12d6bcfac0 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 15 Oct 2012 10:33:38 +0100 Subject: [PATCH 14/31] ARM: KVM: abstract HSR_EC_IABT away Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_emulate.h | 5 +++++ arch/arm/kvm/mmu.c | 8 +++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index 021a59c46dda..e59f8c0494f3 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -147,6 +147,11 @@ static inline u8 kvm_vcpu_trap_get_class(struct kvm_vcpu *vcpu) return kvm_vcpu_get_hsr(vcpu) >> HSR_EC_SHIFT; } +static inline bool kvm_vcpu_trap_is_iabt(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_trap_get_class(vcpu) == HSR_EC_IABT; +} + static inline u8 kvm_vcpu_trap_get_fault(struct kvm_vcpu *vcpu) { return kvm_vcpu_get_hsr(vcpu) & HSR_FSC_TYPE; diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index e00f28d2670c..8e9047a4b3b7 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -585,7 +585,6 @@ out_unlock: */ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) { - unsigned long hsr_ec; unsigned long fault_status; phys_addr_t fault_ipa; struct kvm_memory_slot *memslot; @@ -593,8 +592,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) gfn_t gfn; int ret, idx; - hsr_ec = kvm_vcpu_trap_get_class(vcpu); - is_iabt = (hsr_ec == HSR_EC_IABT); + is_iabt = kvm_vcpu_trap_is_iabt(vcpu); fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_hsr(vcpu), @@ -603,8 +601,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) /* Check the stage-2 fault is trans. fault or write fault */ fault_status = kvm_vcpu_trap_get_fault(vcpu); if (fault_status != FSC_FAULT && fault_status != FSC_PERM) { - kvm_err("Unsupported fault status: EC=%#lx DFCS=%#lx\n", - hsr_ec, fault_status); + kvm_err("Unsupported fault status: EC=%#x DFCS=%#lx\n", + kvm_vcpu_trap_get_class(vcpu), fault_status); return -EFAULT; } From c5997563298bc1b9da5212c15544962d4dbbe27d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 8 Dec 2012 18:13:18 +0000 Subject: [PATCH 15/31] ARM: KVM: move kvm_condition_valid to emulate.c This is really hardware emulation, and as such it better be with its little friends. Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_emulate.h | 1 + arch/arm/kvm/arm.c | 45 ------------------------------ arch/arm/kvm/emulate.c | 45 ++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index e59f8c0494f3..e14268c77af0 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -28,6 +28,7 @@ unsigned long *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num); unsigned long *vcpu_spsr(struct kvm_vcpu *vcpu); int kvm_handle_wfi(struct kvm_vcpu *vcpu, struct kvm_run *run); +bool kvm_condition_valid(struct kvm_vcpu *vcpu); void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr); void kvm_inject_undefined(struct kvm_vcpu *vcpu); void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr); diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index 269900174102..6b776183ff93 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -44,7 +44,6 @@ #include #include #include -#include #ifdef REQUIRES_VIRT __asm__(".arch_extension virt"); @@ -545,50 +544,6 @@ static exit_handle_fn arm_exit_handlers[] = { [HSR_EC_DABT_HYP] = handle_dabt_hyp, }; -/* - * A conditional instruction is allowed to trap, even though it - * wouldn't be executed. So let's re-implement the hardware, in - * software! - */ -static bool kvm_condition_valid(struct kvm_vcpu *vcpu) -{ - unsigned long cpsr, cond, insn; - - /* - * Exception Code 0 can only happen if we set HCR.TGE to 1, to - * catch undefined instructions, and then we won't get past - * the arm_exit_handlers test anyway. - */ - BUG_ON(!kvm_vcpu_trap_get_class(vcpu)); - - /* Top two bits non-zero? Unconditional. */ - if (kvm_vcpu_get_hsr(vcpu) >> 30) - return true; - - cpsr = *vcpu_cpsr(vcpu); - - /* Is condition field valid? */ - if ((kvm_vcpu_get_hsr(vcpu) & HSR_CV) >> HSR_CV_SHIFT) - cond = (kvm_vcpu_get_hsr(vcpu) & HSR_COND) >> HSR_COND_SHIFT; - else { - /* This can happen in Thumb mode: examine IT state. */ - unsigned long it; - - it = ((cpsr >> 8) & 0xFC) | ((cpsr >> 25) & 0x3); - - /* it == 0 => unconditional. */ - if (it == 0) - return true; - - /* The cond for this insn works out as the top 4 bits. */ - cond = (it >> 4); - } - - /* Shift makes it look like an ARM-mode instruction */ - insn = cond << 28; - return arm_check_condition(insn, cpsr) != ARM_OPCODE_CONDTEST_FAIL; -} - /* * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on * proper exit to QEMU. diff --git a/arch/arm/kvm/emulate.c b/arch/arm/kvm/emulate.c index d3094eb4ade6..04dbac6bdf4d 100644 --- a/arch/arm/kvm/emulate.c +++ b/arch/arm/kvm/emulate.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "trace.h" @@ -176,6 +177,50 @@ int kvm_handle_wfi(struct kvm_vcpu *vcpu, struct kvm_run *run) return 1; } +/* + * A conditional instruction is allowed to trap, even though it + * wouldn't be executed. So let's re-implement the hardware, in + * software! + */ +bool kvm_condition_valid(struct kvm_vcpu *vcpu) +{ + unsigned long cpsr, cond, insn; + + /* + * Exception Code 0 can only happen if we set HCR.TGE to 1, to + * catch undefined instructions, and then we won't get past + * the arm_exit_handlers test anyway. + */ + BUG_ON(!kvm_vcpu_trap_get_class(vcpu)); + + /* Top two bits non-zero? Unconditional. */ + if (kvm_vcpu_get_hsr(vcpu) >> 30) + return true; + + cpsr = *vcpu_cpsr(vcpu); + + /* Is condition field valid? */ + if ((kvm_vcpu_get_hsr(vcpu) & HSR_CV) >> HSR_CV_SHIFT) + cond = (kvm_vcpu_get_hsr(vcpu) & HSR_COND) >> HSR_COND_SHIFT; + else { + /* This can happen in Thumb mode: examine IT state. */ + unsigned long it; + + it = ((cpsr >> 8) & 0xFC) | ((cpsr >> 25) & 0x3); + + /* it == 0 => unconditional. */ + if (it == 0) + return true; + + /* The cond for this insn works out as the top 4 bits. */ + cond = (it >> 4); + } + + /* Shift makes it look like an ARM-mode instruction */ + insn = cond << 28; + return arm_check_condition(insn, cpsr) != ARM_OPCODE_CONDTEST_FAIL; +} + /** * adjust_itstate - adjust ITSTATE when emulating instructions in IT-block * @vcpu: The VCPU pointer From 3414bbfff98b2524c0b2cdc4d0a78153ebf4f823 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 5 Oct 2012 11:11:11 +0100 Subject: [PATCH 16/31] ARM: KVM: move exit handler selection to a separate file The exit handler selection code cannot be shared with arm64 (two different modes, more exception classes...). Move it to a separate file (handle_exit.c). Signed-off-by: Marc Zyngier Signed-off-by: Christoffer Dall --- arch/arm/include/asm/kvm_host.h | 3 + arch/arm/kvm/Makefile | 2 +- arch/arm/kvm/arm.c | 113 -------------------------- arch/arm/kvm/handle_exit.c | 140 ++++++++++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 114 deletions(-) create mode 100644 arch/arm/kvm/handle_exit.c diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h index eb836e6d3c59..24f457aeba4d 100644 --- a/arch/arm/include/asm/kvm_host.h +++ b/arch/arm/include/asm/kvm_host.h @@ -183,4 +183,7 @@ struct kvm_one_reg; int kvm_arm_coproc_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *); int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *); +int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, + int exception_index); + #endif /* __ARM_KVM_HOST_H__ */ diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile index fc96ce6f2357..8dc5e76cb789 100644 --- a/arch/arm/kvm/Makefile +++ b/arch/arm/kvm/Makefile @@ -17,7 +17,7 @@ AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt) kvm-arm-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o) obj-y += kvm-arm.o init.o interrupts.o -obj-y += arm.o guest.o mmu.o emulate.o reset.o +obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o obj-y += coproc.o coproc_a15.o mmio.o psci.o obj-$(CONFIG_KVM_ARM_VGIC) += vgic.o obj-$(CONFIG_KVM_ARM_TIMER) += arch_timer.o diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index 6b776183ff93..de783ee8c341 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -30,7 +30,6 @@ #define CREATE_TRACE_POINTS #include "trace.h" -#include #include #include #include @@ -480,118 +479,6 @@ static void update_vttbr(struct kvm *kvm) spin_unlock(&kvm_vmid_lock); } -static int handle_svc_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - /* SVC called from Hyp mode should never get here */ - kvm_debug("SVC called from Hyp mode shouldn't go here\n"); - BUG(); - return -EINVAL; /* Squash warning */ -} - -static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - trace_kvm_hvc(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0), - kvm_vcpu_get_hsr(vcpu) & HSR_HVC_IMM_MASK); - - if (kvm_psci_call(vcpu)) - return 1; - - kvm_inject_undefined(vcpu); - return 1; -} - -static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - if (kvm_psci_call(vcpu)) - return 1; - - kvm_inject_undefined(vcpu); - return 1; -} - -static int handle_pabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - /* The hypervisor should never cause aborts */ - kvm_err("Prefetch Abort taken from Hyp mode at %#08lx (HSR: %#08x)\n", - kvm_vcpu_get_hfar(vcpu), kvm_vcpu_get_hsr(vcpu)); - return -EFAULT; -} - -static int handle_dabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - /* This is either an error in the ws. code or an external abort */ - kvm_err("Data Abort taken from Hyp mode at %#08lx (HSR: %#08x)\n", - kvm_vcpu_get_hfar(vcpu), kvm_vcpu_get_hsr(vcpu)); - return -EFAULT; -} - -typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *); -static exit_handle_fn arm_exit_handlers[] = { - [HSR_EC_WFI] = kvm_handle_wfi, - [HSR_EC_CP15_32] = kvm_handle_cp15_32, - [HSR_EC_CP15_64] = kvm_handle_cp15_64, - [HSR_EC_CP14_MR] = kvm_handle_cp14_access, - [HSR_EC_CP14_LS] = kvm_handle_cp14_load_store, - [HSR_EC_CP14_64] = kvm_handle_cp14_access, - [HSR_EC_CP_0_13] = kvm_handle_cp_0_13_access, - [HSR_EC_CP10_ID] = kvm_handle_cp10_id, - [HSR_EC_SVC_HYP] = handle_svc_hyp, - [HSR_EC_HVC] = handle_hvc, - [HSR_EC_SMC] = handle_smc, - [HSR_EC_IABT] = kvm_handle_guest_abort, - [HSR_EC_IABT_HYP] = handle_pabt_hyp, - [HSR_EC_DABT] = kvm_handle_guest_abort, - [HSR_EC_DABT_HYP] = handle_dabt_hyp, -}; - -/* - * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on - * proper exit to QEMU. - */ -static int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, - int exception_index) -{ - unsigned long hsr_ec; - - switch (exception_index) { - case ARM_EXCEPTION_IRQ: - return 1; - case ARM_EXCEPTION_UNDEFINED: - kvm_err("Undefined exception in Hyp mode at: %#08lx\n", - kvm_vcpu_get_hyp_pc(vcpu)); - BUG(); - panic("KVM: Hypervisor undefined exception!\n"); - case ARM_EXCEPTION_DATA_ABORT: - case ARM_EXCEPTION_PREF_ABORT: - case ARM_EXCEPTION_HVC: - hsr_ec = kvm_vcpu_trap_get_class(vcpu); - - if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) - || !arm_exit_handlers[hsr_ec]) { - kvm_err("Unkown exception class: %#08lx, " - "hsr: %#08x\n", hsr_ec, - (unsigned int)kvm_vcpu_get_hsr(vcpu)); - BUG(); - } - - /* - * See ARM ARM B1.14.1: "Hyp traps on instructions - * that fail their condition code check" - */ - if (!kvm_condition_valid(vcpu)) { - kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); - return 1; - } - - return arm_exit_handlers[hsr_ec](vcpu, run); - default: - kvm_pr_unimpl("Unsupported exception type: %d", - exception_index); - run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - return 0; - } -} - static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) { if (likely(vcpu->arch.has_run_once)) diff --git a/arch/arm/kvm/handle_exit.c b/arch/arm/kvm/handle_exit.c new file mode 100644 index 000000000000..f1cc3a8a9c7e --- /dev/null +++ b/arch/arm/kvm/handle_exit.c @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2012 - Virtual Open Systems and Columbia University + * Author: Christoffer Dall + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include + +typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *); + +static int handle_svc_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + /* SVC called from Hyp mode should never get here */ + kvm_debug("SVC called from Hyp mode shouldn't go here\n"); + BUG(); + return -EINVAL; /* Squash warning */ +} + +static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + if (kvm_psci_call(vcpu)) + return 1; + + kvm_inject_undefined(vcpu); + return 1; +} + +static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + if (kvm_psci_call(vcpu)) + return 1; + + kvm_inject_undefined(vcpu); + return 1; +} + +static int handle_pabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + /* The hypervisor should never cause aborts */ + kvm_err("Prefetch Abort taken from Hyp mode at %#08lx (HSR: %#08x)\n", + kvm_vcpu_get_hfar(vcpu), kvm_vcpu_get_hsr(vcpu)); + return -EFAULT; +} + +static int handle_dabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + /* This is either an error in the ws. code or an external abort */ + kvm_err("Data Abort taken from Hyp mode at %#08lx (HSR: %#08x)\n", + kvm_vcpu_get_hfar(vcpu), kvm_vcpu_get_hsr(vcpu)); + return -EFAULT; +} + +static exit_handle_fn arm_exit_handlers[] = { + [HSR_EC_WFI] = kvm_handle_wfi, + [HSR_EC_CP15_32] = kvm_handle_cp15_32, + [HSR_EC_CP15_64] = kvm_handle_cp15_64, + [HSR_EC_CP14_MR] = kvm_handle_cp14_access, + [HSR_EC_CP14_LS] = kvm_handle_cp14_load_store, + [HSR_EC_CP14_64] = kvm_handle_cp14_access, + [HSR_EC_CP_0_13] = kvm_handle_cp_0_13_access, + [HSR_EC_CP10_ID] = kvm_handle_cp10_id, + [HSR_EC_SVC_HYP] = handle_svc_hyp, + [HSR_EC_HVC] = handle_hvc, + [HSR_EC_SMC] = handle_smc, + [HSR_EC_IABT] = kvm_handle_guest_abort, + [HSR_EC_IABT_HYP] = handle_pabt_hyp, + [HSR_EC_DABT] = kvm_handle_guest_abort, + [HSR_EC_DABT_HYP] = handle_dabt_hyp, +}; + +static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) +{ + u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu); + + if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) || + !arm_exit_handlers[hsr_ec]) { + kvm_err("Unkown exception class: hsr: %#08x\n", + (unsigned int)kvm_vcpu_get_hsr(vcpu)); + BUG(); + } + + return arm_exit_handlers[hsr_ec]; +} + +/* + * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on + * proper exit to userspace. + */ +int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, + int exception_index) +{ + exit_handle_fn exit_handler; + + switch (exception_index) { + case ARM_EXCEPTION_IRQ: + return 1; + case ARM_EXCEPTION_UNDEFINED: + kvm_err("Undefined exception in Hyp mode at: %#08lx\n", + kvm_vcpu_get_hyp_pc(vcpu)); + BUG(); + panic("KVM: Hypervisor undefined exception!\n"); + case ARM_EXCEPTION_DATA_ABORT: + case ARM_EXCEPTION_PREF_ABORT: + case ARM_EXCEPTION_HVC: + /* + * See ARM ARM B1.14.1: "Hyp traps on instructions + * that fail their condition code check" + */ + if (!kvm_condition_valid(vcpu)) { + kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); + return 1; + } + + exit_handler = kvm_get_exit_handler(vcpu); + + return exit_handler(vcpu, run); + default: + kvm_pr_unimpl("Unsupported exception type: %d", + exception_index); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return 0; + } +} From c088f8f0088decf0ce172a6bb4a6f5742cc54c15 Mon Sep 17 00:00:00 2001 From: Christoffer Dall Date: Thu, 21 Feb 2013 11:26:10 -0800 Subject: [PATCH 17/31] KVM: ARM: Reintroduce trace_kvm_hvc This one got lost in the move to handle_exit, so let's reintroduce it using an accessor to the immediate value field like the other ones. Signed-off-by: Christoffer Dall --- arch/arm/include/asm/kvm_emulate.h | 5 +++++ arch/arm/kvm/handle_exit.c | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index e14268c77af0..9cb2fe17cf3a 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -158,4 +158,9 @@ static inline u8 kvm_vcpu_trap_get_fault(struct kvm_vcpu *vcpu) return kvm_vcpu_get_hsr(vcpu) & HSR_FSC_TYPE; } +static inline u32 kvm_vcpu_hvc_get_imm(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_get_hsr(vcpu) & HSR_HVC_IMM_MASK; +} + #endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/kvm/handle_exit.c b/arch/arm/kvm/handle_exit.c index f1cc3a8a9c7e..301301808c4e 100644 --- a/arch/arm/kvm/handle_exit.c +++ b/arch/arm/kvm/handle_exit.c @@ -23,6 +23,8 @@ #include #include +#include "trace.h" + typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *); static int handle_svc_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) @@ -35,6 +37,9 @@ static int handle_svc_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) { + trace_kvm_hvc(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0), + kvm_vcpu_hvc_get_imm(vcpu)); + if (kvm_psci_call(vcpu)) return 1; From c62ee2b22798d7a5a8eb5f799b5183ef993ca7e4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 15 Oct 2012 11:27:37 +0100 Subject: [PATCH 18/31] ARM: KVM: abstract most MMU operations Move low level MMU-related operations to kvm_mmu.h. This makes the MMU code reusable by the arm64 port. Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_mmu.h | 58 ++++++++++++++++++++++++++++++++++ arch/arm/kvm/mmu.c | 58 +++++++--------------------------- 2 files changed, 70 insertions(+), 46 deletions(-) diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h index 421a20b34874..ac784937cc0f 100644 --- a/arch/arm/include/asm/kvm_mmu.h +++ b/arch/arm/include/asm/kvm_mmu.h @@ -19,6 +19,9 @@ #ifndef __ARM_KVM_MMU_H__ #define __ARM_KVM_MMU_H__ +#include +#include + int create_hyp_mappings(void *from, void *to); int create_hyp_io_mappings(void *from, void *to, phys_addr_t); void free_hyp_pmds(void); @@ -36,6 +39,16 @@ phys_addr_t kvm_mmu_get_httbr(void); int kvm_mmu_init(void); void kvm_clear_hyp_idmap(void); +static inline void kvm_set_pte(pte_t *pte, pte_t new_pte) +{ + pte_val(*pte) = new_pte; + /* + * flush_pmd_entry just takes a void pointer and cleans the necessary + * cache entries, so we can reuse the function for ptes. + */ + flush_pmd_entry(pte); +} + static inline bool kvm_is_write_fault(unsigned long hsr) { unsigned long hsr_ec = hsr >> HSR_EC_SHIFT; @@ -47,4 +60,49 @@ static inline bool kvm_is_write_fault(unsigned long hsr) return true; } +static inline void kvm_clean_pgd(pgd_t *pgd) +{ + clean_dcache_area(pgd, PTRS_PER_S2_PGD * sizeof(pgd_t)); +} + +static inline void kvm_clean_pmd_entry(pmd_t *pmd) +{ + clean_pmd_entry(pmd); +} + +static inline void kvm_clean_pte(pte_t *pte) +{ + clean_pte_table(pte); +} + +static inline void kvm_set_s2pte_writable(pte_t *pte) +{ + pte_val(*pte) |= L_PTE_S2_RDWR; +} + +struct kvm; + +static inline void coherent_icache_guest_page(struct kvm *kvm, gfn_t gfn) +{ + /* + * If we are going to insert an instruction page and the icache is + * either VIPT or PIPT, there is a potential problem where the host + * (or another VM) may have used the same page as this guest, and we + * read incorrect data from the icache. If we're using a PIPT cache, + * we can invalidate just that page, but if we are using a VIPT cache + * we need to invalidate the entire icache - damn shame - as written + * in the ARM ARM (DDI 0406C.b - Page B3-1393). + * + * VIVT caches are tagged using both the ASID and the VMID and doesn't + * need any kind of flushing (DDI 0406C.b - Page B3-1392). + */ + if (icache_is_pipt()) { + unsigned long hva = gfn_to_hva(kvm, gfn); + __cpuc_coherent_user_range(hva, hva + PAGE_SIZE); + } else if (!icache_is_vivt_asid_tagged()) { + /* any kind of VIPT cache */ + __flush_icache_all(); + } +} + #endif /* __ARM_KVM_MMU_H__ */ diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index 8e9047a4b3b7..6b4ea185956e 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -28,8 +28,6 @@ #include #include #include -#include -#include #include "trace.h" @@ -42,16 +40,6 @@ static void kvm_tlb_flush_vmid(struct kvm *kvm) kvm_call_hyp(__kvm_tlb_flush_vmid, kvm); } -static void kvm_set_pte(pte_t *pte, pte_t new_pte) -{ - pte_val(*pte) = new_pte; - /* - * flush_pmd_entry just takes a void pointer and cleans the necessary - * cache entries, so we can reuse the function for ptes. - */ - flush_pmd_entry(pte); -} - static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, int min, int max) { @@ -290,7 +278,7 @@ int kvm_alloc_stage2_pgd(struct kvm *kvm) VM_BUG_ON((unsigned long)pgd & (S2_PGD_SIZE - 1)); memset(pgd, 0, PTRS_PER_S2_PGD * sizeof(pgd_t)); - clean_dcache_area(pgd, PTRS_PER_S2_PGD * sizeof(pgd_t)); + kvm_clean_pgd(pgd); kvm->arch.pgd = pgd; return 0; @@ -422,22 +410,22 @@ static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, return 0; /* ignore calls from kvm_set_spte_hva */ pmd = mmu_memory_cache_alloc(cache); pud_populate(NULL, pud, pmd); - pmd += pmd_index(addr); get_page(virt_to_page(pud)); - } else - pmd = pmd_offset(pud, addr); + } + + pmd = pmd_offset(pud, addr); /* Create 2nd stage page table mapping - Level 2 */ if (pmd_none(*pmd)) { if (!cache) return 0; /* ignore calls from kvm_set_spte_hva */ pte = mmu_memory_cache_alloc(cache); - clean_pte_table(pte); + kvm_clean_pte(pte); pmd_populate_kernel(NULL, pmd, pte); - pte += pte_index(addr); get_page(virt_to_page(pmd)); - } else - pte = pte_offset_kernel(pmd, addr); + } + + pte = pte_offset_kernel(pmd, addr); if (iomap && pte_present(*pte)) return -EFAULT; @@ -473,7 +461,8 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, pfn = __phys_to_pfn(pa); for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) { - pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE | L_PTE_S2_RDWR); + pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE); + kvm_set_s2pte_writable(&pte); ret = mmu_topup_memory_cache(&cache, 2, 2); if (ret) @@ -492,29 +481,6 @@ out: return ret; } -static void coherent_icache_guest_page(struct kvm *kvm, gfn_t gfn) -{ - /* - * If we are going to insert an instruction page and the icache is - * either VIPT or PIPT, there is a potential problem where the host - * (or another VM) may have used the same page as this guest, and we - * read incorrect data from the icache. If we're using a PIPT cache, - * we can invalidate just that page, but if we are using a VIPT cache - * we need to invalidate the entire icache - damn shame - as written - * in the ARM ARM (DDI 0406C.b - Page B3-1393). - * - * VIVT caches are tagged using both the ASID and the VMID and doesn't - * need any kind of flushing (DDI 0406C.b - Page B3-1392). - */ - if (icache_is_pipt()) { - unsigned long hva = gfn_to_hva(kvm, gfn); - __cpuc_coherent_user_range(hva, hva + PAGE_SIZE); - } else if (!icache_is_vivt_asid_tagged()) { - /* any kind of VIPT cache */ - __flush_icache_all(); - } -} - static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, gfn_t gfn, struct kvm_memory_slot *memslot, unsigned long fault_status) @@ -560,7 +526,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) goto out_unlock; if (writable) { - pte_val(new_pte) |= L_PTE_S2_RDWR; + kvm_set_s2pte_writable(&new_pte); kvm_set_pfn_dirty(pfn); } stage2_set_pte(vcpu->kvm, memcache, fault_ipa, &new_pte, false); @@ -774,7 +740,7 @@ void kvm_clear_hyp_idmap(void) pmd = pmd_offset(pud, addr); pud_clear(pud); - clean_pmd_entry(pmd); + kvm_clean_pmd_entry(pmd); pmd_free(NULL, (pmd_t *)((unsigned long)pmd & PAGE_MASK)); } while (pgd++, addr = next, addr < end); } From 629dc446a49a1c3b4f31a3fdc2c814a7db6c6199 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 15 Oct 2012 14:29:47 +0100 Subject: [PATCH 19/31] ARM: KVM: remove superfluous include from kvm_vgic.h Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_vgic.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm/include/asm/kvm_vgic.h b/arch/arm/include/asm/kvm_vgic.h index ab97207d9cd3..343744e4809c 100644 --- a/arch/arm/include/asm/kvm_vgic.h +++ b/arch/arm/include/asm/kvm_vgic.h @@ -21,7 +21,6 @@ #include #include -#include #include #include #include From e7858c58d52237a4519e2fdb1ce8f2d9805ce0ce Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 5 Oct 2012 15:10:44 +0100 Subject: [PATCH 20/31] ARM: KVM: move hyp init to kvm_host.h Make the split of the pgd_ptr an implementation specific thing by moving the init call to an inline function. Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_host.h | 19 +++++++++++++++++++ arch/arm/kvm/arm.c | 12 +----------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h index 24f457aeba4d..f00a557c107a 100644 --- a/arch/arm/include/asm/kvm_host.h +++ b/arch/arm/include/asm/kvm_host.h @@ -186,4 +186,23 @@ int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *); int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, int exception_index); +static inline void __cpu_init_hyp_mode(unsigned long long pgd_ptr, + unsigned long hyp_stack_ptr, + unsigned long vector_ptr) +{ + unsigned long pgd_low, pgd_high; + + pgd_low = (pgd_ptr & ((1ULL << 32) - 1)); + pgd_high = (pgd_ptr >> 32ULL); + + /* + * Call initialization code, and switch to the full blown + * HYP code. The init code doesn't need to preserve these registers as + * r1-r3 and r12 are already callee save according to the AAPCS. + * Note that we slightly misuse the prototype by casing the pgd_low to + * a void *. + */ + kvm_call_hyp((void *)pgd_low, pgd_high, hyp_stack_ptr, vector_ptr); +} + #endif /* __ARM_KVM_HOST_H__ */ diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index de783ee8c341..3c7c50a6a286 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -813,7 +813,6 @@ long kvm_arch_vm_ioctl(struct file *filp, static void cpu_init_hyp_mode(void *vector) { unsigned long long pgd_ptr; - unsigned long pgd_low, pgd_high; unsigned long hyp_stack_ptr; unsigned long stack_page; unsigned long vector_ptr; @@ -822,20 +821,11 @@ static void cpu_init_hyp_mode(void *vector) __hyp_set_vectors((unsigned long)vector); pgd_ptr = (unsigned long long)kvm_mmu_get_httbr(); - pgd_low = (pgd_ptr & ((1ULL << 32) - 1)); - pgd_high = (pgd_ptr >> 32ULL); stack_page = __get_cpu_var(kvm_arm_hyp_stack_page); hyp_stack_ptr = stack_page + PAGE_SIZE; vector_ptr = (unsigned long)__kvm_hyp_vector; - /* - * Call initialization code, and switch to the full blown - * HYP code. The init code doesn't need to preserve these registers as - * r1-r3 and r12 are already callee save according to the AAPCS. - * Note that we slightly misuse the prototype by casing the pgd_low to - * a void *. - */ - kvm_call_hyp((void *)pgd_low, pgd_high, hyp_stack_ptr, vector_ptr); + __cpu_init_hyp_mode(pgd_ptr, hyp_stack_ptr, vector_ptr); } /** From 9c7a6432fb081563f084b25bbd2774b1547c4fad Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 27 Oct 2012 18:23:25 +0100 Subject: [PATCH 21/31] ARM: KVM: use kvm_kernel_vfp_t as an abstract type for VFP containers In order to keep the VFP allocation code common, use an abstract type for the VFP containers. Maps onto struct vfp_hard_struct on ARM. Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_host.h | 6 ++++-- arch/arm/kvm/arm.c | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h index f00a557c107a..0c4e643d939e 100644 --- a/arch/arm/include/asm/kvm_host.h +++ b/arch/arm/include/asm/kvm_host.h @@ -87,6 +87,8 @@ struct kvm_vcpu_fault_info { u32 hyp_pc; /* PC when exception was taken from Hyp mode */ }; +typedef struct vfp_hard_struct kvm_kernel_vfp_t; + struct kvm_vcpu_arch { struct kvm_regs regs; @@ -103,8 +105,8 @@ struct kvm_vcpu_arch { struct kvm_vcpu_fault_info fault; /* Floating point registers (VFP and Advanced SIMD/NEON) */ - struct vfp_hard_struct vfp_guest; - struct vfp_hard_struct *vfp_host; + kvm_kernel_vfp_t vfp_guest; + kvm_kernel_vfp_t *vfp_host; /* VGIC state */ struct vgic_cpu vgic_cpu; diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index 3c7c50a6a286..f0530499dec6 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -49,7 +49,7 @@ __asm__(".arch_extension virt"); #endif static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page); -static struct vfp_hard_struct __percpu *kvm_host_vfp_state; +static kvm_kernel_vfp_t __percpu *kvm_host_vfp_state; static unsigned long hyp_default_vectors; /* Per-CPU variable containing the currently running vcpu. */ @@ -908,7 +908,7 @@ static int init_hyp_mode(void) /* * Map the host VFP structures */ - kvm_host_vfp_state = alloc_percpu(struct vfp_hard_struct); + kvm_host_vfp_state = alloc_percpu(kvm_kernel_vfp_t); if (!kvm_host_vfp_state) { err = -ENOMEM; kvm_err("Cannot allocate host VFP state\n"); @@ -916,7 +916,7 @@ static int init_hyp_mode(void) } for_each_possible_cpu(cpu) { - struct vfp_hard_struct *vfp; + kvm_kernel_vfp_t *vfp; vfp = per_cpu_ptr(kvm_host_vfp_state, cpu); err = create_hyp_mappings(vfp, vfp + 1); From 06e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 28 Oct 2012 01:09:14 +0100 Subject: [PATCH 22/31] ARM: KVM: allow HYP mappings to be at an offset from kernel mappings arm64 cannot represent the kernel VAs in HYP mode, because of the lack of TTBR1 at EL2. A way to cope with this situation is to have HYP VAs to be an offset from the kernel VAs. Introduce macros to convert a kernel VA to a HYP VA, make the HYP mapping functions use these conversion macros. Also change the documentation to reflect the existence of the offset. On ARM, where we can have an identity mapping between kernel and HYP, the macros are without any effect. Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_mmu.h | 8 +++++++ arch/arm/kvm/mmu.c | 43 +++++++++++++++++++++------------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h index ac784937cc0f..3c71a1d4b7a3 100644 --- a/arch/arm/include/asm/kvm_mmu.h +++ b/arch/arm/include/asm/kvm_mmu.h @@ -22,6 +22,14 @@ #include #include +/* + * We directly use the kernel VA for the HYP, as we can directly share + * the mapping (HTTBR "covers" TTBR1). + */ +#define HYP_PAGE_OFFSET_MASK (~0UL) +#define HYP_PAGE_OFFSET PAGE_OFFSET +#define KERN_TO_HYP(kva) (kva) + int create_hyp_mappings(void *from, void *to); int create_hyp_io_mappings(void *from, void *to, phys_addr_t); void free_hyp_pmds(void); diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index 6b4ea185956e..ead6b16eeb09 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -101,14 +101,15 @@ void free_hyp_pmds(void) mutex_lock(&kvm_hyp_pgd_mutex); for (addr = PAGE_OFFSET; addr != 0; addr += PGDIR_SIZE) { - pgd = hyp_pgd + pgd_index(addr); - pud = pud_offset(pgd, addr); + unsigned long hyp_addr = KERN_TO_HYP(addr); + pgd = hyp_pgd + pgd_index(hyp_addr); + pud = pud_offset(pgd, hyp_addr); if (pud_none(*pud)) continue; BUG_ON(pud_bad(*pud)); - pmd = pmd_offset(pud, addr); + pmd = pmd_offset(pud, hyp_addr); free_ptes(pmd, addr); pmd_free(NULL, pmd); pud_clear(pud); @@ -124,7 +125,9 @@ static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start, struct page *page; for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) { - pte = pte_offset_kernel(pmd, addr); + unsigned long hyp_addr = KERN_TO_HYP(addr); + + pte = pte_offset_kernel(pmd, hyp_addr); BUG_ON(!virt_addr_valid(addr)); page = virt_to_page(addr); kvm_set_pte(pte, mk_pte(page, PAGE_HYP)); @@ -139,7 +142,9 @@ static void create_hyp_io_pte_mappings(pmd_t *pmd, unsigned long start, unsigned long addr; for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) { - pte = pte_offset_kernel(pmd, addr); + unsigned long hyp_addr = KERN_TO_HYP(addr); + + pte = pte_offset_kernel(pmd, hyp_addr); BUG_ON(pfn_valid(*pfn_base)); kvm_set_pte(pte, pfn_pte(*pfn_base, PAGE_HYP_DEVICE)); (*pfn_base)++; @@ -154,12 +159,13 @@ static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start, unsigned long addr, next; for (addr = start; addr < end; addr = next) { - pmd = pmd_offset(pud, addr); + unsigned long hyp_addr = KERN_TO_HYP(addr); + pmd = pmd_offset(pud, hyp_addr); BUG_ON(pmd_sect(*pmd)); if (pmd_none(*pmd)) { - pte = pte_alloc_one_kernel(NULL, addr); + pte = pte_alloc_one_kernel(NULL, hyp_addr); if (!pte) { kvm_err("Cannot allocate Hyp pte\n"); return -ENOMEM; @@ -200,11 +206,12 @@ static int __create_hyp_mappings(void *from, void *to, unsigned long *pfn_base) mutex_lock(&kvm_hyp_pgd_mutex); for (addr = start; addr < end; addr = next) { - pgd = hyp_pgd + pgd_index(addr); - pud = pud_offset(pgd, addr); + unsigned long hyp_addr = KERN_TO_HYP(addr); + pgd = hyp_pgd + pgd_index(hyp_addr); + pud = pud_offset(pgd, hyp_addr); if (pud_none_or_clear_bad(pud)) { - pmd = pmd_alloc_one(NULL, addr); + pmd = pmd_alloc_one(NULL, hyp_addr); if (!pmd) { kvm_err("Cannot allocate Hyp pmd\n"); err = -ENOMEM; @@ -224,12 +231,13 @@ out: } /** - * create_hyp_mappings - map a kernel virtual address range in Hyp mode + * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode * @from: The virtual kernel start address of the range * @to: The virtual kernel end address of the range (exclusive) * - * The same virtual address as the kernel virtual address is also used in - * Hyp-mode mapping to the same underlying physical pages. + * The same virtual address as the kernel virtual address is also used + * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying + * physical pages. * * Note: Wrapping around zero in the "to" address is not supported. */ @@ -239,10 +247,13 @@ int create_hyp_mappings(void *from, void *to) } /** - * create_hyp_io_mappings - map a physical IO range in Hyp mode - * @from: The virtual HYP start address of the range - * @to: The virtual HYP end address of the range (exclusive) + * create_hyp_io_mappings - duplicate a kernel IO mapping into Hyp mode + * @from: The kernel start VA of the range + * @to: The kernel end VA of the range (exclusive) * @addr: The physical start address which gets mapped + * + * The resulting HYP VA is the same as the kernel VA, modulo + * HYP_PAGE_OFFSET. */ int create_hyp_io_mappings(void *from, void *to, phys_addr_t addr) { From b4034bde5f168f2383a54b4573e1e440dbc169cf Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 28 Oct 2012 11:52:57 +0000 Subject: [PATCH 23/31] ARM: KVM: fix address validation for HYP mappings __create_hyp_mappings() performs some kind of address validation before creating the mapping, by verifying that the start address is above PAGE_OFFSET. This check is not completely correct for kernel memory (the upper boundary has to be checked as well so we do not end up with highmem pages), and wrong for IO mappings (the mapping must exist in the vmalloc region). Fix this by using the proper predicates (virt_addr_valid and is_vmalloc_addr), which also work correctly on ARM64 (where the vmalloc region is below PAGE_OFFSET). Also change the BUG_ON() into a less agressive error return. Signed-off-by: Marc Zyngier --- arch/arm/kvm/mmu.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index ead6b16eeb09..ec14269a791c 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -200,8 +200,13 @@ static int __create_hyp_mappings(void *from, void *to, unsigned long *pfn_base) unsigned long addr, next; int err = 0; - BUG_ON(start > end); - if (start < PAGE_OFFSET) + if (start >= end) + return -EINVAL; + /* Check for a valid kernel memory mapping */ + if (!pfn_base && (!virt_addr_valid(from) || !virt_addr_valid(to - 1))) + return -EINVAL; + /* Check for a valid kernel IO mapping */ + if (pfn_base && (!is_vmalloc_addr(from) || !is_vmalloc_addr(to - 1))) return -EINVAL; mutex_lock(&kvm_hyp_pgd_mutex); From 728d577d357c9caf83f75d0a38f318f343999cc2 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 15 Oct 2012 12:09:45 +0100 Subject: [PATCH 24/31] ARM: KVM: move kvm_target_cpu to guest.c guest.c already contains some target-specific checks. Let's move kvm_target_cpu() over there so arm.c is mostly target agnostic. Signed-off-by: Marc Zyngier --- arch/arm/kvm/arm.c | 17 ----------------- arch/arm/kvm/guest.c | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index f0530499dec6..c10a45fa73f7 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -301,22 +300,6 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) return 0; } -int __attribute_const__ kvm_target_cpu(void) -{ - unsigned long implementor = read_cpuid_implementor(); - unsigned long part_number = read_cpuid_part_number(); - - if (implementor != ARM_CPU_IMP_ARM) - return -EINVAL; - - switch (part_number) { - case ARM_CPU_PART_CORTEX_A15: - return KVM_ARM_TARGET_CORTEX_A15; - default: - return -EINVAL; - } -} - int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) { int ret; diff --git a/arch/arm/kvm/guest.c b/arch/arm/kvm/guest.c index 2339d9609d36..152d03612181 100644 --- a/arch/arm/kvm/guest.c +++ b/arch/arm/kvm/guest.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -180,6 +181,22 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, return -EINVAL; } +int __attribute_const__ kvm_target_cpu(void) +{ + unsigned long implementor = read_cpuid_implementor(); + unsigned long part_number = read_cpuid_part_number(); + + if (implementor != ARM_CPU_IMP_ARM) + return -EINVAL; + + switch (part_number) { + case ARM_CPU_PART_CORTEX_A15: + return KVM_ARM_TARGET_CORTEX_A15; + default: + return -EINVAL; + } +} + int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, const struct kvm_vcpu_init *init) { From cfe3950c2a19c1e1ad85b9dd2622617e309d2845 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 12 Dec 2012 14:42:09 +0000 Subject: [PATCH 25/31] ARM: KVM: fix fault_ipa computing The ARM ARM says that HPFAR reports bits [39:12] of the faulting IPA, and we need to complement it with the bottom 12 bits of the faulting VA. This is always 12 bits, irrespective of the page size. Makes it clearer in the code. Signed-off-by: Marc Zyngier --- arch/arm/kvm/mmu.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index ec14269a791c..efded4f1a11b 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -606,8 +606,13 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) goto out_unlock; } - /* Adjust page offset */ - fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ~PAGE_MASK; + /* + * The IPA is reported as [MAX:12], so we need to + * complement it with the bottom 12 bits from the + * faulting VA. This is always 12 bits, irrespective + * of the page size. + */ + fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1); ret = io_mem_abort(vcpu, run, fault_ipa); goto out_unlock; } From d7bb077737f34ed474a7256e1f368df6373238fa Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 12 Dec 2012 14:37:10 +0000 Subject: [PATCH 26/31] ARM: KVM: vgic: decouple alignment restriction from page size The virtual GIC is supposed to be 4kB aligned. On a 64kB page system, comparing the alignment to PAGE_SIZE is wrong. Use SZ_4K instead. Signed-off-by: Marc Zyngier --- arch/arm/kvm/vgic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/kvm/vgic.c b/arch/arm/kvm/vgic.c index c9a17316e9fe..161d5c15f0f0 100644 --- a/arch/arm/kvm/vgic.c +++ b/arch/arm/kvm/vgic.c @@ -1484,7 +1484,7 @@ int kvm_vgic_set_addr(struct kvm *kvm, unsigned long type, u64 addr) if (addr & ~KVM_PHYS_MASK) return -E2BIG; - if (addr & ~PAGE_MASK) + if (addr & (SZ_4K - 1)) return -EINVAL; mutex_lock(&kvm->lock); From 06fe0b73ff17e5d777af1b26f3e227d79c0d6808 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 17 Dec 2012 11:57:24 +0000 Subject: [PATCH 27/31] ARM: KVM: move include of asm/idmap.h to kvm_mmu.h Since the arm64 code doesn't have a global asm/idmap.h file, move the inclusion to asm/kvm_mmu.h. Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_mmu.h | 1 + arch/arm/kvm/mmu.c | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h index 3c71a1d4b7a3..970f3b5fa109 100644 --- a/arch/arm/include/asm/kvm_mmu.h +++ b/arch/arm/include/asm/kvm_mmu.h @@ -21,6 +21,7 @@ #include #include +#include /* * We directly use the kernel VA for the HYP, as we can directly share diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index efded4f1a11b..692f064fde0a 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include From 48762767e1c150d58c250650f8202b7d4ad65ec4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 28 Jan 2013 15:27:00 +0000 Subject: [PATCH 28/31] ARM: KVM: change kvm_tlb_flush_vmid to kvm_tlb_flush_vmid_ipa v8 is capable of invalidating Stage-2 by IPA, but v7 is not. Change kvm_tlb_flush_vmid() to take an IPA parameter, which is then ignored by the invalidation code (and nuke the whole TLB as it always did). This allows v8 to implement a more optimized strategy. Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_asm.h | 2 +- arch/arm/kvm/interrupts.S | 9 ++++++--- arch/arm/kvm/mmu.c | 8 ++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h index e4956f4e23e1..18d50322a9e2 100644 --- a/arch/arm/include/asm/kvm_asm.h +++ b/arch/arm/include/asm/kvm_asm.h @@ -75,7 +75,7 @@ extern char __kvm_hyp_code_end[]; extern void __kvm_tlb_flush_vmid(struct kvm *kvm); extern void __kvm_flush_vm_context(void); -extern void __kvm_tlb_flush_vmid(struct kvm *kvm); +extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); #endif diff --git a/arch/arm/kvm/interrupts.S b/arch/arm/kvm/interrupts.S index a8e0c2d85cb5..f7793df62f58 100644 --- a/arch/arm/kvm/interrupts.S +++ b/arch/arm/kvm/interrupts.S @@ -35,15 +35,18 @@ __kvm_hyp_code_start: /******************************************************************** * Flush per-VMID TLBs * - * void __kvm_tlb_flush_vmid(struct kvm *kvm); + * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); * * We rely on the hardware to broadcast the TLB invalidation to all CPUs * inside the inner-shareable domain (which is the case for all v7 * implementations). If we come across a non-IS SMP implementation, we'll * have to use an IPI based mechanism. Until then, we stick to the simple * hardware assisted version. + * + * As v7 does not support flushing per IPA, just nuke the whole TLB + * instead, ignoring the ipa value. */ -ENTRY(__kvm_tlb_flush_vmid) +ENTRY(__kvm_tlb_flush_vmid_ipa) push {r2, r3} add r0, r0, #KVM_VTTBR @@ -60,7 +63,7 @@ ENTRY(__kvm_tlb_flush_vmid) pop {r2, r3} bx lr -ENDPROC(__kvm_tlb_flush_vmid) +ENDPROC(__kvm_tlb_flush_vmid_ipa) /******************************************************************** * Flush TLBs and instruction caches of all CPUs inside the inner-shareable diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index 692f064fde0a..0bf2c8551f75 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -34,9 +34,9 @@ extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[]; static DEFINE_MUTEX(kvm_hyp_pgd_mutex); -static void kvm_tlb_flush_vmid(struct kvm *kvm) +static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa) { - kvm_call_hyp(__kvm_tlb_flush_vmid, kvm); + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa); } static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, @@ -449,7 +449,7 @@ static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, old_pte = *pte; kvm_set_pte(pte, *new_pte); if (pte_present(old_pte)) - kvm_tlb_flush_vmid(kvm); + kvm_tlb_flush_vmid_ipa(kvm, addr); else get_page(virt_to_page(pte)); @@ -666,7 +666,7 @@ static void handle_hva_to_gpa(struct kvm *kvm, static void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data) { unmap_stage2_range(kvm, gpa, PAGE_SIZE); - kvm_tlb_flush_vmid(kvm); + kvm_tlb_flush_vmid_ipa(kvm, gpa); } int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) From 6190920a35068a621570cca7f07f3c4477615176 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 5 Oct 2012 13:42:45 +0100 Subject: [PATCH 29/31] ARM: KVM: move kvm_handle_wfi to handle_exit.c It has little to do in emulate.c these days... Signed-off-by: Marc Zyngier --- arch/arm/include/asm/kvm_emulate.h | 1 - arch/arm/kvm/emulate.c | 16 ---------------- arch/arm/kvm/handle_exit.c | 19 +++++++++++++++++++ 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index 9cb2fe17cf3a..82b4babead2c 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -27,7 +27,6 @@ unsigned long *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num); unsigned long *vcpu_spsr(struct kvm_vcpu *vcpu); -int kvm_handle_wfi(struct kvm_vcpu *vcpu, struct kvm_run *run); bool kvm_condition_valid(struct kvm_vcpu *vcpu); void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr); void kvm_inject_undefined(struct kvm_vcpu *vcpu); diff --git a/arch/arm/kvm/emulate.c b/arch/arm/kvm/emulate.c index 04dbac6bdf4d..bdede9e7da51 100644 --- a/arch/arm/kvm/emulate.c +++ b/arch/arm/kvm/emulate.c @@ -161,22 +161,6 @@ unsigned long *vcpu_spsr(struct kvm_vcpu *vcpu) } } -/** - * kvm_handle_wfi - handle a wait-for-interrupts instruction executed by a guest - * @vcpu: the vcpu pointer - * @run: the kvm_run structure pointer - * - * Simply sets the wait_for_interrupts flag on the vcpu structure, which will - * halt execution of world-switches and schedule other host processes until - * there is an incoming IRQ or FIQ to the VM. - */ -int kvm_handle_wfi(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - trace_kvm_wfi(*vcpu_pc(vcpu)); - kvm_vcpu_block(vcpu); - return 1; -} - /* * A conditional instruction is allowed to trap, even though it * wouldn't be executed. So let's re-implement the hardware, in diff --git a/arch/arm/kvm/handle_exit.c b/arch/arm/kvm/handle_exit.c index 301301808c4e..26ad17310a1e 100644 --- a/arch/arm/kvm/handle_exit.c +++ b/arch/arm/kvm/handle_exit.c @@ -22,6 +22,9 @@ #include #include #include +#include + +#include "trace.h" #include "trace.h" @@ -72,6 +75,22 @@ static int handle_dabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run) return -EFAULT; } +/** + * kvm_handle_wfi - handle a wait-for-interrupts instruction executed by a guest + * @vcpu: the vcpu pointer + * @run: the kvm_run structure pointer + * + * Simply sets the wait_for_interrupts flag on the vcpu structure, which will + * halt execution of world-switches and schedule other host processes until + * there is an incoming IRQ or FIQ to the VM. + */ +static int kvm_handle_wfi(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + trace_kvm_wfi(*vcpu_pc(vcpu)); + kvm_vcpu_block(vcpu); + return 1; +} + static exit_handle_fn arm_exit_handlers[] = { [HSR_EC_WFI] = kvm_handle_wfi, [HSR_EC_CP15_32] = kvm_handle_cp15_32, From 000d399625b4b302935508f2fc9ce93ff1bd1ba4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 5 Mar 2013 02:43:17 +0000 Subject: [PATCH 30/31] ARM: KVM: sanitize freeing of HYP page tables Instead of trying to free everything from PAGE_OFFSET to the top of memory, use the virt_addr_valid macro to check the upper limit. Also do the same for the vmalloc region where the IO mappings are allocated. Signed-off-by: Marc Zyngier Signed-off-by: Christoffer Dall --- arch/arm/kvm/mmu.c | 50 +++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index 0bf2c8551f75..2f12e4056408 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -85,34 +85,42 @@ static void free_ptes(pmd_t *pmd, unsigned long addr) } } -/** - * free_hyp_pmds - free a Hyp-mode level-2 tables and child level-3 tables - * - * Assumes this is a page table used strictly in Hyp-mode and therefore contains - * only mappings in the kernel memory area, which is above PAGE_OFFSET. - */ -void free_hyp_pmds(void) +static void free_hyp_pgd_entry(unsigned long addr) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; + unsigned long hyp_addr = KERN_TO_HYP(addr); + + pgd = hyp_pgd + pgd_index(hyp_addr); + pud = pud_offset(pgd, hyp_addr); + + if (pud_none(*pud)) + return; + BUG_ON(pud_bad(*pud)); + + pmd = pmd_offset(pud, hyp_addr); + free_ptes(pmd, addr); + pmd_free(NULL, pmd); + pud_clear(pud); +} + +/** + * free_hyp_pmds - free a Hyp-mode level-2 tables and child level-3 tables + * + * Assumes this is a page table used strictly in Hyp-mode and therefore contains + * either mappings in the kernel memory area (above PAGE_OFFSET), or + * device mappings in the vmalloc range (from VMALLOC_START to VMALLOC_END). + */ +void free_hyp_pmds(void) +{ unsigned long addr; mutex_lock(&kvm_hyp_pgd_mutex); - for (addr = PAGE_OFFSET; addr != 0; addr += PGDIR_SIZE) { - unsigned long hyp_addr = KERN_TO_HYP(addr); - pgd = hyp_pgd + pgd_index(hyp_addr); - pud = pud_offset(pgd, hyp_addr); - - if (pud_none(*pud)) - continue; - BUG_ON(pud_bad(*pud)); - - pmd = pmd_offset(pud, hyp_addr); - free_ptes(pmd, addr); - pmd_free(NULL, pmd); - pud_clear(pud); - } + for (addr = PAGE_OFFSET; virt_addr_valid(addr); addr += PGDIR_SIZE) + free_hyp_pgd_entry(addr); + for (addr = VMALLOC_START; is_vmalloc_addr((void*)addr); addr += PGDIR_SIZE) + free_hyp_pgd_entry(addr); mutex_unlock(&kvm_hyp_pgd_mutex); } From f42798c6898bf1e536673e798d263e492355162f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 5 Mar 2013 02:43:23 +0000 Subject: [PATCH 31/31] ARM: KVM: Fix length of mmio access Instead of hardcoding the maximum MMIO access to be 4 bytes, compare it to sizeof(unsigned long), which will do the right thing on both 32 and 64bit systems. Same thing for sign extention. Signed-off-by: Marc Zyngier Signed-off-by: Christoffer Dall --- arch/arm/kvm/mmio.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/arm/kvm/mmio.c b/arch/arm/kvm/mmio.c index 02ca76555bd4..72a12f2171b2 100644 --- a/arch/arm/kvm/mmio.c +++ b/arch/arm/kvm/mmio.c @@ -39,10 +39,10 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) if (!run->mmio.is_write) { dest = vcpu_reg(vcpu, vcpu->arch.mmio_decode.rt); - memset(dest, 0, sizeof(int)); + *dest = 0; len = run->mmio.len; - if (len > 4) + if (len > sizeof(unsigned long)) return -EINVAL; memcpy(dest, run->mmio.data, len); @@ -50,7 +50,8 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr, *((u64 *)run->mmio.data)); - if (vcpu->arch.mmio_decode.sign_extend && len < 4) { + if (vcpu->arch.mmio_decode.sign_extend && + len < sizeof(unsigned long)) { mask = 1U << ((len * 8) - 1); *dest = (*dest ^ mask) - mask; }