2019-06-02 23:44:50 -06:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2015-11-26 10:19:25 -07:00
|
|
|
/*
|
|
|
|
|
* Copyright (C) 2015, 2016 ARM Ltd.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <linux/irqchip/arm-gic.h>
|
|
|
|
|
#include <linux/kvm.h>
|
|
|
|
|
#include <linux/kvm_host.h>
|
2015-12-01 07:02:35 -07:00
|
|
|
#include <kvm/arm_vgic.h>
|
|
|
|
|
#include <asm/kvm_mmu.h>
|
2015-11-26 10:19:25 -07:00
|
|
|
|
|
|
|
|
#include "vgic.h"
|
|
|
|
|
|
2017-03-18 06:56:56 -06:00
|
|
|
static inline void vgic_v2_write_lr(int lr, u32 val)
|
|
|
|
|
{
|
|
|
|
|
void __iomem *base = kvm_vgic_global_state.vctrl_base;
|
|
|
|
|
|
|
|
|
|
writel_relaxed(val, base + GICH_LR0 + (lr * 4));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void vgic_v2_init_lrs(void)
|
|
|
|
|
{
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < kvm_vgic_global_state.nr_lr; i++)
|
|
|
|
|
vgic_v2_write_lr(i, 0);
|
|
|
|
|
}
|
|
|
|
|
|
2016-12-29 07:44:27 -07:00
|
|
|
void vgic_v2_set_underflow(struct kvm_vcpu *vcpu)
|
2015-11-26 10:19:25 -07:00
|
|
|
{
|
|
|
|
|
struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
|
|
|
|
|
|
2016-12-29 07:44:27 -07:00
|
|
|
cpuif->vgic_hcr |= GICH_HCR_UIE;
|
2015-11-26 10:19:25 -07:00
|
|
|
}
|
|
|
|
|
|
2016-12-29 07:44:27 -07:00
|
|
|
static bool lr_signals_eoi_mi(u32 lr_val)
|
2015-11-26 10:19:25 -07:00
|
|
|
{
|
2016-12-29 07:44:27 -07:00
|
|
|
return !(lr_val & GICH_LR_STATE) && (lr_val & GICH_LR_EOI) &&
|
|
|
|
|
!(lr_val & GICH_LR_HW);
|
2015-11-26 10:19:25 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* transfer the content of the LRs back into the corresponding ap_list:
|
|
|
|
|
* - active bit is transferred as is
|
|
|
|
|
* - pending bit is
|
|
|
|
|
* - transferred as is in case of edge sensitive IRQs
|
|
|
|
|
* - set to the line-level (resample time) for level sensitive IRQs
|
|
|
|
|
*/
|
|
|
|
|
void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
2017-03-18 06:48:42 -06:00
|
|
|
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
|
|
|
|
|
struct vgic_v2_cpu_if *cpuif = &vgic_cpu->vgic_v2;
|
2015-11-26 10:19:25 -07:00
|
|
|
int lr;
|
2018-08-03 07:57:04 -06:00
|
|
|
|
|
|
|
|
DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
|
2015-11-26 10:19:25 -07:00
|
|
|
|
KVM: arm/arm64: vgic: Fix source vcpu issues for GICv2 SGI
Now that we make sure we don't inject multiple instances of the
same GICv2 SGI at the same time, we've made another bug more
obvious:
If we exit with an active SGI, we completely lose track of which
vcpu it came from. On the next entry, we restore it with 0 as a
source, and if that wasn't the right one, too bad. While this
doesn't seem to trouble GIC-400, the architectural model gets
offended and doesn't deactivate the interrupt on EOI.
Another connected issue is that we will happilly make pending
an interrupt from another vcpu, overriding the above zero with
something that is just as inconsistent. Don't do that.
The final issue is that we signal a maintenance interrupt when
no pending interrupts are present in the LR. Assuming we've fixed
the two issues above, we end-up in a situation where we keep
exiting as soon as we've reached the active state, and not be
able to inject the following pending.
The fix comes in 3 parts:
- GICv2 SGIs have their source vcpu saved if they are active on
exit, and restored on entry
- Multi-SGIs cannot go via the Pending+Active state, as this would
corrupt the source field
- Multi-SGIs are converted to using MI on EOI instead of NPIE
Fixes: 16ca6a607d84bef0 ("KVM: arm/arm64: vgic: Don't populate multiple LRs with the same vintid")
Reported-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-04-18 03:39:04 -06:00
|
|
|
cpuif->vgic_hcr &= ~GICH_HCR_UIE;
|
2016-12-29 07:44:27 -07:00
|
|
|
|
2017-03-18 06:48:42 -06:00
|
|
|
for (lr = 0; lr < vgic_cpu->used_lrs; lr++) {
|
2015-11-26 10:19:25 -07:00
|
|
|
u32 val = cpuif->vgic_lr[lr];
|
KVM: arm/arm64: vgic: Fix source vcpu issues for GICv2 SGI
Now that we make sure we don't inject multiple instances of the
same GICv2 SGI at the same time, we've made another bug more
obvious:
If we exit with an active SGI, we completely lose track of which
vcpu it came from. On the next entry, we restore it with 0 as a
source, and if that wasn't the right one, too bad. While this
doesn't seem to trouble GIC-400, the architectural model gets
offended and doesn't deactivate the interrupt on EOI.
Another connected issue is that we will happilly make pending
an interrupt from another vcpu, overriding the above zero with
something that is just as inconsistent. Don't do that.
The final issue is that we signal a maintenance interrupt when
no pending interrupts are present in the LR. Assuming we've fixed
the two issues above, we end-up in a situation where we keep
exiting as soon as we've reached the active state, and not be
able to inject the following pending.
The fix comes in 3 parts:
- GICv2 SGIs have their source vcpu saved if they are active on
exit, and restored on entry
- Multi-SGIs cannot go via the Pending+Active state, as this would
corrupt the source field
- Multi-SGIs are converted to using MI on EOI instead of NPIE
Fixes: 16ca6a607d84bef0 ("KVM: arm/arm64: vgic: Don't populate multiple LRs with the same vintid")
Reported-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-04-18 03:39:04 -06:00
|
|
|
u32 cpuid, intid = val & GICH_LR_VIRTUALID;
|
2015-11-26 10:19:25 -07:00
|
|
|
struct vgic_irq *irq;
|
|
|
|
|
|
KVM: arm/arm64: vgic: Fix source vcpu issues for GICv2 SGI
Now that we make sure we don't inject multiple instances of the
same GICv2 SGI at the same time, we've made another bug more
obvious:
If we exit with an active SGI, we completely lose track of which
vcpu it came from. On the next entry, we restore it with 0 as a
source, and if that wasn't the right one, too bad. While this
doesn't seem to trouble GIC-400, the architectural model gets
offended and doesn't deactivate the interrupt on EOI.
Another connected issue is that we will happilly make pending
an interrupt from another vcpu, overriding the above zero with
something that is just as inconsistent. Don't do that.
The final issue is that we signal a maintenance interrupt when
no pending interrupts are present in the LR. Assuming we've fixed
the two issues above, we end-up in a situation where we keep
exiting as soon as we've reached the active state, and not be
able to inject the following pending.
The fix comes in 3 parts:
- GICv2 SGIs have their source vcpu saved if they are active on
exit, and restored on entry
- Multi-SGIs cannot go via the Pending+Active state, as this would
corrupt the source field
- Multi-SGIs are converted to using MI on EOI instead of NPIE
Fixes: 16ca6a607d84bef0 ("KVM: arm/arm64: vgic: Don't populate multiple LRs with the same vintid")
Reported-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-04-18 03:39:04 -06:00
|
|
|
/* Extract the source vCPU id from the LR */
|
|
|
|
|
cpuid = val & GICH_LR_PHYSID_CPUID;
|
|
|
|
|
cpuid >>= GICH_LR_PHYSID_CPUID_SHIFT;
|
|
|
|
|
cpuid &= 7;
|
|
|
|
|
|
2016-12-29 07:44:27 -07:00
|
|
|
/* Notify fds when the guest EOI'ed a level-triggered SPI */
|
|
|
|
|
if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid))
|
|
|
|
|
kvm_notify_acked_irq(vcpu->kvm, 0,
|
|
|
|
|
intid - VGIC_NR_PRIVATE_IRQS);
|
|
|
|
|
|
2015-11-26 10:19:25 -07:00
|
|
|
irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
|
|
|
|
|
|
2019-01-07 08:06:15 -07:00
|
|
|
raw_spin_lock(&irq->irq_lock);
|
2015-11-26 10:19:25 -07:00
|
|
|
|
|
|
|
|
/* Always preserve the active bit */
|
|
|
|
|
irq->active = !!(val & GICH_LR_ACTIVE_BIT);
|
|
|
|
|
|
KVM: arm/arm64: vgic: Fix source vcpu issues for GICv2 SGI
Now that we make sure we don't inject multiple instances of the
same GICv2 SGI at the same time, we've made another bug more
obvious:
If we exit with an active SGI, we completely lose track of which
vcpu it came from. On the next entry, we restore it with 0 as a
source, and if that wasn't the right one, too bad. While this
doesn't seem to trouble GIC-400, the architectural model gets
offended and doesn't deactivate the interrupt on EOI.
Another connected issue is that we will happilly make pending
an interrupt from another vcpu, overriding the above zero with
something that is just as inconsistent. Don't do that.
The final issue is that we signal a maintenance interrupt when
no pending interrupts are present in the LR. Assuming we've fixed
the two issues above, we end-up in a situation where we keep
exiting as soon as we've reached the active state, and not be
able to inject the following pending.
The fix comes in 3 parts:
- GICv2 SGIs have their source vcpu saved if they are active on
exit, and restored on entry
- Multi-SGIs cannot go via the Pending+Active state, as this would
corrupt the source field
- Multi-SGIs are converted to using MI on EOI instead of NPIE
Fixes: 16ca6a607d84bef0 ("KVM: arm/arm64: vgic: Don't populate multiple LRs with the same vintid")
Reported-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-04-18 03:39:04 -06:00
|
|
|
if (irq->active && vgic_irq_is_sgi(intid))
|
|
|
|
|
irq->active_source = cpuid;
|
|
|
|
|
|
2015-11-26 10:19:25 -07:00
|
|
|
/* Edge is the only case where we preserve the pending bit */
|
|
|
|
|
if (irq->config == VGIC_CONFIG_EDGE &&
|
|
|
|
|
(val & GICH_LR_PENDING_BIT)) {
|
2017-01-23 06:07:18 -07:00
|
|
|
irq->pending_latch = true;
|
2015-11-26 10:19:25 -07:00
|
|
|
|
KVM: arm/arm64: vgic: Fix source vcpu issues for GICv2 SGI
Now that we make sure we don't inject multiple instances of the
same GICv2 SGI at the same time, we've made another bug more
obvious:
If we exit with an active SGI, we completely lose track of which
vcpu it came from. On the next entry, we restore it with 0 as a
source, and if that wasn't the right one, too bad. While this
doesn't seem to trouble GIC-400, the architectural model gets
offended and doesn't deactivate the interrupt on EOI.
Another connected issue is that we will happilly make pending
an interrupt from another vcpu, overriding the above zero with
something that is just as inconsistent. Don't do that.
The final issue is that we signal a maintenance interrupt when
no pending interrupts are present in the LR. Assuming we've fixed
the two issues above, we end-up in a situation where we keep
exiting as soon as we've reached the active state, and not be
able to inject the following pending.
The fix comes in 3 parts:
- GICv2 SGIs have their source vcpu saved if they are active on
exit, and restored on entry
- Multi-SGIs cannot go via the Pending+Active state, as this would
corrupt the source field
- Multi-SGIs are converted to using MI on EOI instead of NPIE
Fixes: 16ca6a607d84bef0 ("KVM: arm/arm64: vgic: Don't populate multiple LRs with the same vintid")
Reported-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-04-18 03:39:04 -06:00
|
|
|
if (vgic_irq_is_sgi(intid))
|
2015-11-26 10:19:25 -07:00
|
|
|
irq->source |= (1 << cpuid);
|
|
|
|
|
}
|
|
|
|
|
|
2016-05-25 08:26:35 -06:00
|
|
|
/*
|
|
|
|
|
* Clear soft pending state when level irqs have been acked.
|
|
|
|
|
*/
|
2018-03-09 07:59:40 -07:00
|
|
|
if (irq->config == VGIC_CONFIG_LEVEL && !(val & GICH_LR_STATE))
|
|
|
|
|
irq->pending_latch = false;
|
2015-11-26 10:19:25 -07:00
|
|
|
|
KVM: arm/arm64: vgic: Support level-triggered mapped interrupts
Level-triggered mapped IRQs are special because we only observe rising
edges as input to the VGIC, and we don't set the EOI flag and therefore
are not told when the level goes down, so that we can re-queue a new
interrupt when the level goes up.
One way to solve this problem is to side-step the logic of the VGIC and
special case the validation in the injection path, but it has the
unfortunate drawback of having to peak into the physical GIC state
whenever we want to know if the interrupt is pending on the virtual
distributor.
Instead, we can maintain the current semantics of a level triggered
interrupt by sort of treating it as an edge-triggered interrupt,
following from the fact that we only observe an asserting edge. This
requires us to be a bit careful when populating the LRs and when folding
the state back in though:
* We lower the line level when populating the LR, so that when
subsequently observing an asserting edge, the VGIC will do the right
thing.
* If the guest never acked the interrupt while running (for example if
it had masked interrupts at the CPU level while running), we have
to preserve the pending state of the LR and move it back to the
line_level field of the struct irq when folding LR state.
If the guest never acked the interrupt while running, but changed the
device state and lowered the line (again with interrupts masked) then
we need to observe this change in the line_level.
Both of the above situations are solved by sampling the physical line
and set the line level when folding the LR back.
* Finally, if the guest never acked the interrupt while running and
sampling the line reveals that the device state has changed and the
line has been lowered, we must clear the physical active state, since
we will otherwise never be told when the interrupt becomes asserted
again.
This has the added benefit of making the timer optimization patches
(https://lists.cs.columbia.edu/pipermail/kvmarm/2017-July/026343.html) a
bit simpler, because the timer code doesn't have to clear the active
state on the sync anymore. It also potentially improves the performance
of the timer implementation because the GIC knows the state or the LR
and only needs to clear the
active state when the pending bit in the LR is still set, where the
timer has to always clear it when returning from running the guest with
an injected timer interrupt.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2017-08-29 02:40:44 -06:00
|
|
|
/*
|
|
|
|
|
* Level-triggered mapped IRQs are special because we only
|
|
|
|
|
* observe rising edges as input to the VGIC.
|
|
|
|
|
*
|
|
|
|
|
* If the guest never acked the interrupt we have to sample
|
|
|
|
|
* the physical line and set the line level, because the
|
|
|
|
|
* device state could have changed or we simply need to
|
|
|
|
|
* process the still pending interrupt later.
|
|
|
|
|
*
|
|
|
|
|
* If this causes us to lower the level, we have to also clear
|
|
|
|
|
* the physical active state, since we will otherwise never be
|
|
|
|
|
* told when the interrupt becomes asserted again.
|
|
|
|
|
*/
|
|
|
|
|
if (vgic_irq_is_mapped_level(irq) && (val & GICH_LR_PENDING_BIT)) {
|
|
|
|
|
irq->line_level = vgic_get_phys_line_level(irq);
|
|
|
|
|
|
|
|
|
|
if (!irq->line_level)
|
|
|
|
|
vgic_irq_set_phys_active(irq, false);
|
|
|
|
|
}
|
|
|
|
|
|
2019-01-07 08:06:15 -07:00
|
|
|
raw_spin_unlock(&irq->irq_lock);
|
2016-07-15 05:43:27 -06:00
|
|
|
vgic_put_irq(vcpu->kvm, irq);
|
2015-11-26 10:19:25 -07:00
|
|
|
}
|
2017-03-18 06:48:42 -06:00
|
|
|
|
|
|
|
|
vgic_cpu->used_lrs = 0;
|
2015-11-26 10:19:25 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Populates the particular LR with the state of a given IRQ:
|
|
|
|
|
* - for an edge sensitive IRQ the pending state is cleared in struct vgic_irq
|
|
|
|
|
* - for a level sensitive IRQ the pending state value is unchanged;
|
|
|
|
|
* it is dictated directly by the input level
|
|
|
|
|
*
|
|
|
|
|
* If @irq describes an SGI with multiple sources, we choose the
|
|
|
|
|
* lowest-numbered source VCPU and clear that bit in the source bitmap.
|
|
|
|
|
*
|
|
|
|
|
* The irq_lock must be held by the caller.
|
|
|
|
|
*/
|
|
|
|
|
void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr)
|
|
|
|
|
{
|
|
|
|
|
u32 val = irq->intid;
|
2018-03-09 07:59:40 -07:00
|
|
|
bool allow_pending = true;
|
|
|
|
|
|
KVM: arm/arm64: vgic: Fix source vcpu issues for GICv2 SGI
Now that we make sure we don't inject multiple instances of the
same GICv2 SGI at the same time, we've made another bug more
obvious:
If we exit with an active SGI, we completely lose track of which
vcpu it came from. On the next entry, we restore it with 0 as a
source, and if that wasn't the right one, too bad. While this
doesn't seem to trouble GIC-400, the architectural model gets
offended and doesn't deactivate the interrupt on EOI.
Another connected issue is that we will happilly make pending
an interrupt from another vcpu, overriding the above zero with
something that is just as inconsistent. Don't do that.
The final issue is that we signal a maintenance interrupt when
no pending interrupts are present in the LR. Assuming we've fixed
the two issues above, we end-up in a situation where we keep
exiting as soon as we've reached the active state, and not be
able to inject the following pending.
The fix comes in 3 parts:
- GICv2 SGIs have their source vcpu saved if they are active on
exit, and restored on entry
- Multi-SGIs cannot go via the Pending+Active state, as this would
corrupt the source field
- Multi-SGIs are converted to using MI on EOI instead of NPIE
Fixes: 16ca6a607d84bef0 ("KVM: arm/arm64: vgic: Don't populate multiple LRs with the same vintid")
Reported-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-04-18 03:39:04 -06:00
|
|
|
if (irq->active) {
|
2018-03-09 07:59:40 -07:00
|
|
|
val |= GICH_LR_ACTIVE_BIT;
|
KVM: arm/arm64: vgic: Fix source vcpu issues for GICv2 SGI
Now that we make sure we don't inject multiple instances of the
same GICv2 SGI at the same time, we've made another bug more
obvious:
If we exit with an active SGI, we completely lose track of which
vcpu it came from. On the next entry, we restore it with 0 as a
source, and if that wasn't the right one, too bad. While this
doesn't seem to trouble GIC-400, the architectural model gets
offended and doesn't deactivate the interrupt on EOI.
Another connected issue is that we will happilly make pending
an interrupt from another vcpu, overriding the above zero with
something that is just as inconsistent. Don't do that.
The final issue is that we signal a maintenance interrupt when
no pending interrupts are present in the LR. Assuming we've fixed
the two issues above, we end-up in a situation where we keep
exiting as soon as we've reached the active state, and not be
able to inject the following pending.
The fix comes in 3 parts:
- GICv2 SGIs have their source vcpu saved if they are active on
exit, and restored on entry
- Multi-SGIs cannot go via the Pending+Active state, as this would
corrupt the source field
- Multi-SGIs are converted to using MI on EOI instead of NPIE
Fixes: 16ca6a607d84bef0 ("KVM: arm/arm64: vgic: Don't populate multiple LRs with the same vintid")
Reported-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-04-18 03:39:04 -06:00
|
|
|
if (vgic_irq_is_sgi(irq->intid))
|
|
|
|
|
val |= irq->active_source << GICH_LR_PHYSID_CPUID_SHIFT;
|
|
|
|
|
if (vgic_irq_is_multi_sgi(irq)) {
|
|
|
|
|
allow_pending = false;
|
|
|
|
|
val |= GICH_LR_EOI;
|
|
|
|
|
}
|
|
|
|
|
}
|
2018-03-09 07:59:40 -07:00
|
|
|
|
2018-07-16 07:06:22 -06:00
|
|
|
if (irq->group)
|
|
|
|
|
val |= GICH_LR_GROUP1;
|
|
|
|
|
|
2018-03-09 07:59:40 -07:00
|
|
|
if (irq->hw) {
|
|
|
|
|
val |= GICH_LR_HW;
|
|
|
|
|
val |= irq->hwintid << GICH_LR_PHYSID_CPUID_SHIFT;
|
|
|
|
|
/*
|
|
|
|
|
* Never set pending+active on a HW interrupt, as the
|
|
|
|
|
* pending state is kept at the physical distributor
|
|
|
|
|
* level.
|
|
|
|
|
*/
|
|
|
|
|
if (irq->active)
|
|
|
|
|
allow_pending = false;
|
|
|
|
|
} else {
|
|
|
|
|
if (irq->config == VGIC_CONFIG_LEVEL) {
|
|
|
|
|
val |= GICH_LR_EOI;
|
2015-11-26 10:19:25 -07:00
|
|
|
|
2018-03-09 07:59:40 -07:00
|
|
|
/*
|
|
|
|
|
* Software resampling doesn't work very well
|
|
|
|
|
* if we allow P+A, so let's not do that.
|
|
|
|
|
*/
|
|
|
|
|
if (irq->active)
|
|
|
|
|
allow_pending = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (allow_pending && irq_is_pending(irq)) {
|
2015-11-26 10:19:25 -07:00
|
|
|
val |= GICH_LR_PENDING_BIT;
|
|
|
|
|
|
|
|
|
|
if (irq->config == VGIC_CONFIG_EDGE)
|
2017-01-23 06:07:18 -07:00
|
|
|
irq->pending_latch = false;
|
2015-11-26 10:19:25 -07:00
|
|
|
|
|
|
|
|
if (vgic_irq_is_sgi(irq->intid)) {
|
|
|
|
|
u32 src = ffs(irq->source);
|
|
|
|
|
|
2019-08-28 04:10:16 -06:00
|
|
|
if (WARN_RATELIMIT(!src, "No SGI source for INTID %d\n",
|
|
|
|
|
irq->intid))
|
|
|
|
|
return;
|
|
|
|
|
|
2015-11-26 10:19:25 -07:00
|
|
|
val |= (src - 1) << GICH_LR_PHYSID_CPUID_SHIFT;
|
|
|
|
|
irq->source &= ~(1 << (src - 1));
|
KVM: arm/arm64: vgic: Fix source vcpu issues for GICv2 SGI
Now that we make sure we don't inject multiple instances of the
same GICv2 SGI at the same time, we've made another bug more
obvious:
If we exit with an active SGI, we completely lose track of which
vcpu it came from. On the next entry, we restore it with 0 as a
source, and if that wasn't the right one, too bad. While this
doesn't seem to trouble GIC-400, the architectural model gets
offended and doesn't deactivate the interrupt on EOI.
Another connected issue is that we will happilly make pending
an interrupt from another vcpu, overriding the above zero with
something that is just as inconsistent. Don't do that.
The final issue is that we signal a maintenance interrupt when
no pending interrupts are present in the LR. Assuming we've fixed
the two issues above, we end-up in a situation where we keep
exiting as soon as we've reached the active state, and not be
able to inject the following pending.
The fix comes in 3 parts:
- GICv2 SGIs have their source vcpu saved if they are active on
exit, and restored on entry
- Multi-SGIs cannot go via the Pending+Active state, as this would
corrupt the source field
- Multi-SGIs are converted to using MI on EOI instead of NPIE
Fixes: 16ca6a607d84bef0 ("KVM: arm/arm64: vgic: Don't populate multiple LRs with the same vintid")
Reported-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-04-18 03:39:04 -06:00
|
|
|
if (irq->source) {
|
2017-01-23 06:07:18 -07:00
|
|
|
irq->pending_latch = true;
|
KVM: arm/arm64: vgic: Fix source vcpu issues for GICv2 SGI
Now that we make sure we don't inject multiple instances of the
same GICv2 SGI at the same time, we've made another bug more
obvious:
If we exit with an active SGI, we completely lose track of which
vcpu it came from. On the next entry, we restore it with 0 as a
source, and if that wasn't the right one, too bad. While this
doesn't seem to trouble GIC-400, the architectural model gets
offended and doesn't deactivate the interrupt on EOI.
Another connected issue is that we will happilly make pending
an interrupt from another vcpu, overriding the above zero with
something that is just as inconsistent. Don't do that.
The final issue is that we signal a maintenance interrupt when
no pending interrupts are present in the LR. Assuming we've fixed
the two issues above, we end-up in a situation where we keep
exiting as soon as we've reached the active state, and not be
able to inject the following pending.
The fix comes in 3 parts:
- GICv2 SGIs have their source vcpu saved if they are active on
exit, and restored on entry
- Multi-SGIs cannot go via the Pending+Active state, as this would
corrupt the source field
- Multi-SGIs are converted to using MI on EOI instead of NPIE
Fixes: 16ca6a607d84bef0 ("KVM: arm/arm64: vgic: Don't populate multiple LRs with the same vintid")
Reported-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-04-18 03:39:04 -06:00
|
|
|
val |= GICH_LR_EOI;
|
|
|
|
|
}
|
2015-11-26 10:19:25 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
KVM: arm/arm64: vgic: Support level-triggered mapped interrupts
Level-triggered mapped IRQs are special because we only observe rising
edges as input to the VGIC, and we don't set the EOI flag and therefore
are not told when the level goes down, so that we can re-queue a new
interrupt when the level goes up.
One way to solve this problem is to side-step the logic of the VGIC and
special case the validation in the injection path, but it has the
unfortunate drawback of having to peak into the physical GIC state
whenever we want to know if the interrupt is pending on the virtual
distributor.
Instead, we can maintain the current semantics of a level triggered
interrupt by sort of treating it as an edge-triggered interrupt,
following from the fact that we only observe an asserting edge. This
requires us to be a bit careful when populating the LRs and when folding
the state back in though:
* We lower the line level when populating the LR, so that when
subsequently observing an asserting edge, the VGIC will do the right
thing.
* If the guest never acked the interrupt while running (for example if
it had masked interrupts at the CPU level while running), we have
to preserve the pending state of the LR and move it back to the
line_level field of the struct irq when folding LR state.
If the guest never acked the interrupt while running, but changed the
device state and lowered the line (again with interrupts masked) then
we need to observe this change in the line_level.
Both of the above situations are solved by sampling the physical line
and set the line level when folding the LR back.
* Finally, if the guest never acked the interrupt while running and
sampling the line reveals that the device state has changed and the
line has been lowered, we must clear the physical active state, since
we will otherwise never be told when the interrupt becomes asserted
again.
This has the added benefit of making the timer optimization patches
(https://lists.cs.columbia.edu/pipermail/kvmarm/2017-July/026343.html) a
bit simpler, because the timer code doesn't have to clear the active
state on the sync anymore. It also potentially improves the performance
of the timer implementation because the GIC knows the state or the LR
and only needs to clear the
active state when the pending bit in the LR is still set, where the
timer has to always clear it when returning from running the guest with
an injected timer interrupt.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2017-08-29 02:40:44 -06:00
|
|
|
/*
|
|
|
|
|
* Level-triggered mapped IRQs are special because we only observe
|
|
|
|
|
* rising edges as input to the VGIC. We therefore lower the line
|
|
|
|
|
* level here, so that we can take new virtual IRQs. See
|
|
|
|
|
* vgic_v2_fold_lr_state for more info.
|
|
|
|
|
*/
|
|
|
|
|
if (vgic_irq_is_mapped_level(irq) && (val & GICH_LR_PENDING_BIT))
|
|
|
|
|
irq->line_level = false;
|
|
|
|
|
|
2015-11-26 10:19:25 -07:00
|
|
|
/* The GICv2 LR only holds five bits of priority. */
|
|
|
|
|
val |= (irq->priority >> 3) << GICH_LR_PRIORITY_SHIFT;
|
|
|
|
|
|
|
|
|
|
vcpu->arch.vgic_cpu.vgic_v2.vgic_lr[lr] = val;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr)
|
|
|
|
|
{
|
|
|
|
|
vcpu->arch.vgic_cpu.vgic_v2.vgic_lr[lr] = 0;
|
|
|
|
|
}
|
2015-12-03 04:47:37 -07:00
|
|
|
|
|
|
|
|
void vgic_v2_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
|
|
|
|
|
{
|
2016-03-24 04:21:04 -06:00
|
|
|
struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
|
2015-12-03 04:47:37 -07:00
|
|
|
u32 vmcr;
|
|
|
|
|
|
2017-05-20 06:12:34 -06:00
|
|
|
vmcr = (vmcrp->grpen0 << GICH_VMCR_ENABLE_GRP0_SHIFT) &
|
|
|
|
|
GICH_VMCR_ENABLE_GRP0_MASK;
|
|
|
|
|
vmcr |= (vmcrp->grpen1 << GICH_VMCR_ENABLE_GRP1_SHIFT) &
|
|
|
|
|
GICH_VMCR_ENABLE_GRP1_MASK;
|
|
|
|
|
vmcr |= (vmcrp->ackctl << GICH_VMCR_ACK_CTL_SHIFT) &
|
|
|
|
|
GICH_VMCR_ACK_CTL_MASK;
|
|
|
|
|
vmcr |= (vmcrp->fiqen << GICH_VMCR_FIQ_EN_SHIFT) &
|
|
|
|
|
GICH_VMCR_FIQ_EN_MASK;
|
|
|
|
|
vmcr |= (vmcrp->cbpr << GICH_VMCR_CBPR_SHIFT) &
|
|
|
|
|
GICH_VMCR_CBPR_MASK;
|
|
|
|
|
vmcr |= (vmcrp->eoim << GICH_VMCR_EOI_MODE_SHIFT) &
|
|
|
|
|
GICH_VMCR_EOI_MODE_MASK;
|
2015-12-03 04:47:37 -07:00
|
|
|
vmcr |= (vmcrp->abpr << GICH_VMCR_ALIAS_BINPOINT_SHIFT) &
|
|
|
|
|
GICH_VMCR_ALIAS_BINPOINT_MASK;
|
|
|
|
|
vmcr |= (vmcrp->bpr << GICH_VMCR_BINPOINT_SHIFT) &
|
|
|
|
|
GICH_VMCR_BINPOINT_MASK;
|
2017-03-21 15:05:22 -06:00
|
|
|
vmcr |= ((vmcrp->pmr >> GICV_PMR_PRIORITY_SHIFT) <<
|
|
|
|
|
GICH_VMCR_PRIMASK_SHIFT) & GICH_VMCR_PRIMASK_MASK;
|
2015-12-03 04:47:37 -07:00
|
|
|
|
2016-03-24 04:21:04 -06:00
|
|
|
cpu_if->vgic_vmcr = vmcr;
|
2015-12-03 04:47:37 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void vgic_v2_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
|
|
|
|
|
{
|
2016-03-24 04:21:04 -06:00
|
|
|
struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
|
|
|
|
|
u32 vmcr;
|
|
|
|
|
|
|
|
|
|
vmcr = cpu_if->vgic_vmcr;
|
2015-12-03 04:47:37 -07:00
|
|
|
|
2017-05-20 06:12:34 -06:00
|
|
|
vmcrp->grpen0 = (vmcr & GICH_VMCR_ENABLE_GRP0_MASK) >>
|
|
|
|
|
GICH_VMCR_ENABLE_GRP0_SHIFT;
|
|
|
|
|
vmcrp->grpen1 = (vmcr & GICH_VMCR_ENABLE_GRP1_MASK) >>
|
|
|
|
|
GICH_VMCR_ENABLE_GRP1_SHIFT;
|
|
|
|
|
vmcrp->ackctl = (vmcr & GICH_VMCR_ACK_CTL_MASK) >>
|
|
|
|
|
GICH_VMCR_ACK_CTL_SHIFT;
|
|
|
|
|
vmcrp->fiqen = (vmcr & GICH_VMCR_FIQ_EN_MASK) >>
|
|
|
|
|
GICH_VMCR_FIQ_EN_SHIFT;
|
|
|
|
|
vmcrp->cbpr = (vmcr & GICH_VMCR_CBPR_MASK) >>
|
|
|
|
|
GICH_VMCR_CBPR_SHIFT;
|
|
|
|
|
vmcrp->eoim = (vmcr & GICH_VMCR_EOI_MODE_MASK) >>
|
|
|
|
|
GICH_VMCR_EOI_MODE_SHIFT;
|
|
|
|
|
|
2015-12-03 04:47:37 -07:00
|
|
|
vmcrp->abpr = (vmcr & GICH_VMCR_ALIAS_BINPOINT_MASK) >>
|
|
|
|
|
GICH_VMCR_ALIAS_BINPOINT_SHIFT;
|
|
|
|
|
vmcrp->bpr = (vmcr & GICH_VMCR_BINPOINT_MASK) >>
|
|
|
|
|
GICH_VMCR_BINPOINT_SHIFT;
|
2017-03-21 15:05:22 -06:00
|
|
|
vmcrp->pmr = ((vmcr & GICH_VMCR_PRIMASK_MASK) >>
|
|
|
|
|
GICH_VMCR_PRIMASK_SHIFT) << GICV_PMR_PRIORITY_SHIFT;
|
2015-12-03 04:47:37 -07:00
|
|
|
}
|
2015-12-01 07:02:35 -07:00
|
|
|
|
2015-12-21 10:09:38 -07:00
|
|
|
void vgic_v2_enable(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
2015-12-02 02:30:13 -07:00
|
|
|
/*
|
|
|
|
|
* By forcing VMCR to zero, the GIC will restore the binary
|
|
|
|
|
* points to their reset values. Anything else resets to zero
|
|
|
|
|
* anyway.
|
|
|
|
|
*/
|
|
|
|
|
vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr = 0;
|
|
|
|
|
|
|
|
|
|
/* Get the show on the road... */
|
|
|
|
|
vcpu->arch.vgic_cpu.vgic_v2.vgic_hcr = GICH_HCR_EN;
|
2015-12-21 10:09:38 -07:00
|
|
|
}
|
|
|
|
|
|
2015-12-21 07:04:42 -07:00
|
|
|
/* check for overlapping regions and for regions crossing the end of memory */
|
|
|
|
|
static bool vgic_v2_check_base(gpa_t dist_base, gpa_t cpu_base)
|
|
|
|
|
{
|
|
|
|
|
if (dist_base + KVM_VGIC_V2_DIST_SIZE < dist_base)
|
|
|
|
|
return false;
|
|
|
|
|
if (cpu_base + KVM_VGIC_V2_CPU_SIZE < cpu_base)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
if (dist_base + KVM_VGIC_V2_DIST_SIZE <= cpu_base)
|
|
|
|
|
return true;
|
|
|
|
|
if (cpu_base + KVM_VGIC_V2_CPU_SIZE <= dist_base)
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int vgic_v2_map_resources(struct kvm *kvm)
|
|
|
|
|
{
|
|
|
|
|
struct vgic_dist *dist = &kvm->arch.vgic;
|
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
|
|
if (vgic_ready(kvm))
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
|
|
|
|
|
IS_VGIC_ADDR_UNDEF(dist->vgic_cpu_base)) {
|
|
|
|
|
kvm_err("Need to set vgic cpu and dist addresses first\n");
|
|
|
|
|
ret = -ENXIO;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!vgic_v2_check_base(dist->vgic_dist_base, dist->vgic_cpu_base)) {
|
|
|
|
|
kvm_err("VGIC CPU and dist frames overlap\n");
|
|
|
|
|
ret = -EINVAL;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Initialize the vgic if this hasn't already been done on demand by
|
|
|
|
|
* accessing the vgic state from userspace.
|
|
|
|
|
*/
|
|
|
|
|
ret = vgic_init(kvm);
|
|
|
|
|
if (ret) {
|
|
|
|
|
kvm_err("Unable to initialize VGIC dynamic data structures\n");
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ret = vgic_register_dist_iodev(kvm, dist->vgic_dist_base, VGIC_V2);
|
|
|
|
|
if (ret) {
|
|
|
|
|
kvm_err("Unable to register VGIC MMIO regions\n");
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
2016-09-06 02:28:47 -06:00
|
|
|
if (!static_branch_unlikely(&vgic_v2_cpuif_trap)) {
|
|
|
|
|
ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base,
|
|
|
|
|
kvm_vgic_global_state.vcpu_base,
|
2016-07-26 07:12:30 -06:00
|
|
|
KVM_VGIC_V2_CPU_SIZE, true,
|
|
|
|
|
PAGE_S2_DEVICE);
|
2016-09-06 02:28:47 -06:00
|
|
|
if (ret) {
|
|
|
|
|
kvm_err("Unable to remap VGIC CPU to VCPU\n");
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
2015-12-21 07:04:42 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
dist->ready = true;
|
|
|
|
|
|
|
|
|
|
out:
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
arm64: KVM: vgic-v2: Add the GICV emulation infrastructure
In order to efficiently perform the GICV access on behalf of the
guest, we need to be able to avoid going back all the way to
the host kernel.
For this, we introduce a new hook in the world switch code,
conveniently placed just after populating the fault info.
At that point, we only have saved/restored the GP registers,
and we can quickly perform all the required checks (data abort,
translation fault, valid faulting syndrome, not an external
abort, not a PTW).
Coming back from the emulation code, we need to skip the emulated
instruction. This involves an additional bit of save/restore in
order to be able to access the guest's PC (and possibly CPSR if
this is a 32bit guest).
At this stage, no emulation code is provided.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2016-09-06 02:28:45 -06:00
|
|
|
DEFINE_STATIC_KEY_FALSE(vgic_v2_cpuif_trap);
|
|
|
|
|
|
2015-12-01 07:02:35 -07:00
|
|
|
/**
|
2019-08-15 03:56:22 -06:00
|
|
|
* vgic_v2_probe - probe for a VGICv2 compatible interrupt controller
|
|
|
|
|
* @info: pointer to the GIC description
|
2015-12-01 07:02:35 -07:00
|
|
|
*
|
2019-08-15 03:56:22 -06:00
|
|
|
* Returns 0 if the VGICv2 has been probed successfully, returns an error code
|
|
|
|
|
* otherwise
|
2015-12-01 07:02:35 -07:00
|
|
|
*/
|
|
|
|
|
int vgic_v2_probe(const struct gic_kvm_info *info)
|
|
|
|
|
{
|
|
|
|
|
int ret;
|
|
|
|
|
u32 vtr;
|
|
|
|
|
|
|
|
|
|
if (!info->vctrl.start) {
|
|
|
|
|
kvm_err("GICH not present in the firmware table\n");
|
|
|
|
|
return -ENXIO;
|
|
|
|
|
}
|
|
|
|
|
|
2016-09-06 02:28:47 -06:00
|
|
|
if (!PAGE_ALIGNED(info->vcpu.start) ||
|
|
|
|
|
!PAGE_ALIGNED(resource_size(&info->vcpu))) {
|
|
|
|
|
kvm_info("GICV region size/alignment is unsafe, using trapping (reduced performance)\n");
|
2015-12-01 07:02:35 -07:00
|
|
|
|
2017-12-04 09:26:09 -07:00
|
|
|
ret = create_hyp_io_mappings(info->vcpu.start,
|
|
|
|
|
resource_size(&info->vcpu),
|
2017-12-04 09:43:23 -07:00
|
|
|
&kvm_vgic_global_state.vcpu_base_va,
|
|
|
|
|
&kvm_vgic_global_state.vcpu_hyp_va);
|
2016-09-06 02:28:47 -06:00
|
|
|
if (ret) {
|
|
|
|
|
kvm_err("Cannot map GICV into hyp\n");
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static_branch_enable(&vgic_v2_cpuif_trap);
|
2015-12-01 07:02:35 -07:00
|
|
|
}
|
|
|
|
|
|
2017-12-04 09:26:09 -07:00
|
|
|
ret = create_hyp_io_mappings(info->vctrl.start,
|
|
|
|
|
resource_size(&info->vctrl),
|
2017-12-04 09:43:23 -07:00
|
|
|
&kvm_vgic_global_state.vctrl_base,
|
|
|
|
|
&kvm_vgic_global_state.vctrl_hyp);
|
2017-12-04 09:26:09 -07:00
|
|
|
if (ret) {
|
|
|
|
|
kvm_err("Cannot map VCTRL into hyp\n");
|
2016-09-06 02:28:47 -06:00
|
|
|
goto out;
|
2015-12-01 07:02:35 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
vtr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_VTR);
|
|
|
|
|
kvm_vgic_global_state.nr_lr = (vtr & 0x3f) + 1;
|
|
|
|
|
|
2016-09-06 02:28:47 -06:00
|
|
|
ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2);
|
|
|
|
|
if (ret) {
|
|
|
|
|
kvm_err("Cannot register GICv2 KVM device\n");
|
|
|
|
|
goto out;
|
2015-12-01 07:02:35 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
kvm_vgic_global_state.can_emulate_gicv2 = true;
|
|
|
|
|
kvm_vgic_global_state.vcpu_base = info->vcpu.start;
|
|
|
|
|
kvm_vgic_global_state.type = VGIC_V2;
|
|
|
|
|
kvm_vgic_global_state.max_gic_vcpus = VGIC_V2_MAX_CPUS;
|
|
|
|
|
|
2018-03-02 01:16:30 -07:00
|
|
|
kvm_debug("vgic-v2@%llx\n", info->vctrl.start);
|
2015-12-01 07:02:35 -07:00
|
|
|
|
|
|
|
|
return 0;
|
2016-09-06 02:28:47 -06:00
|
|
|
out:
|
|
|
|
|
if (kvm_vgic_global_state.vctrl_base)
|
|
|
|
|
iounmap(kvm_vgic_global_state.vctrl_base);
|
|
|
|
|
if (kvm_vgic_global_state.vcpu_base_va)
|
|
|
|
|
iounmap(kvm_vgic_global_state.vcpu_base_va);
|
|
|
|
|
|
|
|
|
|
return ret;
|
2015-12-01 07:02:35 -07:00
|
|
|
}
|
2016-03-24 04:21:04 -06:00
|
|
|
|
2016-12-22 12:39:10 -07:00
|
|
|
static void save_lrs(struct kvm_vcpu *vcpu, void __iomem *base)
|
|
|
|
|
{
|
|
|
|
|
struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
|
|
|
|
|
u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
|
|
|
|
|
u64 elrsr;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
elrsr = readl_relaxed(base + GICH_ELRSR0);
|
|
|
|
|
if (unlikely(used_lrs > 32))
|
|
|
|
|
elrsr |= ((u64)readl_relaxed(base + GICH_ELRSR1)) << 32;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < used_lrs; i++) {
|
|
|
|
|
if (elrsr & (1UL << i))
|
|
|
|
|
cpu_if->vgic_lr[i] &= ~GICH_LR_STATE;
|
|
|
|
|
else
|
|
|
|
|
cpu_if->vgic_lr[i] = readl_relaxed(base + GICH_LR0 + (i * 4));
|
|
|
|
|
|
|
|
|
|
writel_relaxed(0, base + GICH_LR0 + (i * 4));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void vgic_v2_save_state(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
2017-12-04 09:43:23 -07:00
|
|
|
void __iomem *base = kvm_vgic_global_state.vctrl_base;
|
2016-12-22 12:39:10 -07:00
|
|
|
u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
|
|
|
|
|
|
|
|
|
|
if (!base)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
if (used_lrs) {
|
|
|
|
|
save_lrs(vcpu, base);
|
|
|
|
|
writel_relaxed(0, base + GICH_HCR);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void vgic_v2_restore_state(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
|
2017-12-04 09:43:23 -07:00
|
|
|
void __iomem *base = kvm_vgic_global_state.vctrl_base;
|
2016-12-22 12:39:10 -07:00
|
|
|
u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
if (!base)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
if (used_lrs) {
|
|
|
|
|
writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR);
|
|
|
|
|
for (i = 0; i < used_lrs; i++) {
|
|
|
|
|
writel_relaxed(cpu_if->vgic_lr[i],
|
|
|
|
|
base + GICH_LR0 + (i * 4));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-24 04:21:04 -06:00
|
|
|
void vgic_v2_load(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
|
|
|
|
|
|
2017-12-04 09:43:23 -07:00
|
|
|
writel_relaxed(cpu_if->vgic_vmcr,
|
|
|
|
|
kvm_vgic_global_state.vctrl_base + GICH_VMCR);
|
|
|
|
|
writel_relaxed(cpu_if->vgic_apr,
|
|
|
|
|
kvm_vgic_global_state.vctrl_base + GICH_APR);
|
2016-03-24 04:21:04 -06:00
|
|
|
}
|
|
|
|
|
|
KVM: arm/arm64: Sync ICH_VMCR_EL2 back when about to block
Since commit commit 328e56647944 ("KVM: arm/arm64: vgic: Defer
touching GICH_VMCR to vcpu_load/put"), we leave ICH_VMCR_EL2 (or
its GICv2 equivalent) loaded as long as we can, only syncing it
back when we're scheduled out.
There is a small snag with that though: kvm_vgic_vcpu_pending_irq(),
which is indirectly called from kvm_vcpu_check_block(), needs to
evaluate the guest's view of ICC_PMR_EL1. At the point were we
call kvm_vcpu_check_block(), the vcpu is still loaded, and whatever
changes to PMR is not visible in memory until we do a vcpu_put().
Things go really south if the guest does the following:
mov x0, #0 // or any small value masking interrupts
msr ICC_PMR_EL1, x0
[vcpu preempted, then rescheduled, VMCR sampled]
mov x0, #ff // allow all interrupts
msr ICC_PMR_EL1, x0
wfi // traps to EL2, so samping of VMCR
[interrupt arrives just after WFI]
Here, the hypervisor's view of PMR is zero, while the guest has enabled
its interrupts. kvm_vgic_vcpu_pending_irq() will then say that no
interrupts are pending (despite an interrupt being received) and we'll
block for no reason. If the guest doesn't have a periodic interrupt
firing once it has blocked, it will stay there forever.
To avoid this unfortuante situation, let's resync VMCR from
kvm_arch_vcpu_blocking(), ensuring that a following kvm_vcpu_check_block()
will observe the latest value of PMR.
This has been found by booting an arm64 Linux guest with the pseudo NMI
feature, and thus using interrupt priorities to mask interrupts instead
of the usual PSTATE masking.
Cc: stable@vger.kernel.org # 4.12
Fixes: 328e56647944 ("KVM: arm/arm64: vgic: Defer touching GICH_VMCR to vcpu_load/put")
Signed-off-by: Marc Zyngier <maz@kernel.org>
2019-08-02 03:28:32 -06:00
|
|
|
void vgic_v2_vmcr_sync(struct kvm_vcpu *vcpu)
|
2016-03-24 04:21:04 -06:00
|
|
|
{
|
|
|
|
|
struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
|
|
|
|
|
|
2017-12-04 09:43:23 -07:00
|
|
|
cpu_if->vgic_vmcr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_VMCR);
|
KVM: arm/arm64: Sync ICH_VMCR_EL2 back when about to block
Since commit commit 328e56647944 ("KVM: arm/arm64: vgic: Defer
touching GICH_VMCR to vcpu_load/put"), we leave ICH_VMCR_EL2 (or
its GICv2 equivalent) loaded as long as we can, only syncing it
back when we're scheduled out.
There is a small snag with that though: kvm_vgic_vcpu_pending_irq(),
which is indirectly called from kvm_vcpu_check_block(), needs to
evaluate the guest's view of ICC_PMR_EL1. At the point were we
call kvm_vcpu_check_block(), the vcpu is still loaded, and whatever
changes to PMR is not visible in memory until we do a vcpu_put().
Things go really south if the guest does the following:
mov x0, #0 // or any small value masking interrupts
msr ICC_PMR_EL1, x0
[vcpu preempted, then rescheduled, VMCR sampled]
mov x0, #ff // allow all interrupts
msr ICC_PMR_EL1, x0
wfi // traps to EL2, so samping of VMCR
[interrupt arrives just after WFI]
Here, the hypervisor's view of PMR is zero, while the guest has enabled
its interrupts. kvm_vgic_vcpu_pending_irq() will then say that no
interrupts are pending (despite an interrupt being received) and we'll
block for no reason. If the guest doesn't have a periodic interrupt
firing once it has blocked, it will stay there forever.
To avoid this unfortuante situation, let's resync VMCR from
kvm_arch_vcpu_blocking(), ensuring that a following kvm_vcpu_check_block()
will observe the latest value of PMR.
This has been found by booting an arm64 Linux guest with the pseudo NMI
feature, and thus using interrupt priorities to mask interrupts instead
of the usual PSTATE masking.
Cc: stable@vger.kernel.org # 4.12
Fixes: 328e56647944 ("KVM: arm/arm64: vgic: Defer touching GICH_VMCR to vcpu_load/put")
Signed-off-by: Marc Zyngier <maz@kernel.org>
2019-08-02 03:28:32 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void vgic_v2_put(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
|
|
|
|
|
|
|
|
|
|
vgic_v2_vmcr_sync(vcpu);
|
2017-12-04 09:43:23 -07:00
|
|
|
cpu_if->vgic_apr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_APR);
|
2016-03-24 04:21:04 -06:00
|
|
|
}
|