2008-07-28 12:53:57 -06:00
|
|
|
#include <linux/hardirq.h>
|
|
|
|
|
|
2009-08-20 01:59:09 -06:00
|
|
|
#include <asm/x86_init.h>
|
|
|
|
|
|
2008-07-28 12:53:57 -06:00
|
|
|
#include <xen/interface/xen.h>
|
|
|
|
|
#include <xen/interface/sched.h>
|
|
|
|
|
#include <xen/interface/vcpu.h>
|
2013-12-11 13:36:51 -07:00
|
|
|
#include <xen/features.h>
|
2012-09-14 07:37:32 -06:00
|
|
|
#include <xen/events.h>
|
2008-07-28 12:53:57 -06:00
|
|
|
|
|
|
|
|
#include <asm/xen/hypercall.h>
|
|
|
|
|
#include <asm/xen/hypervisor.h>
|
|
|
|
|
|
|
|
|
|
#include "xen-ops.h"
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Force a proper event-channel callback from Xen after clearing the
|
|
|
|
|
* callback mask. We do this in a very simple manner, by making a call
|
|
|
|
|
* down into Xen. The pending flag will be checked by Xen on return.
|
|
|
|
|
*/
|
|
|
|
|
void xen_force_evtchn_callback(void)
|
|
|
|
|
{
|
|
|
|
|
(void)HYPERVISOR_xen_version(0, NULL);
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-01 16:44:37 -06:00
|
|
|
asmlinkage __visible unsigned long xen_save_fl(void)
|
2008-07-28 12:53:57 -06:00
|
|
|
{
|
|
|
|
|
struct vcpu_info *vcpu;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
2012-01-13 08:53:35 -07:00
|
|
|
vcpu = this_cpu_read(xen_vcpu);
|
2008-07-28 12:53:57 -06:00
|
|
|
|
|
|
|
|
/* flag has opposite sense of mask */
|
|
|
|
|
flags = !vcpu->evtchn_upcall_mask;
|
|
|
|
|
|
|
|
|
|
/* convert to IF type flag
|
|
|
|
|
-0 -> 0x00000000
|
|
|
|
|
-1 -> 0xffffffff
|
|
|
|
|
*/
|
|
|
|
|
return (-flags) & X86_EFLAGS_IF;
|
|
|
|
|
}
|
x86/paravirt: add register-saving thunks to reduce caller register pressure
Impact: Optimization
One of the problems with inserting a pile of C calls where previously
there were none is that the register pressure is greatly increased.
The C calling convention says that the caller must expect a certain
set of registers may be trashed by the callee, and that the callee can
use those registers without restriction. This includes the function
argument registers, and several others.
This patch seeks to alleviate this pressure by introducing wrapper
thunks that will do the register saving/restoring, so that the
callsite doesn't need to worry about it, but the callee function can
be conventional compiler-generated code. In many cases (particularly
performance-sensitive cases) the callee will be in assembler anyway,
and need not use the compiler's calling convention.
Standard calling convention is:
arguments return scratch
x86-32 eax edx ecx eax ?
x86-64 rdi rsi rdx rcx rax r8 r9 r10 r11
The thunk preserves all argument and scratch registers. The return
register is not preserved, and is available as a scratch register for
unwrapped callee code (and of course the return value).
Wrapped function pointers are themselves wrapped in a struct
paravirt_callee_save structure, in order to get some warning from the
compiler when functions with mismatched calling conventions are used.
The most common paravirt ops, both statically and dynamically, are
interrupt enable/disable/save/restore, so handle them first. This is
particularly easy since their calls are handled specially anyway.
XXX Deal with VMI. What's their calling convention?
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2009-01-28 15:35:05 -07:00
|
|
|
PV_CALLEE_SAVE_REGS_THUNK(xen_save_fl);
|
2008-07-28 12:53:57 -06:00
|
|
|
|
2013-10-22 10:07:56 -06:00
|
|
|
__visible void xen_restore_fl(unsigned long flags)
|
2008-07-28 12:53:57 -06:00
|
|
|
{
|
|
|
|
|
struct vcpu_info *vcpu;
|
|
|
|
|
|
|
|
|
|
/* convert from IF type flag */
|
|
|
|
|
flags = !(flags & X86_EFLAGS_IF);
|
|
|
|
|
|
2013-08-15 06:21:04 -06:00
|
|
|
/* See xen_irq_enable() for why preemption must be disabled. */
|
2008-07-28 12:53:57 -06:00
|
|
|
preempt_disable();
|
2012-01-13 08:53:35 -07:00
|
|
|
vcpu = this_cpu_read(xen_vcpu);
|
2008-07-28 12:53:57 -06:00
|
|
|
vcpu->evtchn_upcall_mask = flags;
|
|
|
|
|
|
|
|
|
|
if (flags == 0) {
|
|
|
|
|
barrier(); /* unmask then check (avoid races) */
|
|
|
|
|
if (unlikely(vcpu->evtchn_upcall_pending))
|
|
|
|
|
xen_force_evtchn_callback();
|
2013-08-15 06:21:04 -06:00
|
|
|
preempt_enable();
|
|
|
|
|
} else
|
|
|
|
|
preempt_enable_no_resched();
|
2008-07-28 12:53:57 -06:00
|
|
|
}
|
x86/paravirt: add register-saving thunks to reduce caller register pressure
Impact: Optimization
One of the problems with inserting a pile of C calls where previously
there were none is that the register pressure is greatly increased.
The C calling convention says that the caller must expect a certain
set of registers may be trashed by the callee, and that the callee can
use those registers without restriction. This includes the function
argument registers, and several others.
This patch seeks to alleviate this pressure by introducing wrapper
thunks that will do the register saving/restoring, so that the
callsite doesn't need to worry about it, but the callee function can
be conventional compiler-generated code. In many cases (particularly
performance-sensitive cases) the callee will be in assembler anyway,
and need not use the compiler's calling convention.
Standard calling convention is:
arguments return scratch
x86-32 eax edx ecx eax ?
x86-64 rdi rsi rdx rcx rax r8 r9 r10 r11
The thunk preserves all argument and scratch registers. The return
register is not preserved, and is available as a scratch register for
unwrapped callee code (and of course the return value).
Wrapped function pointers are themselves wrapped in a struct
paravirt_callee_save structure, in order to get some warning from the
compiler when functions with mismatched calling conventions are used.
The most common paravirt ops, both statically and dynamically, are
interrupt enable/disable/save/restore, so handle them first. This is
particularly easy since their calls are handled specially anyway.
XXX Deal with VMI. What's their calling convention?
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2009-01-28 15:35:05 -07:00
|
|
|
PV_CALLEE_SAVE_REGS_THUNK(xen_restore_fl);
|
2008-07-28 12:53:57 -06:00
|
|
|
|
2014-05-01 16:44:37 -06:00
|
|
|
asmlinkage __visible void xen_irq_disable(void)
|
2008-07-28 12:53:57 -06:00
|
|
|
{
|
|
|
|
|
/* There's a one instruction preempt window here. We need to
|
|
|
|
|
make sure we're don't switch CPUs between getting the vcpu
|
|
|
|
|
pointer and updating the mask. */
|
|
|
|
|
preempt_disable();
|
2012-01-13 08:53:35 -07:00
|
|
|
this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
|
2008-07-28 12:53:57 -06:00
|
|
|
preempt_enable_no_resched();
|
|
|
|
|
}
|
x86/paravirt: add register-saving thunks to reduce caller register pressure
Impact: Optimization
One of the problems with inserting a pile of C calls where previously
there were none is that the register pressure is greatly increased.
The C calling convention says that the caller must expect a certain
set of registers may be trashed by the callee, and that the callee can
use those registers without restriction. This includes the function
argument registers, and several others.
This patch seeks to alleviate this pressure by introducing wrapper
thunks that will do the register saving/restoring, so that the
callsite doesn't need to worry about it, but the callee function can
be conventional compiler-generated code. In many cases (particularly
performance-sensitive cases) the callee will be in assembler anyway,
and need not use the compiler's calling convention.
Standard calling convention is:
arguments return scratch
x86-32 eax edx ecx eax ?
x86-64 rdi rsi rdx rcx rax r8 r9 r10 r11
The thunk preserves all argument and scratch registers. The return
register is not preserved, and is available as a scratch register for
unwrapped callee code (and of course the return value).
Wrapped function pointers are themselves wrapped in a struct
paravirt_callee_save structure, in order to get some warning from the
compiler when functions with mismatched calling conventions are used.
The most common paravirt ops, both statically and dynamically, are
interrupt enable/disable/save/restore, so handle them first. This is
particularly easy since their calls are handled specially anyway.
XXX Deal with VMI. What's their calling convention?
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2009-01-28 15:35:05 -07:00
|
|
|
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
|
2008-07-28 12:53:57 -06:00
|
|
|
|
2014-05-01 16:44:37 -06:00
|
|
|
asmlinkage __visible void xen_irq_enable(void)
|
2008-07-28 12:53:57 -06:00
|
|
|
{
|
|
|
|
|
struct vcpu_info *vcpu;
|
|
|
|
|
|
2013-08-15 06:21:04 -06:00
|
|
|
/*
|
|
|
|
|
* We may be preempted as soon as vcpu->evtchn_upcall_mask is
|
|
|
|
|
* cleared, so disable preemption to ensure we check for
|
|
|
|
|
* events on the VCPU we are still running on.
|
|
|
|
|
*/
|
|
|
|
|
preempt_disable();
|
2008-07-28 12:53:57 -06:00
|
|
|
|
2012-01-13 08:53:35 -07:00
|
|
|
vcpu = this_cpu_read(xen_vcpu);
|
2008-07-28 12:53:57 -06:00
|
|
|
vcpu->evtchn_upcall_mask = 0;
|
|
|
|
|
|
|
|
|
|
/* Doesn't matter if we get preempted here, because any
|
|
|
|
|
pending event will get dealt with anyway. */
|
|
|
|
|
|
|
|
|
|
barrier(); /* unmask then check (avoid races) */
|
|
|
|
|
if (unlikely(vcpu->evtchn_upcall_pending))
|
|
|
|
|
xen_force_evtchn_callback();
|
2013-08-15 06:21:04 -06:00
|
|
|
|
|
|
|
|
preempt_enable();
|
2008-07-28 12:53:57 -06:00
|
|
|
}
|
x86/paravirt: add register-saving thunks to reduce caller register pressure
Impact: Optimization
One of the problems with inserting a pile of C calls where previously
there were none is that the register pressure is greatly increased.
The C calling convention says that the caller must expect a certain
set of registers may be trashed by the callee, and that the callee can
use those registers without restriction. This includes the function
argument registers, and several others.
This patch seeks to alleviate this pressure by introducing wrapper
thunks that will do the register saving/restoring, so that the
callsite doesn't need to worry about it, but the callee function can
be conventional compiler-generated code. In many cases (particularly
performance-sensitive cases) the callee will be in assembler anyway,
and need not use the compiler's calling convention.
Standard calling convention is:
arguments return scratch
x86-32 eax edx ecx eax ?
x86-64 rdi rsi rdx rcx rax r8 r9 r10 r11
The thunk preserves all argument and scratch registers. The return
register is not preserved, and is available as a scratch register for
unwrapped callee code (and of course the return value).
Wrapped function pointers are themselves wrapped in a struct
paravirt_callee_save structure, in order to get some warning from the
compiler when functions with mismatched calling conventions are used.
The most common paravirt ops, both statically and dynamically, are
interrupt enable/disable/save/restore, so handle them first. This is
particularly easy since their calls are handled specially anyway.
XXX Deal with VMI. What's their calling convention?
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2009-01-28 15:35:05 -07:00
|
|
|
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_enable);
|
2008-07-28 12:53:57 -06:00
|
|
|
|
|
|
|
|
static void xen_safe_halt(void)
|
|
|
|
|
{
|
|
|
|
|
/* Blocking includes an implicit local_irq_enable(). */
|
|
|
|
|
if (HYPERVISOR_sched_op(SCHEDOP_block, NULL) != 0)
|
|
|
|
|
BUG();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void xen_halt(void)
|
|
|
|
|
{
|
|
|
|
|
if (irqs_disabled())
|
2016-06-30 09:56:38 -06:00
|
|
|
HYPERVISOR_vcpu_op(VCPUOP_down,
|
|
|
|
|
xen_vcpu_nr(smp_processor_id()), NULL);
|
2008-07-28 12:53:57 -06:00
|
|
|
else
|
|
|
|
|
xen_safe_halt();
|
|
|
|
|
}
|
|
|
|
|
|
2011-05-04 12:16:07 -06:00
|
|
|
static const struct pv_irq_ops xen_irq_ops __initconst = {
|
x86/paravirt: add register-saving thunks to reduce caller register pressure
Impact: Optimization
One of the problems with inserting a pile of C calls where previously
there were none is that the register pressure is greatly increased.
The C calling convention says that the caller must expect a certain
set of registers may be trashed by the callee, and that the callee can
use those registers without restriction. This includes the function
argument registers, and several others.
This patch seeks to alleviate this pressure by introducing wrapper
thunks that will do the register saving/restoring, so that the
callsite doesn't need to worry about it, but the callee function can
be conventional compiler-generated code. In many cases (particularly
performance-sensitive cases) the callee will be in assembler anyway,
and need not use the compiler's calling convention.
Standard calling convention is:
arguments return scratch
x86-32 eax edx ecx eax ?
x86-64 rdi rsi rdx rcx rax r8 r9 r10 r11
The thunk preserves all argument and scratch registers. The return
register is not preserved, and is available as a scratch register for
unwrapped callee code (and of course the return value).
Wrapped function pointers are themselves wrapped in a struct
paravirt_callee_save structure, in order to get some warning from the
compiler when functions with mismatched calling conventions are used.
The most common paravirt ops, both statically and dynamically, are
interrupt enable/disable/save/restore, so handle them first. This is
particularly easy since their calls are handled specially anyway.
XXX Deal with VMI. What's their calling convention?
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2009-01-28 15:35:05 -07:00
|
|
|
.save_fl = PV_CALLEE_SAVE(xen_save_fl),
|
|
|
|
|
.restore_fl = PV_CALLEE_SAVE(xen_restore_fl),
|
|
|
|
|
.irq_disable = PV_CALLEE_SAVE(xen_irq_disable),
|
|
|
|
|
.irq_enable = PV_CALLEE_SAVE(xen_irq_enable),
|
|
|
|
|
|
2008-07-28 12:53:57 -06:00
|
|
|
.safe_halt = xen_safe_halt,
|
|
|
|
|
.halt = xen_halt,
|
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
|
.adjust_exception_frame = xen_adjust_exception_frame,
|
|
|
|
|
#endif
|
|
|
|
|
};
|
|
|
|
|
|
2011-01-08 21:00:36 -07:00
|
|
|
void __init xen_init_irq_ops(void)
|
2008-07-28 12:53:57 -06:00
|
|
|
{
|
2013-12-11 13:36:51 -07:00
|
|
|
/* For PVH we use default pv_irq_ops settings. */
|
|
|
|
|
if (!xen_feature(XENFEAT_hvm_callback_vector))
|
|
|
|
|
pv_irq_ops = xen_irq_ops;
|
2009-08-20 01:59:09 -06:00
|
|
|
x86_init.irqs.intr_init = xen_init_IRQ;
|
2008-07-28 12:53:57 -06:00
|
|
|
}
|