9fb1b36ca1
We have been observing hangs, both of KVM guest vcpu tasks and more generally, where a process that is woken doesn't properly wake up and continue to run, but instead sticks in TASK_WAKING state. This happens because the update of rq->wake_list in ttwu_queue_remote() is not ordered with the update of ipi_message in smp_muxed_ipi_message_pass(), and the reading of rq->wake_list in scheduler_ipi() is not ordered with the reading of ipi_message in smp_ipi_demux(). Thus it is possible for the IPI receiver not to see the updated rq->wake_list and therefore conclude that there is nothing for it to do. In order to make sure that anything done before smp_send_reschedule() is ordered before anything done in the resulting call to scheduler_ipi(), this adds barriers in smp_muxed_message_pass() and smp_ipi_demux(). The barrier in smp_muxed_message_pass() is a full barrier to ensure that there is a full ordering between the smp_send_reschedule() caller and scheduler_ipi(). In smp_ipi_demux(), we use xchg() rather than xchg_local() because xchg() includes release and acquire barriers. Using xchg() rather than xchg_local() makes sense given that ipi_message is not just accessed locally. This moves the barrier between setting the message and calling the cause_ipi() function into the individual cause_ipi implementations. Most of them -- those that used outb, out_8 or similar -- already had a full barrier because out_8 etc. include a sync before the MMIO store. This adds an explicit barrier in the two remaining cases. These changes made no measurable difference to the speed of IPIs as measured using a simple ping-pong latency test across two CPUs on different cores of a POWER7 machine. The analysis of the reason why processes were not waking up properly is due to Milton Miller. Cc: stable@vger.kernel.org # v3.0+ Reported-by: Milton Miller <miltonm@bga.com> Signed-off-by: Paul Mackerras <paulus@samba.org> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
56 lines
1.2 KiB
C
56 lines
1.2 KiB
C
/*
|
|
* Author: Kumar Gala <galak@kernel.crashing.org>
|
|
*
|
|
* Copyright 2009 Freescale Semiconductor Inc.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License as published by the
|
|
* Free Software Foundation; either version 2 of the License, or (at your
|
|
* option) any later version.
|
|
*/
|
|
|
|
#include <linux/stddef.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/threads.h>
|
|
#include <linux/hardirq.h>
|
|
|
|
#include <asm/dbell.h>
|
|
#include <asm/irq_regs.h>
|
|
|
|
#ifdef CONFIG_SMP
|
|
void doorbell_setup_this_cpu(void)
|
|
{
|
|
unsigned long tag = mfspr(SPRN_PIR) & 0x3fff;
|
|
|
|
smp_muxed_ipi_set_data(smp_processor_id(), tag);
|
|
}
|
|
|
|
void doorbell_cause_ipi(int cpu, unsigned long data)
|
|
{
|
|
/* Order previous accesses vs. msgsnd, which is treated as a store */
|
|
mb();
|
|
ppc_msgsnd(PPC_DBELL, 0, data);
|
|
}
|
|
|
|
void doorbell_exception(struct pt_regs *regs)
|
|
{
|
|
struct pt_regs *old_regs = set_irq_regs(regs);
|
|
|
|
irq_enter();
|
|
|
|
may_hard_irq_enable();
|
|
|
|
smp_ipi_demux();
|
|
|
|
irq_exit();
|
|
set_irq_regs(old_regs);
|
|
}
|
|
#else /* CONFIG_SMP */
|
|
void doorbell_exception(struct pt_regs *regs)
|
|
{
|
|
printk(KERN_WARNING "Received doorbell on non-smp system\n");
|
|
}
|
|
#endif /* CONFIG_SMP */
|
|
|