2008-06-26 03:21:34 -06:00
|
|
|
/*
|
|
|
|
* Generic helpers for smp ipi calls
|
|
|
|
*
|
|
|
|
* (C) Jens Axboe <jens.axboe@oracle.com> 2008
|
|
|
|
*/
|
2016-10-25 23:37:53 -06:00
|
|
|
|
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
|
2014-05-07 17:37:48 -06:00
|
|
|
#include <linux/irq_work.h>
|
2008-06-26 03:21:34 -06:00
|
|
|
#include <linux/rcupdate.h>
|
2008-07-15 15:02:33 -06:00
|
|
|
#include <linux/rculist.h>
|
2009-03-13 03:47:34 -06:00
|
|
|
#include <linux/kernel.h>
|
2011-05-23 12:51:41 -06:00
|
|
|
#include <linux/export.h>
|
2009-02-25 08:52:11 -07:00
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/init.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 02:04:11 -06:00
|
|
|
#include <linux/gfp.h>
|
2008-06-26 03:21:34 -06:00
|
|
|
#include <linux/smp.h>
|
2009-02-25 05:59:47 -07:00
|
|
|
#include <linux/cpu.h>
|
2014-09-04 01:17:54 -06:00
|
|
|
#include <linux/sched.h>
|
2017-02-01 08:36:40 -07:00
|
|
|
#include <linux/sched/idle.h>
|
2016-08-29 00:48:43 -06:00
|
|
|
#include <linux/hypervisor.h>
|
2008-06-26 03:21:34 -06:00
|
|
|
|
2012-04-20 18:08:50 -06:00
|
|
|
#include "smpboot.h"
|
|
|
|
|
2008-06-26 03:21:34 -06:00
|
|
|
enum {
|
2009-02-25 05:59:48 -07:00
|
|
|
CSD_FLAG_LOCK = 0x01,
|
2015-02-11 13:42:10 -07:00
|
|
|
CSD_FLAG_SYNCHRONOUS = 0x02,
|
2008-06-26 03:21:34 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
struct call_function_data {
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
call_single_data_t __percpu *csd;
|
2009-02-25 08:52:11 -07:00
|
|
|
cpumask_var_t cpumask;
|
2017-05-19 01:53:31 -06:00
|
|
|
cpumask_var_t cpumask_ipi;
|
2008-06-26 03:21:34 -06:00
|
|
|
};
|
|
|
|
|
2010-01-17 19:00:51 -07:00
|
|
|
static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
|
|
|
|
|
2014-01-30 16:45:47 -07:00
|
|
|
static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
|
2009-02-25 05:59:47 -07:00
|
|
|
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:02 -06:00
|
|
|
static void flush_smp_call_function_queue(bool warn_cpu_offline);
|
|
|
|
|
2016-07-13 11:17:01 -06:00
|
|
|
int smpcfd_prepare_cpu(unsigned int cpu)
|
2009-02-25 05:59:47 -07:00
|
|
|
{
|
|
|
|
struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
|
|
|
|
|
2016-07-13 11:17:01 -06:00
|
|
|
if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
|
|
|
|
cpu_to_node(cpu)))
|
|
|
|
return -ENOMEM;
|
2017-05-19 01:53:31 -06:00
|
|
|
if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL,
|
|
|
|
cpu_to_node(cpu))) {
|
|
|
|
free_cpumask_var(cfd->cpumask);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
cfd->csd = alloc_percpu(call_single_data_t);
|
2016-07-13 11:17:01 -06:00
|
|
|
if (!cfd->csd) {
|
2009-02-25 05:59:47 -07:00
|
|
|
free_cpumask_var(cfd->cpumask);
|
2017-05-19 01:53:31 -06:00
|
|
|
free_cpumask_var(cfd->cpumask_ipi);
|
2016-07-13 11:17:01 -06:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
2009-02-25 05:59:47 -07:00
|
|
|
}
|
|
|
|
|
2016-07-13 11:17:01 -06:00
|
|
|
int smpcfd_dead_cpu(unsigned int cpu)
|
|
|
|
{
|
|
|
|
struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
|
|
|
|
|
|
|
|
free_cpumask_var(cfd->cpumask);
|
2017-05-19 01:53:31 -06:00
|
|
|
free_cpumask_var(cfd->cpumask_ipi);
|
2016-07-13 11:17:01 -06:00
|
|
|
free_percpu(cfd->csd);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int smpcfd_dying_cpu(unsigned int cpu)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The IPIs for the smp-call-function callbacks queued by other
|
|
|
|
* CPUs might arrive late, either due to hardware latencies or
|
|
|
|
* because this CPU disabled interrupts (inside stop-machine)
|
|
|
|
* before the IPIs were sent. So flush out any pending callbacks
|
|
|
|
* explicitly (without waiting for the IPIs to arrive), to
|
|
|
|
* ensure that the outgoing CPU doesn't go offline with work
|
|
|
|
* still pending.
|
|
|
|
*/
|
|
|
|
flush_smp_call_function_queue(false);
|
|
|
|
return 0;
|
|
|
|
}
|
2009-02-25 05:59:47 -07:00
|
|
|
|
2011-03-29 10:35:04 -06:00
|
|
|
void __init call_function_init(void)
|
2008-06-26 03:21:34 -06:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2014-01-30 16:45:47 -07:00
|
|
|
for_each_possible_cpu(i)
|
|
|
|
init_llist_head(&per_cpu(call_single_queue, i));
|
2009-02-25 05:59:47 -07:00
|
|
|
|
2016-07-13 11:17:01 -06:00
|
|
|
smpcfd_prepare_cpu(smp_processor_id());
|
2008-06-26 03:21:34 -06:00
|
|
|
}
|
|
|
|
|
2009-02-25 05:59:47 -07:00
|
|
|
/*
|
|
|
|
* csd_lock/csd_unlock used to serialize access to per-cpu csd resources
|
|
|
|
*
|
2009-02-25 08:52:11 -07:00
|
|
|
* For non-synchronous ipi calls the csd can still be in use by the
|
|
|
|
* previous function call. For multi-cpu calls its even more interesting
|
|
|
|
* as we'll have to ensure no other cpu is observing our csd.
|
2009-02-25 05:59:47 -07:00
|
|
|
*/
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
static __always_inline void csd_lock_wait(call_single_data_t *csd)
|
2009-02-25 05:59:47 -07:00
|
|
|
{
|
2016-04-04 02:57:12 -06:00
|
|
|
smp_cond_load_acquire(&csd->flags, !(VAL & CSD_FLAG_LOCK));
|
2009-02-25 05:59:48 -07:00
|
|
|
}
|
|
|
|
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
static __always_inline void csd_lock(call_single_data_t *csd)
|
2009-02-25 05:59:48 -07:00
|
|
|
{
|
2013-04-30 16:27:28 -06:00
|
|
|
csd_lock_wait(csd);
|
|
|
|
csd->flags |= CSD_FLAG_LOCK;
|
2009-02-25 05:59:47 -07:00
|
|
|
|
|
|
|
/*
|
2009-02-25 08:52:11 -07:00
|
|
|
* prevent CPU from reordering the above assignment
|
|
|
|
* to ->flags with any subsequent assignments to other
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
* fields of the specified call_single_data_t structure:
|
2009-02-25 05:59:47 -07:00
|
|
|
*/
|
2015-02-11 13:42:10 -07:00
|
|
|
smp_wmb();
|
2009-02-25 05:59:47 -07:00
|
|
|
}
|
|
|
|
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
static __always_inline void csd_unlock(call_single_data_t *csd)
|
2009-02-25 05:59:47 -07:00
|
|
|
{
|
2015-02-11 13:42:10 -07:00
|
|
|
WARN_ON(!(csd->flags & CSD_FLAG_LOCK));
|
2009-02-25 08:52:11 -07:00
|
|
|
|
2009-02-25 05:59:47 -07:00
|
|
|
/*
|
2009-02-25 08:52:11 -07:00
|
|
|
* ensure we're all done before releasing data:
|
2009-02-25 05:59:47 -07:00
|
|
|
*/
|
2015-02-11 13:42:10 -07:00
|
|
|
smp_store_release(&csd->flags, 0);
|
2008-06-26 03:21:34 -06:00
|
|
|
}
|
|
|
|
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
|
2014-02-24 08:39:58 -07:00
|
|
|
|
2008-06-26 03:21:34 -06:00
|
|
|
/*
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
* Insert a previously allocated call_single_data_t element
|
2009-02-25 08:52:11 -07:00
|
|
|
* for execution on the given CPU. data must already have
|
|
|
|
* ->func, ->info, and ->flags set.
|
2008-06-26 03:21:34 -06:00
|
|
|
*/
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
static int generic_exec_single(int cpu, call_single_data_t *csd,
|
2015-02-11 13:42:10 -07:00
|
|
|
smp_call_func_t func, void *info)
|
2008-06-26 03:21:34 -06:00
|
|
|
{
|
2014-02-24 08:39:58 -07:00
|
|
|
if (cpu == smp_processor_id()) {
|
2015-02-11 13:42:10 -07:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We can unlock early even for the synchronous on-stack case,
|
|
|
|
* since we're doing this from the same CPU..
|
|
|
|
*/
|
|
|
|
csd_unlock(csd);
|
2014-02-24 08:39:58 -07:00
|
|
|
local_irq_save(flags);
|
|
|
|
func(info);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2015-04-19 02:56:03 -06:00
|
|
|
if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu)) {
|
|
|
|
csd_unlock(csd);
|
2014-02-24 08:39:58 -07:00
|
|
|
return -ENXIO;
|
2015-04-19 02:56:03 -06:00
|
|
|
}
|
2014-02-24 08:39:58 -07:00
|
|
|
|
|
|
|
csd->func = func;
|
|
|
|
csd->info = info;
|
|
|
|
|
2008-10-30 11:28:41 -06:00
|
|
|
/*
|
generic IPI: simplify barriers and locking
Simplify the barriers in generic remote function call interrupt
code.
Firstly, just unconditionally take the lock and check the list
in the generic_call_function_single_interrupt IPI handler. As
we've just taken an IPI here, the chances are fairly high that
there will be work on the list for us, so do the locking
unconditionally. This removes the tricky lockless list_empty
check and dubious barriers. The change looks bigger than it is
because it is just removing an outer loop.
Secondly, clarify architecture specific IPI locking rules.
Generic code has no tools to impose any sane ordering on IPIs if
they go outside normal cache coherency, ergo the arch code must
make them appear to obey cache coherency as a "memory operation"
to initiate an IPI, and a "memory operation" to receive one.
This way at least they can be reasoned about in generic code,
and smp_mb used to provide ordering.
The combination of these two changes means that explict barriers
can be taken out of queue handling for the single case -- shared
data is explicitly locked, and ipi ordering must conform to
that, so no barriers needed. An extra barrier is needed in the
many handler, so as to ensure we load the list element after the
IPI is received.
Does any architecture actually *need* these barriers? For the
initiator I could see it, but for the handler I would be
surprised. So the other thing we could do for simplicity is just
to require that, rather than just matching with cache coherency,
we just require a full barrier before generating an IPI, and
after receiving an IPI. In which case, the smp_mb()s can go
away. But just for now, we'll be on the safe side and use the
barriers (they're in the slow case anyway).
Signed-off-by: Nick Piggin <npiggin@suse.de>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: linux-arch@vger.kernel.org
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jens Axboe <jens.axboe@oracle.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Suresh Siddha <suresh.b.siddha@intel.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-24 22:22:45 -07:00
|
|
|
* The list addition should be visible before sending the IPI
|
|
|
|
* handler locks the list to pull the entry off it because of
|
|
|
|
* normal cache coherency rules implied by spinlocks.
|
|
|
|
*
|
|
|
|
* If IPIs can go out of order to the cache coherency protocol
|
|
|
|
* in an architecture, sufficient synchronisation should be added
|
|
|
|
* to arch code to make it appear to obey cache coherency WRT
|
2009-02-25 08:52:11 -07:00
|
|
|
* locking and barrier primitives. Generic code isn't really
|
|
|
|
* equipped to do the right thing...
|
2008-10-30 11:28:41 -06:00
|
|
|
*/
|
2014-01-30 16:45:47 -07:00
|
|
|
if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
|
2008-06-26 03:21:34 -06:00
|
|
|
arch_send_call_function_single_ipi(cpu);
|
|
|
|
|
2014-02-24 08:39:58 -07:00
|
|
|
return 0;
|
2008-06-26 03:21:34 -06:00
|
|
|
}
|
|
|
|
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:02 -06:00
|
|
|
/**
|
|
|
|
* generic_smp_call_function_single_interrupt - Execute SMP IPI callbacks
|
|
|
|
*
|
|
|
|
* Invoked by arch to handle an IPI for call function single.
|
|
|
|
* Must be called with interrupts disabled.
|
2008-06-26 03:21:34 -06:00
|
|
|
*/
|
|
|
|
void generic_smp_call_function_single_interrupt(void)
|
|
|
|
{
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:02 -06:00
|
|
|
flush_smp_call_function_queue(true);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* flush_smp_call_function_queue - Flush pending smp-call-function callbacks
|
|
|
|
*
|
|
|
|
* @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
|
|
|
|
* offline CPU. Skip this check if set to 'false'.
|
|
|
|
*
|
|
|
|
* Flush any pending smp-call-function callbacks queued on this CPU. This is
|
|
|
|
* invoked by the generic IPI handler, as well as by a CPU about to go offline,
|
|
|
|
* to ensure that all pending IPI callbacks are run before it goes completely
|
|
|
|
* offline.
|
|
|
|
*
|
|
|
|
* Loop through the call_single_queue and run all the queued callbacks.
|
|
|
|
* Must be called with interrupts disabled.
|
|
|
|
*/
|
|
|
|
static void flush_smp_call_function_queue(bool warn_cpu_offline)
|
|
|
|
{
|
|
|
|
struct llist_head *head;
|
2014-02-24 08:39:55 -07:00
|
|
|
struct llist_node *entry;
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
call_single_data_t *csd, *csd_next;
|
smp: print more useful debug info upon receiving IPI on an offline CPU
There is a longstanding problem related to CPU hotplug which causes IPIs
to be delivered to offline CPUs, and the smp-call-function IPI handler
code prints out a warning whenever this is detected. Every once in a
while this (usually harmless) warning gets reported on LKML, but so far
it has not been completely fixed. Usually the solution involves finding
out the IPI sender and fixing it by adding appropriate synchronization
with CPU hotplug.
However, while going through one such internal bug reports, I found that
there is a significant bug in the receiver side itself (more
specifically, in stop-machine) that can lead to this problem even when
the sender code is perfectly fine. This patchset fixes that
synchronization problem in the CPU hotplug stop-machine code.
Patch 1 adds some additional debug code to the smp-call-function
framework, to help debug such issues easily.
Patch 2 modifies the stop-machine code to ensure that any IPIs that were
sent while the target CPU was online, would be noticed and handled by
that CPU without fail before it goes offline. Thus, this avoids
scenarios where IPIs are received on offline CPUs (as long as the sender
uses proper hotplug synchronization).
In fact, I debugged the problem by using Patch 1, and found that the
payload of the IPI was always the block layer's trigger_softirq()
function. But I was not able to find anything wrong with the block
layer code. That's when I started looking at the stop-machine code and
realized that there is a race-window which makes the IPI _receiver_ the
culprit, not the sender. Patch 2 fixes that race and hence this should
put an end to most of the hard-to-debug IPI-to-offline-CPU issues.
This patch (of 2):
Today the smp-call-function code just prints a warning if we get an IPI
on an offline CPU. This info is sufficient to let us know that
something went wrong, but often it is very hard to debug exactly who
sent the IPI and why, from this info alone.
In most cases, we get the warning about the IPI to an offline CPU,
immediately after the CPU going offline comes out of the stop-machine
phase and reenables interrupts. Since all online CPUs participate in
stop-machine, the information regarding the sender of the IPI is already
lost by the time we exit the stop-machine loop. So even if we dump the
stack on each CPU at this point, we won't find anything useful since all
of them will show the stack-trace of the stopper thread. So we need a
better way to figure out who sent the IPI and why.
To achieve this, when we detect an IPI targeted to an offline CPU, loop
through the call-single-data linked list and print out the payload
(i.e., the name of the function which was supposed to be executed by the
target CPU). This would give us an insight as to who might have sent
the IPI and help us debug this further.
[akpm@linux-foundation.org: correctly suppress warning output on second and later occurrences]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-06 15:37:05 -06:00
|
|
|
static bool warned;
|
|
|
|
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:02 -06:00
|
|
|
WARN_ON(!irqs_disabled());
|
|
|
|
|
2014-08-17 11:30:24 -06:00
|
|
|
head = this_cpu_ptr(&call_single_queue);
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:02 -06:00
|
|
|
entry = llist_del_all(head);
|
smp: print more useful debug info upon receiving IPI on an offline CPU
There is a longstanding problem related to CPU hotplug which causes IPIs
to be delivered to offline CPUs, and the smp-call-function IPI handler
code prints out a warning whenever this is detected. Every once in a
while this (usually harmless) warning gets reported on LKML, but so far
it has not been completely fixed. Usually the solution involves finding
out the IPI sender and fixing it by adding appropriate synchronization
with CPU hotplug.
However, while going through one such internal bug reports, I found that
there is a significant bug in the receiver side itself (more
specifically, in stop-machine) that can lead to this problem even when
the sender code is perfectly fine. This patchset fixes that
synchronization problem in the CPU hotplug stop-machine code.
Patch 1 adds some additional debug code to the smp-call-function
framework, to help debug such issues easily.
Patch 2 modifies the stop-machine code to ensure that any IPIs that were
sent while the target CPU was online, would be noticed and handled by
that CPU without fail before it goes offline. Thus, this avoids
scenarios where IPIs are received on offline CPUs (as long as the sender
uses proper hotplug synchronization).
In fact, I debugged the problem by using Patch 1, and found that the
payload of the IPI was always the block layer's trigger_softirq()
function. But I was not able to find anything wrong with the block
layer code. That's when I started looking at the stop-machine code and
realized that there is a race-window which makes the IPI _receiver_ the
culprit, not the sender. Patch 2 fixes that race and hence this should
put an end to most of the hard-to-debug IPI-to-offline-CPU issues.
This patch (of 2):
Today the smp-call-function code just prints a warning if we get an IPI
on an offline CPU. This info is sufficient to let us know that
something went wrong, but often it is very hard to debug exactly who
sent the IPI and why, from this info alone.
In most cases, we get the warning about the IPI to an offline CPU,
immediately after the CPU going offline comes out of the stop-machine
phase and reenables interrupts. Since all online CPUs participate in
stop-machine, the information regarding the sender of the IPI is already
lost by the time we exit the stop-machine loop. So even if we dump the
stack on each CPU at this point, we won't find anything useful since all
of them will show the stack-trace of the stopper thread. So we need a
better way to figure out who sent the IPI and why.
To achieve this, when we detect an IPI targeted to an offline CPU, loop
through the call-single-data linked list and print out the payload
(i.e., the name of the function which was supposed to be executed by the
target CPU). This would give us an insight as to who might have sent
the IPI and help us debug this further.
[akpm@linux-foundation.org: correctly suppress warning output on second and later occurrences]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-06 15:37:05 -06:00
|
|
|
entry = llist_reverse_order(entry);
|
2008-06-26 03:21:34 -06:00
|
|
|
|
CPU hotplug, smp: flush any pending IPI callbacks before CPU offline
There is a race between the CPU offline code (within stop-machine) and
the smp-call-function code, which can lead to getting IPIs on the
outgoing CPU, *after* it has gone offline.
Specifically, this can happen when using
smp_call_function_single_async() to send the IPI, since this API allows
sending asynchronous IPIs from IRQ disabled contexts. The exact race
condition is described below.
During CPU offline, in stop-machine, we don't enforce any rule in the
_DISABLE_IRQ stage, regarding the order in which the outgoing CPU and
the other CPUs disable their local interrupts. Due to this, we can
encounter a situation in which an IPI is sent by one of the other CPUs
to the outgoing CPU (while it is *still* online), but the outgoing CPU
ends up noticing it only *after* it has gone offline.
CPU 1 CPU 2
(Online CPU) (CPU going offline)
Enter _PREPARE stage Enter _PREPARE stage
Enter _DISABLE_IRQ stage
=
Got a device interrupt, and | Didn't notice the IPI
the interrupt handler sent an | since interrupts were
IPI to CPU 2 using | disabled on this CPU.
smp_call_function_single_async() |
=
Enter _DISABLE_IRQ stage
Enter _RUN stage Enter _RUN stage
=
Busy loop with interrupts | Invoke take_cpu_down()
disabled. | and take CPU 2 offline
=
Enter _EXIT stage Enter _EXIT stage
Re-enable interrupts Re-enable interrupts
The pending IPI is noted
immediately, but alas,
the CPU is offline at
this point.
This of course, makes the smp-call-function IPI handler code running on
CPU 2 unhappy and it complains about "receiving an IPI on an offline
CPU".
One real example of the scenario on CPU 1 is the block layer's
complete-request call-path:
__blk_complete_request() [interrupt-handler]
raise_blk_irq()
smp_call_function_single_async()
However, if we look closely, the block layer does check that the target
CPU is online before firing the IPI. So in this case, it is actually
the unfortunate ordering/timing of events in the stop-machine phase that
leads to receiving IPIs after the target CPU has gone offline.
In reality, getting a late IPI on an offline CPU is not too bad by
itself (this can happen even due to hardware latencies in IPI
send-receive). It is a bug only if the target CPU really went offline
without executing all the callbacks queued on its list. (Note that a
CPU is free to execute its pending smp-call-function callbacks in a
batch, without waiting for the corresponding IPIs to arrive for each one
of those callbacks).
So, fixing this issue can be broken up into two parts:
1. Ensure that a CPU goes offline only after executing all the
callbacks queued on it.
2. Modify the warning condition in the smp-call-function IPI handler
code such that it warns only if an offline CPU got an IPI *and* that
CPU had gone offline with callbacks still pending in its queue.
Achieving part 1 is straight-forward - just flush (execute) all the
queued callbacks on the outgoing CPU in the CPU_DYING stage[1],
including those callbacks for which the source CPU's IPIs might not have
been received on the outgoing CPU yet. Once we do this, an IPI that
arrives late on the CPU going offline (either due to the race mentioned
above, or due to hardware latencies) will be completely harmless, since
the outgoing CPU would have executed all the queued callbacks before
going offline.
Overall, this fix (parts 1 and 2 put together) additionally guarantees
that we will see a warning only when the *IPI-sender code* is buggy -
that is, if it queues the callback _after_ the target CPU has gone
offline.
[1]. The CPU_DYING part needs a little more explanation: by the time we
execute the CPU_DYING notifier callbacks, the CPU would have already
been marked offline. But we want to flush out the pending callbacks at
this stage, ignoring the fact that the CPU is offline. So restructure
the IPI handler code so that we can by-pass the "is-cpu-offline?" check
in this particular case. (Of course, the right solution here is to fix
CPU hotplug to mark the CPU offline _after_ invoking the CPU_DYING
notifiers, but this requires a lot of audit to ensure that this change
doesn't break any existing code; hence lets go with the solution
proposed above until that is done).
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Suggested-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Cc: Rik van Riel <riel@redhat.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sachin Kamat <sachin.kamat@samsung.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-23 14:22:02 -06:00
|
|
|
/* There shouldn't be any pending callbacks on an offline CPU. */
|
|
|
|
if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
|
|
|
|
!warned && !llist_empty(head))) {
|
smp: print more useful debug info upon receiving IPI on an offline CPU
There is a longstanding problem related to CPU hotplug which causes IPIs
to be delivered to offline CPUs, and the smp-call-function IPI handler
code prints out a warning whenever this is detected. Every once in a
while this (usually harmless) warning gets reported on LKML, but so far
it has not been completely fixed. Usually the solution involves finding
out the IPI sender and fixing it by adding appropriate synchronization
with CPU hotplug.
However, while going through one such internal bug reports, I found that
there is a significant bug in the receiver side itself (more
specifically, in stop-machine) that can lead to this problem even when
the sender code is perfectly fine. This patchset fixes that
synchronization problem in the CPU hotplug stop-machine code.
Patch 1 adds some additional debug code to the smp-call-function
framework, to help debug such issues easily.
Patch 2 modifies the stop-machine code to ensure that any IPIs that were
sent while the target CPU was online, would be noticed and handled by
that CPU without fail before it goes offline. Thus, this avoids
scenarios where IPIs are received on offline CPUs (as long as the sender
uses proper hotplug synchronization).
In fact, I debugged the problem by using Patch 1, and found that the
payload of the IPI was always the block layer's trigger_softirq()
function. But I was not able to find anything wrong with the block
layer code. That's when I started looking at the stop-machine code and
realized that there is a race-window which makes the IPI _receiver_ the
culprit, not the sender. Patch 2 fixes that race and hence this should
put an end to most of the hard-to-debug IPI-to-offline-CPU issues.
This patch (of 2):
Today the smp-call-function code just prints a warning if we get an IPI
on an offline CPU. This info is sufficient to let us know that
something went wrong, but often it is very hard to debug exactly who
sent the IPI and why, from this info alone.
In most cases, we get the warning about the IPI to an offline CPU,
immediately after the CPU going offline comes out of the stop-machine
phase and reenables interrupts. Since all online CPUs participate in
stop-machine, the information regarding the sender of the IPI is already
lost by the time we exit the stop-machine loop. So even if we dump the
stack on each CPU at this point, we won't find anything useful since all
of them will show the stack-trace of the stopper thread. So we need a
better way to figure out who sent the IPI and why.
To achieve this, when we detect an IPI targeted to an offline CPU, loop
through the call-single-data linked list and print out the payload
(i.e., the name of the function which was supposed to be executed by the
target CPU). This would give us an insight as to who might have sent
the IPI and help us debug this further.
[akpm@linux-foundation.org: correctly suppress warning output on second and later occurrences]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Mike Galbraith <mgalbraith@suse.de>
Cc: Gautham R Shenoy <ego@linux.vnet.ibm.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Rafael J. Wysocki <rjw@rjwysocki.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-06-06 15:37:05 -06:00
|
|
|
warned = true;
|
|
|
|
WARN(1, "IPI on offline CPU %d\n", smp_processor_id());
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't have to use the _safe() variant here
|
|
|
|
* because we are not invoking the IPI handlers yet.
|
|
|
|
*/
|
|
|
|
llist_for_each_entry(csd, entry, llist)
|
|
|
|
pr_warn("IPI callback %pS sent to offline CPU\n",
|
|
|
|
csd->func);
|
|
|
|
}
|
2008-06-26 03:21:34 -06:00
|
|
|
|
2014-02-24 08:39:55 -07:00
|
|
|
llist_for_each_entry_safe(csd, csd_next, entry, llist) {
|
2015-02-11 13:42:10 -07:00
|
|
|
smp_call_func_t func = csd->func;
|
|
|
|
void *info = csd->info;
|
|
|
|
|
|
|
|
/* Do we wait until *after* callback? */
|
|
|
|
if (csd->flags & CSD_FLAG_SYNCHRONOUS) {
|
|
|
|
func(info);
|
|
|
|
csd_unlock(csd);
|
|
|
|
} else {
|
|
|
|
csd_unlock(csd);
|
|
|
|
func(info);
|
|
|
|
}
|
2008-06-26 03:21:34 -06:00
|
|
|
}
|
2014-05-07 17:37:48 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Handle irq works queued remotely by irq_work_queue_on().
|
|
|
|
* Smp functions above are typically synchronous so they
|
|
|
|
* better run first since some other CPUs may be busy waiting
|
|
|
|
* for them.
|
|
|
|
*/
|
|
|
|
irq_work_run();
|
2008-06-26 03:21:34 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* smp_call_function_single - Run a function on a specific CPU
|
|
|
|
* @func: The function to run. This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to the function.
|
|
|
|
* @wait: If true, wait until function has completed on other CPUs.
|
|
|
|
*
|
2009-10-22 05:19:34 -06:00
|
|
|
* Returns 0 on success, else a negative status code.
|
2008-06-26 03:21:34 -06:00
|
|
|
*/
|
2010-10-27 10:28:36 -06:00
|
|
|
int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
|
2008-06-06 03:18:06 -06:00
|
|
|
int wait)
|
2008-06-26 03:21:34 -06:00
|
|
|
{
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
call_single_data_t *csd;
|
|
|
|
call_single_data_t csd_stack = {
|
|
|
|
.flags = CSD_FLAG_LOCK | CSD_FLAG_SYNCHRONOUS,
|
|
|
|
};
|
2009-02-25 08:52:11 -07:00
|
|
|
int this_cpu;
|
2014-02-24 08:39:58 -07:00
|
|
|
int err;
|
2008-06-26 03:21:34 -06:00
|
|
|
|
2009-02-25 08:52:11 -07:00
|
|
|
/*
|
|
|
|
* prevent preemption and reschedule on another processor,
|
|
|
|
* as well as CPU removal
|
|
|
|
*/
|
|
|
|
this_cpu = get_cpu();
|
|
|
|
|
2009-08-19 19:05:35 -06:00
|
|
|
/*
|
|
|
|
* Can deadlock when called with interrupts disabled.
|
|
|
|
* We allow cpu's that are not yet online though, as no one else can
|
|
|
|
* send smp call function interrupt to this cpu and as such deadlocks
|
|
|
|
* can't happen.
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
|
|
|
|
&& !oops_in_progress);
|
2008-06-26 03:21:34 -06:00
|
|
|
|
2015-02-11 13:42:10 -07:00
|
|
|
csd = &csd_stack;
|
|
|
|
if (!wait) {
|
|
|
|
csd = this_cpu_ptr(&csd_data);
|
|
|
|
csd_lock(csd);
|
|
|
|
}
|
|
|
|
|
|
|
|
err = generic_exec_single(cpu, csd, func, info);
|
|
|
|
|
|
|
|
if (wait)
|
|
|
|
csd_lock_wait(csd);
|
2008-06-26 03:21:34 -06:00
|
|
|
|
|
|
|
put_cpu();
|
2009-02-25 08:52:11 -07:00
|
|
|
|
2008-08-25 18:07:14 -06:00
|
|
|
return err;
|
2008-06-26 03:21:34 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(smp_call_function_single);
|
|
|
|
|
2014-02-24 08:39:59 -07:00
|
|
|
/**
|
2014-02-24 08:40:02 -07:00
|
|
|
* smp_call_function_single_async(): Run an asynchronous function on a
|
|
|
|
* specific CPU.
|
2014-02-24 08:39:59 -07:00
|
|
|
* @cpu: The CPU to run on.
|
|
|
|
* @csd: Pre-allocated and setup data structure
|
|
|
|
*
|
2014-02-24 08:40:02 -07:00
|
|
|
* Like smp_call_function_single(), but the call is asynchonous and
|
|
|
|
* can thus be done from contexts with disabled interrupts.
|
|
|
|
*
|
|
|
|
* The caller passes his own pre-allocated data structure
|
|
|
|
* (ie: embedded in an object) and is responsible for synchronizing it
|
|
|
|
* such that the IPIs performed on the @csd are strictly serialized.
|
|
|
|
*
|
|
|
|
* NOTE: Be careful, there is unfortunately no current debugging facility to
|
|
|
|
* validate the correctness of this serialization.
|
2014-02-24 08:39:59 -07:00
|
|
|
*/
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
int smp_call_function_single_async(int cpu, call_single_data_t *csd)
|
2014-02-24 08:39:59 -07:00
|
|
|
{
|
|
|
|
int err = 0;
|
|
|
|
|
2014-02-24 08:40:01 -07:00
|
|
|
preempt_disable();
|
2015-02-11 13:42:10 -07:00
|
|
|
|
|
|
|
/* We could deadlock if we have to wait here with interrupts disabled! */
|
|
|
|
if (WARN_ON_ONCE(csd->flags & CSD_FLAG_LOCK))
|
|
|
|
csd_lock_wait(csd);
|
|
|
|
|
|
|
|
csd->flags = CSD_FLAG_LOCK;
|
|
|
|
smp_wmb();
|
|
|
|
|
|
|
|
err = generic_exec_single(cpu, csd, csd->func, csd->info);
|
2014-02-24 08:40:01 -07:00
|
|
|
preempt_enable();
|
2014-02-24 08:39:59 -07:00
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
2014-02-24 08:40:02 -07:00
|
|
|
EXPORT_SYMBOL_GPL(smp_call_function_single_async);
|
2014-02-24 08:39:59 -07:00
|
|
|
|
2009-11-17 15:27:27 -07:00
|
|
|
/*
|
|
|
|
* smp_call_function_any - Run a function on any of the given cpus
|
|
|
|
* @mask: The mask of cpus it can run on.
|
|
|
|
* @func: The function to run. This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to the function.
|
|
|
|
* @wait: If true, wait until function has completed.
|
|
|
|
*
|
|
|
|
* Returns 0 on success, else a negative status code (if no cpus were online).
|
|
|
|
*
|
|
|
|
* Selection preference:
|
|
|
|
* 1) current cpu if in @mask
|
|
|
|
* 2) any cpu of current node if in @mask
|
|
|
|
* 3) any other online cpu in @mask
|
|
|
|
*/
|
|
|
|
int smp_call_function_any(const struct cpumask *mask,
|
2010-10-27 10:28:36 -06:00
|
|
|
smp_call_func_t func, void *info, int wait)
|
2009-11-17 15:27:27 -07:00
|
|
|
{
|
|
|
|
unsigned int cpu;
|
|
|
|
const struct cpumask *nodemask;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
/* Try for same CPU (cheapest) */
|
|
|
|
cpu = get_cpu();
|
|
|
|
if (cpumask_test_cpu(cpu, mask))
|
|
|
|
goto call;
|
|
|
|
|
|
|
|
/* Try for same node. */
|
2010-01-15 18:01:23 -07:00
|
|
|
nodemask = cpumask_of_node(cpu_to_node(cpu));
|
2009-11-17 15:27:27 -07:00
|
|
|
for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids;
|
|
|
|
cpu = cpumask_next_and(cpu, nodemask, mask)) {
|
|
|
|
if (cpu_online(cpu))
|
|
|
|
goto call;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Any online will do: smp_call_function_single handles nr_cpu_ids. */
|
|
|
|
cpu = cpumask_any_and(mask, cpu_online_mask);
|
|
|
|
call:
|
|
|
|
ret = smp_call_function_single(cpu, func, info, wait);
|
|
|
|
put_cpu();
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(smp_call_function_any);
|
|
|
|
|
2008-06-26 03:21:34 -06:00
|
|
|
/**
|
2008-12-29 15:35:16 -07:00
|
|
|
* smp_call_function_many(): Run a function on a set of other CPUs.
|
|
|
|
* @mask: The set of cpus to run on (only runs on online subset).
|
2008-06-26 03:21:34 -06:00
|
|
|
* @func: The function to run. This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to the function.
|
2009-02-25 08:52:11 -07:00
|
|
|
* @wait: If true, wait (atomically) until function has completed
|
|
|
|
* on other CPUs.
|
2008-06-26 03:21:34 -06:00
|
|
|
*
|
2009-10-22 05:19:34 -06:00
|
|
|
* If @wait is true, then returns once @func has returned.
|
2008-06-26 03:21:34 -06:00
|
|
|
*
|
|
|
|
* You must not call this function with disabled interrupts or from a
|
|
|
|
* hardware interrupt handler or from a bottom half handler. Preemption
|
|
|
|
* must be disabled when calling this function.
|
|
|
|
*/
|
2008-12-29 15:35:16 -07:00
|
|
|
void smp_call_function_many(const struct cpumask *mask,
|
2010-10-27 10:28:36 -06:00
|
|
|
smp_call_func_t func, void *info, bool wait)
|
2008-06-26 03:21:34 -06:00
|
|
|
{
|
2013-04-30 16:27:28 -06:00
|
|
|
struct call_function_data *cfd;
|
smp: make smp_call_function_many() use logic similar to smp_call_function_single()
I'm testing swapout workload in a two-socket Xeon machine. The workload
has 10 threads, each thread sequentially accesses separate memory
region. TLB flush overhead is very big in the workload. For each page,
page reclaim need move it from active lru list and then unmap it. Both
need a TLB flush. And this is a multthread workload, TLB flush happens
in 10 CPUs. In X86, TLB flush uses generic smp_call)function. So this
workload stress smp_call_function_many heavily.
Without patch, perf shows:
+ 24.49% [k] generic_smp_call_function_interrupt
- 21.72% [k] _raw_spin_lock
- _raw_spin_lock
+ 79.80% __page_check_address
+ 6.42% generic_smp_call_function_interrupt
+ 3.31% get_swap_page
+ 2.37% free_pcppages_bulk
+ 1.75% handle_pte_fault
+ 1.54% put_super
+ 1.41% grab_super_passive
+ 1.36% __swap_duplicate
+ 0.68% blk_flush_plug_list
+ 0.62% swap_info_get
+ 6.55% [k] flush_tlb_func
+ 6.46% [k] smp_call_function_many
+ 5.09% [k] call_function_interrupt
+ 4.75% [k] default_send_IPI_mask_sequence_phys
+ 2.18% [k] find_next_bit
swapout throughput is around 1300M/s.
With the patch, perf shows:
- 27.23% [k] _raw_spin_lock
- _raw_spin_lock
+ 80.53% __page_check_address
+ 8.39% generic_smp_call_function_single_interrupt
+ 2.44% get_swap_page
+ 1.76% free_pcppages_bulk
+ 1.40% handle_pte_fault
+ 1.15% __swap_duplicate
+ 1.05% put_super
+ 0.98% grab_super_passive
+ 0.86% blk_flush_plug_list
+ 0.57% swap_info_get
+ 8.25% [k] default_send_IPI_mask_sequence_phys
+ 7.55% [k] call_function_interrupt
+ 7.47% [k] smp_call_function_many
+ 7.25% [k] flush_tlb_func
+ 3.81% [k] _raw_spin_lock_irqsave
+ 3.78% [k] generic_smp_call_function_single_interrupt
swapout throughput is around 1400M/s. So there is around a 7%
improvement, and total cpu utilization doesn't change.
Without the patch, cfd_data is shared by all CPUs.
generic_smp_call_function_interrupt does read/write cfd_data several times
which will create a lot of cache ping-pong. With the patch, the data
becomes per-cpu. The ping-pong is avoided. And from the perf data, this
doesn't make call_single_queue lock contend.
Next step is to remove generic_smp_call_function_interrupt() from arch
code.
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-21 17:43:03 -07:00
|
|
|
int cpu, next_cpu, this_cpu = smp_processor_id();
|
2008-06-26 03:21:34 -06:00
|
|
|
|
2009-08-19 19:05:35 -06:00
|
|
|
/*
|
|
|
|
* Can deadlock when called with interrupts disabled.
|
|
|
|
* We allow cpu's that are not yet online though, as no one else can
|
|
|
|
* send smp call function interrupt to this cpu and as such deadlocks
|
|
|
|
* can't happen.
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
|
2011-01-20 04:07:13 -07:00
|
|
|
&& !oops_in_progress && !early_boot_irqs_disabled);
|
2008-06-26 03:21:34 -06:00
|
|
|
|
2011-03-15 13:27:17 -06:00
|
|
|
/* Try to fastpath. So, what's a CPU they want? Ignoring this one. */
|
2008-12-29 15:35:16 -07:00
|
|
|
cpu = cpumask_first_and(mask, cpu_online_mask);
|
2009-02-25 08:52:11 -07:00
|
|
|
if (cpu == this_cpu)
|
2008-12-29 15:35:16 -07:00
|
|
|
cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
|
2009-02-25 08:52:11 -07:00
|
|
|
|
2008-12-29 15:35:16 -07:00
|
|
|
/* No online cpus? We're done. */
|
|
|
|
if (cpu >= nr_cpu_ids)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Do we have another CPU which isn't us? */
|
|
|
|
next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
|
2009-02-25 08:52:11 -07:00
|
|
|
if (next_cpu == this_cpu)
|
2008-12-29 15:35:16 -07:00
|
|
|
next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
|
|
|
|
|
|
|
|
/* Fastpath: do that cpu by itself. */
|
|
|
|
if (next_cpu >= nr_cpu_ids) {
|
|
|
|
smp_call_function_single(cpu, func, info, wait);
|
|
|
|
return;
|
2008-06-26 03:21:34 -06:00
|
|
|
}
|
|
|
|
|
2014-08-17 11:30:24 -06:00
|
|
|
cfd = this_cpu_ptr(&cfd_data);
|
2011-03-15 13:27:16 -06:00
|
|
|
|
2013-04-30 16:27:28 -06:00
|
|
|
cpumask_and(cfd->cpumask, mask, cpu_online_mask);
|
2017-05-19 04:58:25 -06:00
|
|
|
__cpumask_clear_cpu(this_cpu, cfd->cpumask);
|
2011-03-15 13:27:17 -06:00
|
|
|
|
|
|
|
/* Some callers race with other cpus changing the passed mask */
|
2013-04-30 16:27:28 -06:00
|
|
|
if (unlikely(!cpumask_weight(cfd->cpumask)))
|
2011-03-15 13:27:17 -06:00
|
|
|
return;
|
2008-06-26 03:21:34 -06:00
|
|
|
|
2017-05-19 01:53:31 -06:00
|
|
|
cpumask_clear(cfd->cpumask_ipi);
|
2013-04-30 16:27:28 -06:00
|
|
|
for_each_cpu(cpu, cfd->cpumask) {
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);
|
smp: make smp_call_function_many() use logic similar to smp_call_function_single()
I'm testing swapout workload in a two-socket Xeon machine. The workload
has 10 threads, each thread sequentially accesses separate memory
region. TLB flush overhead is very big in the workload. For each page,
page reclaim need move it from active lru list and then unmap it. Both
need a TLB flush. And this is a multthread workload, TLB flush happens
in 10 CPUs. In X86, TLB flush uses generic smp_call)function. So this
workload stress smp_call_function_many heavily.
Without patch, perf shows:
+ 24.49% [k] generic_smp_call_function_interrupt
- 21.72% [k] _raw_spin_lock
- _raw_spin_lock
+ 79.80% __page_check_address
+ 6.42% generic_smp_call_function_interrupt
+ 3.31% get_swap_page
+ 2.37% free_pcppages_bulk
+ 1.75% handle_pte_fault
+ 1.54% put_super
+ 1.41% grab_super_passive
+ 1.36% __swap_duplicate
+ 0.68% blk_flush_plug_list
+ 0.62% swap_info_get
+ 6.55% [k] flush_tlb_func
+ 6.46% [k] smp_call_function_many
+ 5.09% [k] call_function_interrupt
+ 4.75% [k] default_send_IPI_mask_sequence_phys
+ 2.18% [k] find_next_bit
swapout throughput is around 1300M/s.
With the patch, perf shows:
- 27.23% [k] _raw_spin_lock
- _raw_spin_lock
+ 80.53% __page_check_address
+ 8.39% generic_smp_call_function_single_interrupt
+ 2.44% get_swap_page
+ 1.76% free_pcppages_bulk
+ 1.40% handle_pte_fault
+ 1.15% __swap_duplicate
+ 1.05% put_super
+ 0.98% grab_super_passive
+ 0.86% blk_flush_plug_list
+ 0.57% swap_info_get
+ 8.25% [k] default_send_IPI_mask_sequence_phys
+ 7.55% [k] call_function_interrupt
+ 7.47% [k] smp_call_function_many
+ 7.25% [k] flush_tlb_func
+ 3.81% [k] _raw_spin_lock_irqsave
+ 3.78% [k] generic_smp_call_function_single_interrupt
swapout throughput is around 1400M/s. So there is around a 7%
improvement, and total cpu utilization doesn't change.
Without the patch, cfd_data is shared by all CPUs.
generic_smp_call_function_interrupt does read/write cfd_data several times
which will create a lot of cache ping-pong. With the patch, the data
becomes per-cpu. The ping-pong is avoided. And from the perf data, this
doesn't make call_single_queue lock contend.
Next step is to remove generic_smp_call_function_interrupt() from arch
code.
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-21 17:43:03 -07:00
|
|
|
|
|
|
|
csd_lock(csd);
|
2015-02-11 13:42:10 -07:00
|
|
|
if (wait)
|
|
|
|
csd->flags |= CSD_FLAG_SYNCHRONOUS;
|
smp: make smp_call_function_many() use logic similar to smp_call_function_single()
I'm testing swapout workload in a two-socket Xeon machine. The workload
has 10 threads, each thread sequentially accesses separate memory
region. TLB flush overhead is very big in the workload. For each page,
page reclaim need move it from active lru list and then unmap it. Both
need a TLB flush. And this is a multthread workload, TLB flush happens
in 10 CPUs. In X86, TLB flush uses generic smp_call)function. So this
workload stress smp_call_function_many heavily.
Without patch, perf shows:
+ 24.49% [k] generic_smp_call_function_interrupt
- 21.72% [k] _raw_spin_lock
- _raw_spin_lock
+ 79.80% __page_check_address
+ 6.42% generic_smp_call_function_interrupt
+ 3.31% get_swap_page
+ 2.37% free_pcppages_bulk
+ 1.75% handle_pte_fault
+ 1.54% put_super
+ 1.41% grab_super_passive
+ 1.36% __swap_duplicate
+ 0.68% blk_flush_plug_list
+ 0.62% swap_info_get
+ 6.55% [k] flush_tlb_func
+ 6.46% [k] smp_call_function_many
+ 5.09% [k] call_function_interrupt
+ 4.75% [k] default_send_IPI_mask_sequence_phys
+ 2.18% [k] find_next_bit
swapout throughput is around 1300M/s.
With the patch, perf shows:
- 27.23% [k] _raw_spin_lock
- _raw_spin_lock
+ 80.53% __page_check_address
+ 8.39% generic_smp_call_function_single_interrupt
+ 2.44% get_swap_page
+ 1.76% free_pcppages_bulk
+ 1.40% handle_pte_fault
+ 1.15% __swap_duplicate
+ 1.05% put_super
+ 0.98% grab_super_passive
+ 0.86% blk_flush_plug_list
+ 0.57% swap_info_get
+ 8.25% [k] default_send_IPI_mask_sequence_phys
+ 7.55% [k] call_function_interrupt
+ 7.47% [k] smp_call_function_many
+ 7.25% [k] flush_tlb_func
+ 3.81% [k] _raw_spin_lock_irqsave
+ 3.78% [k] generic_smp_call_function_single_interrupt
swapout throughput is around 1400M/s. So there is around a 7%
improvement, and total cpu utilization doesn't change.
Without the patch, cfd_data is shared by all CPUs.
generic_smp_call_function_interrupt does read/write cfd_data several times
which will create a lot of cache ping-pong. With the patch, the data
becomes per-cpu. The ping-pong is avoided. And from the perf data, this
doesn't make call_single_queue lock contend.
Next step is to remove generic_smp_call_function_interrupt() from arch
code.
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-21 17:43:03 -07:00
|
|
|
csd->func = func;
|
|
|
|
csd->info = info;
|
2017-05-19 01:53:31 -06:00
|
|
|
if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
|
2017-05-19 04:58:25 -06:00
|
|
|
__cpumask_set_cpu(cpu, cfd->cpumask_ipi);
|
smp: make smp_call_function_many() use logic similar to smp_call_function_single()
I'm testing swapout workload in a two-socket Xeon machine. The workload
has 10 threads, each thread sequentially accesses separate memory
region. TLB flush overhead is very big in the workload. For each page,
page reclaim need move it from active lru list and then unmap it. Both
need a TLB flush. And this is a multthread workload, TLB flush happens
in 10 CPUs. In X86, TLB flush uses generic smp_call)function. So this
workload stress smp_call_function_many heavily.
Without patch, perf shows:
+ 24.49% [k] generic_smp_call_function_interrupt
- 21.72% [k] _raw_spin_lock
- _raw_spin_lock
+ 79.80% __page_check_address
+ 6.42% generic_smp_call_function_interrupt
+ 3.31% get_swap_page
+ 2.37% free_pcppages_bulk
+ 1.75% handle_pte_fault
+ 1.54% put_super
+ 1.41% grab_super_passive
+ 1.36% __swap_duplicate
+ 0.68% blk_flush_plug_list
+ 0.62% swap_info_get
+ 6.55% [k] flush_tlb_func
+ 6.46% [k] smp_call_function_many
+ 5.09% [k] call_function_interrupt
+ 4.75% [k] default_send_IPI_mask_sequence_phys
+ 2.18% [k] find_next_bit
swapout throughput is around 1300M/s.
With the patch, perf shows:
- 27.23% [k] _raw_spin_lock
- _raw_spin_lock
+ 80.53% __page_check_address
+ 8.39% generic_smp_call_function_single_interrupt
+ 2.44% get_swap_page
+ 1.76% free_pcppages_bulk
+ 1.40% handle_pte_fault
+ 1.15% __swap_duplicate
+ 1.05% put_super
+ 0.98% grab_super_passive
+ 0.86% blk_flush_plug_list
+ 0.57% swap_info_get
+ 8.25% [k] default_send_IPI_mask_sequence_phys
+ 7.55% [k] call_function_interrupt
+ 7.47% [k] smp_call_function_many
+ 7.25% [k] flush_tlb_func
+ 3.81% [k] _raw_spin_lock_irqsave
+ 3.78% [k] generic_smp_call_function_single_interrupt
swapout throughput is around 1400M/s. So there is around a 7%
improvement, and total cpu utilization doesn't change.
Without the patch, cfd_data is shared by all CPUs.
generic_smp_call_function_interrupt does read/write cfd_data several times
which will create a lot of cache ping-pong. With the patch, the data
becomes per-cpu. The ping-pong is avoided. And from the perf data, this
doesn't make call_single_queue lock contend.
Next step is to remove generic_smp_call_function_interrupt() from arch
code.
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-21 17:43:03 -07:00
|
|
|
}
|
2008-10-30 11:28:41 -06:00
|
|
|
|
2008-06-26 03:21:34 -06:00
|
|
|
/* Send a message to all CPUs in the map */
|
2017-05-19 01:53:31 -06:00
|
|
|
arch_send_call_function_ipi_mask(cfd->cpumask_ipi);
|
2008-06-26 03:21:34 -06:00
|
|
|
|
smp: make smp_call_function_many() use logic similar to smp_call_function_single()
I'm testing swapout workload in a two-socket Xeon machine. The workload
has 10 threads, each thread sequentially accesses separate memory
region. TLB flush overhead is very big in the workload. For each page,
page reclaim need move it from active lru list and then unmap it. Both
need a TLB flush. And this is a multthread workload, TLB flush happens
in 10 CPUs. In X86, TLB flush uses generic smp_call)function. So this
workload stress smp_call_function_many heavily.
Without patch, perf shows:
+ 24.49% [k] generic_smp_call_function_interrupt
- 21.72% [k] _raw_spin_lock
- _raw_spin_lock
+ 79.80% __page_check_address
+ 6.42% generic_smp_call_function_interrupt
+ 3.31% get_swap_page
+ 2.37% free_pcppages_bulk
+ 1.75% handle_pte_fault
+ 1.54% put_super
+ 1.41% grab_super_passive
+ 1.36% __swap_duplicate
+ 0.68% blk_flush_plug_list
+ 0.62% swap_info_get
+ 6.55% [k] flush_tlb_func
+ 6.46% [k] smp_call_function_many
+ 5.09% [k] call_function_interrupt
+ 4.75% [k] default_send_IPI_mask_sequence_phys
+ 2.18% [k] find_next_bit
swapout throughput is around 1300M/s.
With the patch, perf shows:
- 27.23% [k] _raw_spin_lock
- _raw_spin_lock
+ 80.53% __page_check_address
+ 8.39% generic_smp_call_function_single_interrupt
+ 2.44% get_swap_page
+ 1.76% free_pcppages_bulk
+ 1.40% handle_pte_fault
+ 1.15% __swap_duplicate
+ 1.05% put_super
+ 0.98% grab_super_passive
+ 0.86% blk_flush_plug_list
+ 0.57% swap_info_get
+ 8.25% [k] default_send_IPI_mask_sequence_phys
+ 7.55% [k] call_function_interrupt
+ 7.47% [k] smp_call_function_many
+ 7.25% [k] flush_tlb_func
+ 3.81% [k] _raw_spin_lock_irqsave
+ 3.78% [k] generic_smp_call_function_single_interrupt
swapout throughput is around 1400M/s. So there is around a 7%
improvement, and total cpu utilization doesn't change.
Without the patch, cfd_data is shared by all CPUs.
generic_smp_call_function_interrupt does read/write cfd_data several times
which will create a lot of cache ping-pong. With the patch, the data
becomes per-cpu. The ping-pong is avoided. And from the perf data, this
doesn't make call_single_queue lock contend.
Next step is to remove generic_smp_call_function_interrupt() from arch
code.
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-21 17:43:03 -07:00
|
|
|
if (wait) {
|
2013-04-30 16:27:28 -06:00
|
|
|
for_each_cpu(cpu, cfd->cpumask) {
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-07 22:30:00 -06:00
|
|
|
call_single_data_t *csd;
|
2013-04-30 16:27:28 -06:00
|
|
|
|
|
|
|
csd = per_cpu_ptr(cfd->csd, cpu);
|
smp: make smp_call_function_many() use logic similar to smp_call_function_single()
I'm testing swapout workload in a two-socket Xeon machine. The workload
has 10 threads, each thread sequentially accesses separate memory
region. TLB flush overhead is very big in the workload. For each page,
page reclaim need move it from active lru list and then unmap it. Both
need a TLB flush. And this is a multthread workload, TLB flush happens
in 10 CPUs. In X86, TLB flush uses generic smp_call)function. So this
workload stress smp_call_function_many heavily.
Without patch, perf shows:
+ 24.49% [k] generic_smp_call_function_interrupt
- 21.72% [k] _raw_spin_lock
- _raw_spin_lock
+ 79.80% __page_check_address
+ 6.42% generic_smp_call_function_interrupt
+ 3.31% get_swap_page
+ 2.37% free_pcppages_bulk
+ 1.75% handle_pte_fault
+ 1.54% put_super
+ 1.41% grab_super_passive
+ 1.36% __swap_duplicate
+ 0.68% blk_flush_plug_list
+ 0.62% swap_info_get
+ 6.55% [k] flush_tlb_func
+ 6.46% [k] smp_call_function_many
+ 5.09% [k] call_function_interrupt
+ 4.75% [k] default_send_IPI_mask_sequence_phys
+ 2.18% [k] find_next_bit
swapout throughput is around 1300M/s.
With the patch, perf shows:
- 27.23% [k] _raw_spin_lock
- _raw_spin_lock
+ 80.53% __page_check_address
+ 8.39% generic_smp_call_function_single_interrupt
+ 2.44% get_swap_page
+ 1.76% free_pcppages_bulk
+ 1.40% handle_pte_fault
+ 1.15% __swap_duplicate
+ 1.05% put_super
+ 0.98% grab_super_passive
+ 0.86% blk_flush_plug_list
+ 0.57% swap_info_get
+ 8.25% [k] default_send_IPI_mask_sequence_phys
+ 7.55% [k] call_function_interrupt
+ 7.47% [k] smp_call_function_many
+ 7.25% [k] flush_tlb_func
+ 3.81% [k] _raw_spin_lock_irqsave
+ 3.78% [k] generic_smp_call_function_single_interrupt
swapout throughput is around 1400M/s. So there is around a 7%
improvement, and total cpu utilization doesn't change.
Without the patch, cfd_data is shared by all CPUs.
generic_smp_call_function_interrupt does read/write cfd_data several times
which will create a lot of cache ping-pong. With the patch, the data
becomes per-cpu. The ping-pong is avoided. And from the perf data, this
doesn't make call_single_queue lock contend.
Next step is to remove generic_smp_call_function_interrupt() from arch
code.
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-21 17:43:03 -07:00
|
|
|
csd_lock_wait(csd);
|
|
|
|
}
|
|
|
|
}
|
2008-06-26 03:21:34 -06:00
|
|
|
}
|
2008-12-29 15:35:16 -07:00
|
|
|
EXPORT_SYMBOL(smp_call_function_many);
|
2008-06-26 03:21:34 -06:00
|
|
|
|
|
|
|
/**
|
|
|
|
* smp_call_function(): Run a function on all other CPUs.
|
|
|
|
* @func: The function to run. This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to the function.
|
2009-02-25 08:52:11 -07:00
|
|
|
* @wait: If true, wait (atomically) until function has completed
|
|
|
|
* on other CPUs.
|
2008-06-26 03:21:34 -06:00
|
|
|
*
|
2008-12-29 15:35:16 -07:00
|
|
|
* Returns 0.
|
2008-06-26 03:21:34 -06:00
|
|
|
*
|
|
|
|
* If @wait is true, then returns once @func has returned; otherwise
|
2009-10-22 05:19:34 -06:00
|
|
|
* it returns just before the target cpu calls @func.
|
2008-06-26 03:21:34 -06:00
|
|
|
*
|
|
|
|
* You must not call this function with disabled interrupts or from a
|
|
|
|
* hardware interrupt handler or from a bottom half handler.
|
|
|
|
*/
|
2010-10-27 10:28:36 -06:00
|
|
|
int smp_call_function(smp_call_func_t func, void *info, int wait)
|
2008-06-26 03:21:34 -06:00
|
|
|
{
|
|
|
|
preempt_disable();
|
2008-12-29 15:35:16 -07:00
|
|
|
smp_call_function_many(cpu_online_mask, func, info, wait);
|
2008-06-26 03:21:34 -06:00
|
|
|
preempt_enable();
|
2009-02-25 08:52:11 -07:00
|
|
|
|
2008-12-29 15:35:16 -07:00
|
|
|
return 0;
|
2008-06-26 03:21:34 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(smp_call_function);
|
2011-01-12 17:59:39 -07:00
|
|
|
|
2011-03-22 17:34:06 -06:00
|
|
|
/* Setup configured maximum number of CPUs to activate */
|
|
|
|
unsigned int setup_max_cpus = NR_CPUS;
|
|
|
|
EXPORT_SYMBOL(setup_max_cpus);
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Setup routine for controlling SMP activation
|
|
|
|
*
|
|
|
|
* Command-line option of "nosmp" or "maxcpus=0" will disable SMP
|
|
|
|
* activation entirely (the MPS table probe still happens, though).
|
|
|
|
*
|
|
|
|
* Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
|
|
|
|
* greater than 0, limits the maximum number of CPUs activated in
|
|
|
|
* SMP mode to <NUM>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
void __weak arch_disable_smp_support(void) { }
|
|
|
|
|
|
|
|
static int __init nosmp(char *str)
|
|
|
|
{
|
|
|
|
setup_max_cpus = 0;
|
|
|
|
arch_disable_smp_support();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
early_param("nosmp", nosmp);
|
|
|
|
|
|
|
|
/* this is hard limit */
|
|
|
|
static int __init nrcpus(char *str)
|
|
|
|
{
|
|
|
|
int nr_cpus;
|
|
|
|
|
|
|
|
get_option(&str, &nr_cpus);
|
|
|
|
if (nr_cpus > 0 && nr_cpus < nr_cpu_ids)
|
|
|
|
nr_cpu_ids = nr_cpus;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
early_param("nr_cpus", nrcpus);
|
|
|
|
|
|
|
|
static int __init maxcpus(char *str)
|
|
|
|
{
|
|
|
|
get_option(&str, &setup_max_cpus);
|
|
|
|
if (setup_max_cpus == 0)
|
|
|
|
arch_disable_smp_support();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
early_param("maxcpus", maxcpus);
|
|
|
|
|
|
|
|
/* Setup number of possible processor ids */
|
2017-09-08 17:14:18 -06:00
|
|
|
unsigned int nr_cpu_ids __read_mostly = NR_CPUS;
|
2011-03-22 17:34:06 -06:00
|
|
|
EXPORT_SYMBOL(nr_cpu_ids);
|
|
|
|
|
|
|
|
/* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
|
|
|
|
void __init setup_nr_cpu_ids(void)
|
|
|
|
{
|
|
|
|
nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Called by boot processor to activate the rest. */
|
|
|
|
void __init smp_init(void)
|
|
|
|
{
|
2016-10-25 23:37:54 -06:00
|
|
|
int num_nodes, num_cpus;
|
2011-03-22 17:34:06 -06:00
|
|
|
unsigned int cpu;
|
|
|
|
|
2012-04-20 18:08:50 -06:00
|
|
|
idle_threads_init();
|
2016-02-26 11:43:38 -07:00
|
|
|
cpuhp_threads_init();
|
2012-04-20 18:08:50 -06:00
|
|
|
|
2016-10-25 23:37:55 -06:00
|
|
|
pr_info("Bringing up secondary CPUs ...\n");
|
|
|
|
|
2011-03-22 17:34:06 -06:00
|
|
|
/* FIXME: This should be done in userspace --RR */
|
|
|
|
for_each_present_cpu(cpu) {
|
|
|
|
if (num_online_cpus() >= setup_max_cpus)
|
|
|
|
break;
|
|
|
|
if (!cpu_online(cpu))
|
|
|
|
cpu_up(cpu);
|
|
|
|
}
|
|
|
|
|
2016-10-25 23:37:54 -06:00
|
|
|
num_nodes = num_online_nodes();
|
|
|
|
num_cpus = num_online_cpus();
|
|
|
|
pr_info("Brought up %d node%s, %d CPU%s\n",
|
|
|
|
num_nodes, (num_nodes > 1 ? "s" : ""),
|
|
|
|
num_cpus, (num_cpus > 1 ? "s" : ""));
|
|
|
|
|
2011-03-22 17:34:06 -06:00
|
|
|
/* Any cleanup work */
|
|
|
|
smp_cpus_done(setup_max_cpus);
|
|
|
|
}
|
|
|
|
|
2011-01-12 17:59:39 -07:00
|
|
|
/*
|
2011-01-20 04:07:13 -07:00
|
|
|
* Call a function on all processors. May be used during early boot while
|
|
|
|
* early_boot_irqs_disabled is set. Use local_irq_save/restore() instead
|
|
|
|
* of local_irq_disable/enable().
|
2011-01-12 17:59:39 -07:00
|
|
|
*/
|
|
|
|
int on_each_cpu(void (*func) (void *info), void *info, int wait)
|
|
|
|
{
|
2011-01-20 04:07:13 -07:00
|
|
|
unsigned long flags;
|
2011-01-12 17:59:39 -07:00
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
preempt_disable();
|
|
|
|
ret = smp_call_function(func, info, wait);
|
2011-01-20 04:07:13 -07:00
|
|
|
local_irq_save(flags);
|
2011-01-12 17:59:39 -07:00
|
|
|
func(info);
|
2011-01-20 04:07:13 -07:00
|
|
|
local_irq_restore(flags);
|
2011-01-12 17:59:39 -07:00
|
|
|
preempt_enable();
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(on_each_cpu);
|
smp: introduce a generic on_each_cpu_mask() function
We have lots of infrastructure in place to partition multi-core systems
such that we have a group of CPUs that are dedicated to specific task:
cgroups, scheduler and interrupt affinity, and cpuisol= boot parameter.
Still, kernel code will at times interrupt all CPUs in the system via IPIs
for various needs. These IPIs are useful and cannot be avoided
altogether, but in certain cases it is possible to interrupt only specific
CPUs that have useful work to do and not the entire system.
This patch set, inspired by discussions with Peter Zijlstra and Frederic
Weisbecker when testing the nohz task patch set, is a first stab at trying
to explore doing this by locating the places where such global IPI calls
are being made and turning the global IPI into an IPI for a specific group
of CPUs. The purpose of the patch set is to get feedback if this is the
right way to go for dealing with this issue and indeed, if the issue is
even worth dealing with at all. Based on the feedback from this patch set
I plan to offer further patches that address similar issue in other code
paths.
This patch creates an on_each_cpu_mask() and on_each_cpu_cond()
infrastructure API (the former derived from existing arch specific
versions in Tile and Arm) and uses them to turn several global IPI
invocation to per CPU group invocations.
Core kernel:
on_each_cpu_mask() calls a function on processors specified by cpumask,
which may or may not include the local processor.
You must not call this function with disabled interrupts or from a
hardware interrupt handler or from a bottom half handler.
arch/arm:
Note that the generic version is a little different then the Arm one:
1. It has the mask as first parameter
2. It calls the function on the calling CPU with interrupts disabled,
but this should be OK since the function is called on the other CPUs
with interrupts disabled anyway.
arch/tile:
The API is the same as the tile private one, but the generic version
also calls the function on the with interrupts disabled in UP case
This is OK since the function is called on the other CPUs
with interrupts disabled.
Signed-off-by: Gilad Ben-Yossef <gilad@benyossef.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Acked-by: Chris Metcalf <cmetcalf@tilera.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Matt Mackall <mpm@selenic.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Sasha Levin <levinsasha928@gmail.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Avi Kivity <avi@redhat.com>
Acked-by: Michal Nazarewicz <mina86@mina86.org>
Cc: Kosaki Motohiro <kosaki.motohiro@gmail.com>
Cc: Milton Miller <miltonm@bga.com>
Cc: Russell King <linux@arm.linux.org.uk>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-28 15:42:43 -06:00
|
|
|
|
|
|
|
/**
|
|
|
|
* on_each_cpu_mask(): Run a function on processors specified by
|
|
|
|
* cpumask, which may include the local processor.
|
|
|
|
* @mask: The set of cpus to run on (only runs on online subset).
|
|
|
|
* @func: The function to run. This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to the function.
|
|
|
|
* @wait: If true, wait (atomically) until function has completed
|
|
|
|
* on other CPUs.
|
|
|
|
*
|
|
|
|
* If @wait is true, then returns once @func has returned.
|
|
|
|
*
|
2013-09-11 15:23:29 -06:00
|
|
|
* You must not call this function with disabled interrupts or from a
|
|
|
|
* hardware interrupt handler or from a bottom half handler. The
|
|
|
|
* exception is that it may be used during early boot while
|
|
|
|
* early_boot_irqs_disabled is set.
|
smp: introduce a generic on_each_cpu_mask() function
We have lots of infrastructure in place to partition multi-core systems
such that we have a group of CPUs that are dedicated to specific task:
cgroups, scheduler and interrupt affinity, and cpuisol= boot parameter.
Still, kernel code will at times interrupt all CPUs in the system via IPIs
for various needs. These IPIs are useful and cannot be avoided
altogether, but in certain cases it is possible to interrupt only specific
CPUs that have useful work to do and not the entire system.
This patch set, inspired by discussions with Peter Zijlstra and Frederic
Weisbecker when testing the nohz task patch set, is a first stab at trying
to explore doing this by locating the places where such global IPI calls
are being made and turning the global IPI into an IPI for a specific group
of CPUs. The purpose of the patch set is to get feedback if this is the
right way to go for dealing with this issue and indeed, if the issue is
even worth dealing with at all. Based on the feedback from this patch set
I plan to offer further patches that address similar issue in other code
paths.
This patch creates an on_each_cpu_mask() and on_each_cpu_cond()
infrastructure API (the former derived from existing arch specific
versions in Tile and Arm) and uses them to turn several global IPI
invocation to per CPU group invocations.
Core kernel:
on_each_cpu_mask() calls a function on processors specified by cpumask,
which may or may not include the local processor.
You must not call this function with disabled interrupts or from a
hardware interrupt handler or from a bottom half handler.
arch/arm:
Note that the generic version is a little different then the Arm one:
1. It has the mask as first parameter
2. It calls the function on the calling CPU with interrupts disabled,
but this should be OK since the function is called on the other CPUs
with interrupts disabled anyway.
arch/tile:
The API is the same as the tile private one, but the generic version
also calls the function on the with interrupts disabled in UP case
This is OK since the function is called on the other CPUs
with interrupts disabled.
Signed-off-by: Gilad Ben-Yossef <gilad@benyossef.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Acked-by: Chris Metcalf <cmetcalf@tilera.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Matt Mackall <mpm@selenic.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Sasha Levin <levinsasha928@gmail.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Avi Kivity <avi@redhat.com>
Acked-by: Michal Nazarewicz <mina86@mina86.org>
Cc: Kosaki Motohiro <kosaki.motohiro@gmail.com>
Cc: Milton Miller <miltonm@bga.com>
Cc: Russell King <linux@arm.linux.org.uk>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-28 15:42:43 -06:00
|
|
|
*/
|
|
|
|
void on_each_cpu_mask(const struct cpumask *mask, smp_call_func_t func,
|
|
|
|
void *info, bool wait)
|
|
|
|
{
|
|
|
|
int cpu = get_cpu();
|
|
|
|
|
|
|
|
smp_call_function_many(mask, func, info, wait);
|
|
|
|
if (cpumask_test_cpu(cpu, mask)) {
|
2013-09-11 15:23:29 -06:00
|
|
|
unsigned long flags;
|
|
|
|
local_irq_save(flags);
|
smp: introduce a generic on_each_cpu_mask() function
We have lots of infrastructure in place to partition multi-core systems
such that we have a group of CPUs that are dedicated to specific task:
cgroups, scheduler and interrupt affinity, and cpuisol= boot parameter.
Still, kernel code will at times interrupt all CPUs in the system via IPIs
for various needs. These IPIs are useful and cannot be avoided
altogether, but in certain cases it is possible to interrupt only specific
CPUs that have useful work to do and not the entire system.
This patch set, inspired by discussions with Peter Zijlstra and Frederic
Weisbecker when testing the nohz task patch set, is a first stab at trying
to explore doing this by locating the places where such global IPI calls
are being made and turning the global IPI into an IPI for a specific group
of CPUs. The purpose of the patch set is to get feedback if this is the
right way to go for dealing with this issue and indeed, if the issue is
even worth dealing with at all. Based on the feedback from this patch set
I plan to offer further patches that address similar issue in other code
paths.
This patch creates an on_each_cpu_mask() and on_each_cpu_cond()
infrastructure API (the former derived from existing arch specific
versions in Tile and Arm) and uses them to turn several global IPI
invocation to per CPU group invocations.
Core kernel:
on_each_cpu_mask() calls a function on processors specified by cpumask,
which may or may not include the local processor.
You must not call this function with disabled interrupts or from a
hardware interrupt handler or from a bottom half handler.
arch/arm:
Note that the generic version is a little different then the Arm one:
1. It has the mask as first parameter
2. It calls the function on the calling CPU with interrupts disabled,
but this should be OK since the function is called on the other CPUs
with interrupts disabled anyway.
arch/tile:
The API is the same as the tile private one, but the generic version
also calls the function on the with interrupts disabled in UP case
This is OK since the function is called on the other CPUs
with interrupts disabled.
Signed-off-by: Gilad Ben-Yossef <gilad@benyossef.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Acked-by: Chris Metcalf <cmetcalf@tilera.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Matt Mackall <mpm@selenic.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Sasha Levin <levinsasha928@gmail.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Avi Kivity <avi@redhat.com>
Acked-by: Michal Nazarewicz <mina86@mina86.org>
Cc: Kosaki Motohiro <kosaki.motohiro@gmail.com>
Cc: Milton Miller <miltonm@bga.com>
Cc: Russell King <linux@arm.linux.org.uk>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-28 15:42:43 -06:00
|
|
|
func(info);
|
2013-09-11 15:23:29 -06:00
|
|
|
local_irq_restore(flags);
|
smp: introduce a generic on_each_cpu_mask() function
We have lots of infrastructure in place to partition multi-core systems
such that we have a group of CPUs that are dedicated to specific task:
cgroups, scheduler and interrupt affinity, and cpuisol= boot parameter.
Still, kernel code will at times interrupt all CPUs in the system via IPIs
for various needs. These IPIs are useful and cannot be avoided
altogether, but in certain cases it is possible to interrupt only specific
CPUs that have useful work to do and not the entire system.
This patch set, inspired by discussions with Peter Zijlstra and Frederic
Weisbecker when testing the nohz task patch set, is a first stab at trying
to explore doing this by locating the places where such global IPI calls
are being made and turning the global IPI into an IPI for a specific group
of CPUs. The purpose of the patch set is to get feedback if this is the
right way to go for dealing with this issue and indeed, if the issue is
even worth dealing with at all. Based on the feedback from this patch set
I plan to offer further patches that address similar issue in other code
paths.
This patch creates an on_each_cpu_mask() and on_each_cpu_cond()
infrastructure API (the former derived from existing arch specific
versions in Tile and Arm) and uses them to turn several global IPI
invocation to per CPU group invocations.
Core kernel:
on_each_cpu_mask() calls a function on processors specified by cpumask,
which may or may not include the local processor.
You must not call this function with disabled interrupts or from a
hardware interrupt handler or from a bottom half handler.
arch/arm:
Note that the generic version is a little different then the Arm one:
1. It has the mask as first parameter
2. It calls the function on the calling CPU with interrupts disabled,
but this should be OK since the function is called on the other CPUs
with interrupts disabled anyway.
arch/tile:
The API is the same as the tile private one, but the generic version
also calls the function on the with interrupts disabled in UP case
This is OK since the function is called on the other CPUs
with interrupts disabled.
Signed-off-by: Gilad Ben-Yossef <gilad@benyossef.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Acked-by: Chris Metcalf <cmetcalf@tilera.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Matt Mackall <mpm@selenic.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Sasha Levin <levinsasha928@gmail.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Avi Kivity <avi@redhat.com>
Acked-by: Michal Nazarewicz <mina86@mina86.org>
Cc: Kosaki Motohiro <kosaki.motohiro@gmail.com>
Cc: Milton Miller <miltonm@bga.com>
Cc: Russell King <linux@arm.linux.org.uk>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-28 15:42:43 -06:00
|
|
|
}
|
|
|
|
put_cpu();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(on_each_cpu_mask);
|
2012-03-28 15:42:43 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* on_each_cpu_cond(): Call a function on each processor for which
|
|
|
|
* the supplied function cond_func returns true, optionally waiting
|
|
|
|
* for all the required CPUs to finish. This may include the local
|
|
|
|
* processor.
|
|
|
|
* @cond_func: A callback function that is passed a cpu id and
|
|
|
|
* the the info parameter. The function is called
|
|
|
|
* with preemption disabled. The function should
|
|
|
|
* return a blooean value indicating whether to IPI
|
|
|
|
* the specified CPU.
|
|
|
|
* @func: The function to run on all applicable CPUs.
|
|
|
|
* This must be fast and non-blocking.
|
|
|
|
* @info: An arbitrary pointer to pass to both functions.
|
|
|
|
* @wait: If true, wait (atomically) until function has
|
|
|
|
* completed on other CPUs.
|
|
|
|
* @gfp_flags: GFP flags to use when allocating the cpumask
|
|
|
|
* used internally by the function.
|
|
|
|
*
|
|
|
|
* The function might sleep if the GFP flags indicates a non
|
|
|
|
* atomic allocation is allowed.
|
|
|
|
*
|
|
|
|
* Preemption is disabled to protect against CPUs going offline but not online.
|
|
|
|
* CPUs going online during the call will not be seen or sent an IPI.
|
|
|
|
*
|
|
|
|
* You must not call this function with disabled interrupts or
|
|
|
|
* from a hardware interrupt handler or from a bottom half handler.
|
|
|
|
*/
|
|
|
|
void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info),
|
|
|
|
smp_call_func_t func, void *info, bool wait,
|
|
|
|
gfp_t gfp_flags)
|
|
|
|
{
|
|
|
|
cpumask_var_t cpus;
|
|
|
|
int cpu, ret;
|
|
|
|
|
2015-11-06 17:28:21 -07:00
|
|
|
might_sleep_if(gfpflags_allow_blocking(gfp_flags));
|
2012-03-28 15:42:43 -06:00
|
|
|
|
|
|
|
if (likely(zalloc_cpumask_var(&cpus, (gfp_flags|__GFP_NOWARN)))) {
|
|
|
|
preempt_disable();
|
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
if (cond_func(cpu, info))
|
|
|
|
cpumask_set_cpu(cpu, cpus);
|
|
|
|
on_each_cpu_mask(cpus, func, info, wait);
|
|
|
|
preempt_enable();
|
|
|
|
free_cpumask_var(cpus);
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* No free cpumask, bother. No matter, we'll
|
|
|
|
* just have to IPI them one by one.
|
|
|
|
*/
|
|
|
|
preempt_disable();
|
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
if (cond_func(cpu, info)) {
|
|
|
|
ret = smp_call_function_single(cpu, func,
|
|
|
|
info, wait);
|
2014-08-06 17:08:14 -06:00
|
|
|
WARN_ON_ONCE(ret);
|
2012-03-28 15:42:43 -06:00
|
|
|
}
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(on_each_cpu_cond);
|
2012-05-07 11:59:48 -06:00
|
|
|
|
|
|
|
static void do_nothing(void *unused)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* kick_all_cpus_sync - Force all cpus out of idle
|
|
|
|
*
|
|
|
|
* Used to synchronize the update of pm_idle function pointer. It's
|
|
|
|
* called after the pointer is updated and returns after the dummy
|
|
|
|
* callback function has been executed on all cpus. The execution of
|
|
|
|
* the function can only happen on the remote cpus after they have
|
|
|
|
* left the idle function which had been called via pm_idle function
|
|
|
|
* pointer. So it's guaranteed that nothing uses the previous pointer
|
|
|
|
* anymore.
|
|
|
|
*/
|
|
|
|
void kick_all_cpus_sync(void)
|
|
|
|
{
|
|
|
|
/* Make sure the change is visible before we kick the cpus */
|
|
|
|
smp_mb();
|
|
|
|
smp_call_function(do_nothing, NULL, 1);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(kick_all_cpus_sync);
|
2014-09-04 01:17:54 -06:00
|
|
|
|
|
|
|
/**
|
|
|
|
* wake_up_all_idle_cpus - break all cpus out of idle
|
|
|
|
* wake_up_all_idle_cpus try to break all cpus which is in idle state even
|
|
|
|
* including idle polling cpus, for non-idle cpus, we will do nothing
|
|
|
|
* for them.
|
|
|
|
*/
|
|
|
|
void wake_up_all_idle_cpus(void)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
preempt_disable();
|
|
|
|
for_each_online_cpu(cpu) {
|
|
|
|
if (cpu == smp_processor_id())
|
|
|
|
continue;
|
|
|
|
|
|
|
|
wake_up_if_idle(cpu);
|
|
|
|
}
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(wake_up_all_idle_cpus);
|
2016-08-29 00:48:44 -06:00
|
|
|
|
|
|
|
/**
|
|
|
|
* smp_call_on_cpu - Call a function on a specific cpu
|
|
|
|
*
|
|
|
|
* Used to call a function on a specific cpu and wait for it to return.
|
|
|
|
* Optionally make sure the call is done on a specified physical cpu via vcpu
|
|
|
|
* pinning in order to support virtualized environments.
|
|
|
|
*/
|
|
|
|
struct smp_call_on_cpu_struct {
|
|
|
|
struct work_struct work;
|
|
|
|
struct completion done;
|
|
|
|
int (*func)(void *);
|
|
|
|
void *data;
|
|
|
|
int ret;
|
|
|
|
int cpu;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void smp_call_on_cpu_callback(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct smp_call_on_cpu_struct *sscs;
|
|
|
|
|
|
|
|
sscs = container_of(work, struct smp_call_on_cpu_struct, work);
|
|
|
|
if (sscs->cpu >= 0)
|
|
|
|
hypervisor_pin_vcpu(sscs->cpu);
|
|
|
|
sscs->ret = sscs->func(sscs->data);
|
|
|
|
if (sscs->cpu >= 0)
|
|
|
|
hypervisor_pin_vcpu(-1);
|
|
|
|
|
|
|
|
complete(&sscs->done);
|
|
|
|
}
|
|
|
|
|
|
|
|
int smp_call_on_cpu(unsigned int cpu, int (*func)(void *), void *par, bool phys)
|
|
|
|
{
|
|
|
|
struct smp_call_on_cpu_struct sscs = {
|
|
|
|
.done = COMPLETION_INITIALIZER_ONSTACK(sscs.done),
|
|
|
|
.func = func,
|
|
|
|
.data = par,
|
|
|
|
.cpu = phys ? cpu : -1,
|
|
|
|
};
|
|
|
|
|
2016-09-11 02:36:26 -06:00
|
|
|
INIT_WORK_ONSTACK(&sscs.work, smp_call_on_cpu_callback);
|
|
|
|
|
2016-08-29 00:48:44 -06:00
|
|
|
if (cpu >= nr_cpu_ids || !cpu_online(cpu))
|
|
|
|
return -ENXIO;
|
|
|
|
|
|
|
|
queue_work_on(cpu, system_wq, &sscs.work);
|
|
|
|
wait_for_completion(&sscs.done);
|
|
|
|
|
|
|
|
return sscs.ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(smp_call_on_cpu);
|