2006-01-09 16:59:19 -07:00
|
|
|
/*
|
|
|
|
* kernel/mutex.c
|
|
|
|
*
|
|
|
|
* Mutexes: blocking mutual exclusion locks
|
|
|
|
*
|
|
|
|
* Started by Ingo Molnar:
|
|
|
|
*
|
|
|
|
* Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
|
|
|
|
*
|
|
|
|
* Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
|
|
|
|
* David Howells for suggestions and improvements.
|
|
|
|
*
|
2009-01-12 06:01:47 -07:00
|
|
|
* - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline
|
|
|
|
* from the -rt tree, where it was originally implemented for rtmutexes
|
|
|
|
* by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
|
|
|
|
* and Sven Dietrich.
|
|
|
|
*
|
2006-01-09 16:59:19 -07:00
|
|
|
* Also see Documentation/mutex-design.txt.
|
|
|
|
*/
|
|
|
|
#include <linux/mutex.h>
|
2013-07-05 01:29:32 -06:00
|
|
|
#include <linux/ww_mutex.h>
|
2006-01-09 16:59:19 -07:00
|
|
|
#include <linux/sched.h>
|
2013-02-07 08:47:07 -07:00
|
|
|
#include <linux/sched/rt.h>
|
2011-05-23 12:51:41 -06:00
|
|
|
#include <linux/export.h>
|
2006-01-09 16:59:19 -07:00
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/interrupt.h>
|
2006-07-03 01:24:33 -06:00
|
|
|
#include <linux/debug_locks.h>
|
2006-01-09 16:59:19 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* In the DEBUG case we are using the "NULL fastpath" for mutexes,
|
|
|
|
* which forces all calls into the slowpath:
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
# include "mutex-debug.h"
|
|
|
|
# include <asm-generic/mutex-null.h>
|
|
|
|
#else
|
|
|
|
# include "mutex.h"
|
|
|
|
# include <asm/mutex.h>
|
|
|
|
#endif
|
|
|
|
|
mutex: Make more scalable by doing less atomic operations
In the __mutex_lock_common() function, an initial entry into
the lock slow path will cause two atomic_xchg instructions to be
issued. Together with the atomic decrement in the fast path, a
total of three atomic read-modify-write instructions will be
issued in rapid succession. This can cause a lot of cache
bouncing when many tasks are trying to acquire the mutex at the
same time.
This patch will reduce the number of atomic_xchg instructions
used by checking the counter value first before issuing the
instruction. The atomic_read() function is just a simple memory
read. The atomic_xchg() function, on the other hand, can be up
to 2 order of magnitude or even more in cost when compared with
atomic_read(). By using atomic_read() to check the value first
before calling atomic_xchg(), we can avoid a lot of unnecessary
cache coherency traffic. The only downside with this change is
that a task on the slow path will have a tiny bit less chance of
getting the mutex when competing with another task in the fast
path.
The same is true for the atomic_cmpxchg() function in the
mutex-spin-on-owner loop. So an atomic_read() is also performed
before calling atomic_cmpxchg().
The mutex locking and unlocking code for the x86 architecture
can allow any negative number to be used in the mutex count to
indicate that some tasks are waiting for the mutex. I am not so
sure if that is the case for the other architectures. So the
default is to avoid atomic_xchg() if the count has already been
set to -1. For x86, the check is modified to include all
negative numbers to cover a larger case.
The following table shows the jobs per minutes (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the
high_systime workloads to 1/2/4/8 nodes with hyperthreading on
and off.
+-----------------+-----------+------------+----------+
| Configuration | Mean JPM | Mean JPM | % Change |
| | w/o patch | with patch | |
+-----------------+-----------------------------------+
| | User Range 1100 - 2000 |
+-----------------+-----------------------------------+
| 8 nodes, HT on | 36980 | 148590 | +301.8% |
| 8 nodes, HT off | 42799 | 145011 | +238.8% |
| 4 nodes, HT on | 61318 | 118445 | +51.1% |
| 4 nodes, HT off | 158481 | 158592 | +0.1% |
| 2 nodes, HT on | 180602 | 173967 | -3.7% |
| 2 nodes, HT off | 198409 | 198073 | -0.2% |
| 1 node , HT on | 149042 | 147671 | -0.9% |
| 1 node , HT off | 126036 | 126533 | +0.4% |
+-----------------+-----------------------------------+
| | User Range 200 - 1000 |
+-----------------+-----------------------------------+
| 8 nodes, HT on | 41525 | 122349 | +194.6% |
| 8 nodes, HT off | 49866 | 124032 | +148.7% |
| 4 nodes, HT on | 66409 | 106984 | +61.1% |
| 4 nodes, HT off | 119880 | 130508 | +8.9% |
| 2 nodes, HT on | 138003 | 133948 | -2.9% |
| 2 nodes, HT off | 132792 | 131997 | -0.6% |
| 1 node , HT on | 116593 | 115859 | -0.6% |
| 1 node , HT off | 104499 | 104597 | +0.1% |
+-----------------+------------+-----------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
AIM7 benchmark run has a pretty large run-to-run variance due to
random nature of the subtests executed. So a difference of less
than +-5% may not be really significant.
This patch improves high_systime workload performance at 4 nodes
and up by maintaining transaction rates without significant
drop-off at high node count. The patch has practically no
impact on 1 and 2 nodes system.
The table below shows the percentage time (as reported by perf
record -a -s -g) spent on the __mutex_lock_slowpath() function
by the high_systime workload at 1500 users for 2/4/8-node
configurations with hyperthreading off.
+---------------+-----------------+------------------+---------+
| Configuration | %Time w/o patch | %Time with patch | %Change |
+---------------+-----------------+------------------+---------+
| 8 nodes | 65.34% | 0.69% | -99% |
| 4 nodes | 8.70% | 1.02% | -88% |
| 2 nodes | 0.41% | 0.32% | -22% |
+---------------+-----------------+------------------+---------+
It is obvious that the dramatic performance improvement at 8
nodes was due to the drastic cut in the time spent within the
__mutex_lock_slowpath() function.
The table below show the improvements in other AIM7 workloads
(at 8 nodes, hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | +0.6% | +104.2% | +185.9% |
| five_sec | +1.9% | +0.9% | +0.9% |
| fserver | +1.4% | -7.7% | +5.1% |
| new_fserver | -0.5% | +3.2% | +3.1% |
| shared | +13.1% | +146.1% | +181.5% |
| short | +7.4% | +5.0% | +4.2% |
+--------------+---------------+----------------+-----------------+
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton: Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-3-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:12 -06:00
|
|
|
/*
|
2013-04-17 13:23:14 -06:00
|
|
|
* A negative mutex count indicates that waiters are sleeping waiting for the
|
|
|
|
* mutex.
|
mutex: Make more scalable by doing less atomic operations
In the __mutex_lock_common() function, an initial entry into
the lock slow path will cause two atomic_xchg instructions to be
issued. Together with the atomic decrement in the fast path, a
total of three atomic read-modify-write instructions will be
issued in rapid succession. This can cause a lot of cache
bouncing when many tasks are trying to acquire the mutex at the
same time.
This patch will reduce the number of atomic_xchg instructions
used by checking the counter value first before issuing the
instruction. The atomic_read() function is just a simple memory
read. The atomic_xchg() function, on the other hand, can be up
to 2 order of magnitude or even more in cost when compared with
atomic_read(). By using atomic_read() to check the value first
before calling atomic_xchg(), we can avoid a lot of unnecessary
cache coherency traffic. The only downside with this change is
that a task on the slow path will have a tiny bit less chance of
getting the mutex when competing with another task in the fast
path.
The same is true for the atomic_cmpxchg() function in the
mutex-spin-on-owner loop. So an atomic_read() is also performed
before calling atomic_cmpxchg().
The mutex locking and unlocking code for the x86 architecture
can allow any negative number to be used in the mutex count to
indicate that some tasks are waiting for the mutex. I am not so
sure if that is the case for the other architectures. So the
default is to avoid atomic_xchg() if the count has already been
set to -1. For x86, the check is modified to include all
negative numbers to cover a larger case.
The following table shows the jobs per minutes (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the
high_systime workloads to 1/2/4/8 nodes with hyperthreading on
and off.
+-----------------+-----------+------------+----------+
| Configuration | Mean JPM | Mean JPM | % Change |
| | w/o patch | with patch | |
+-----------------+-----------------------------------+
| | User Range 1100 - 2000 |
+-----------------+-----------------------------------+
| 8 nodes, HT on | 36980 | 148590 | +301.8% |
| 8 nodes, HT off | 42799 | 145011 | +238.8% |
| 4 nodes, HT on | 61318 | 118445 | +51.1% |
| 4 nodes, HT off | 158481 | 158592 | +0.1% |
| 2 nodes, HT on | 180602 | 173967 | -3.7% |
| 2 nodes, HT off | 198409 | 198073 | -0.2% |
| 1 node , HT on | 149042 | 147671 | -0.9% |
| 1 node , HT off | 126036 | 126533 | +0.4% |
+-----------------+-----------------------------------+
| | User Range 200 - 1000 |
+-----------------+-----------------------------------+
| 8 nodes, HT on | 41525 | 122349 | +194.6% |
| 8 nodes, HT off | 49866 | 124032 | +148.7% |
| 4 nodes, HT on | 66409 | 106984 | +61.1% |
| 4 nodes, HT off | 119880 | 130508 | +8.9% |
| 2 nodes, HT on | 138003 | 133948 | -2.9% |
| 2 nodes, HT off | 132792 | 131997 | -0.6% |
| 1 node , HT on | 116593 | 115859 | -0.6% |
| 1 node , HT off | 104499 | 104597 | +0.1% |
+-----------------+------------+-----------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
AIM7 benchmark run has a pretty large run-to-run variance due to
random nature of the subtests executed. So a difference of less
than +-5% may not be really significant.
This patch improves high_systime workload performance at 4 nodes
and up by maintaining transaction rates without significant
drop-off at high node count. The patch has practically no
impact on 1 and 2 nodes system.
The table below shows the percentage time (as reported by perf
record -a -s -g) spent on the __mutex_lock_slowpath() function
by the high_systime workload at 1500 users for 2/4/8-node
configurations with hyperthreading off.
+---------------+-----------------+------------------+---------+
| Configuration | %Time w/o patch | %Time with patch | %Change |
+---------------+-----------------+------------------+---------+
| 8 nodes | 65.34% | 0.69% | -99% |
| 4 nodes | 8.70% | 1.02% | -88% |
| 2 nodes | 0.41% | 0.32% | -22% |
+---------------+-----------------+------------------+---------+
It is obvious that the dramatic performance improvement at 8
nodes was due to the drastic cut in the time spent within the
__mutex_lock_slowpath() function.
The table below show the improvements in other AIM7 workloads
(at 8 nodes, hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | +0.6% | +104.2% | +185.9% |
| five_sec | +1.9% | +0.9% | +0.9% |
| fserver | +1.4% | -7.7% | +5.1% |
| new_fserver | -0.5% | +3.2% | +3.1% |
| shared | +13.1% | +146.1% | +181.5% |
| short | +7.4% | +5.0% | +4.2% |
+--------------+---------------+----------------+-----------------+
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton: Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-3-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:12 -06:00
|
|
|
*/
|
|
|
|
#define MUTEX_SHOW_NO_WAITER(mutex) (atomic_read(&(mutex)->count) >= 0)
|
|
|
|
|
2006-07-03 01:24:55 -06:00
|
|
|
void
|
|
|
|
__mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
|
2006-01-09 16:59:19 -07:00
|
|
|
{
|
|
|
|
atomic_set(&lock->count, 1);
|
|
|
|
spin_lock_init(&lock->wait_lock);
|
|
|
|
INIT_LIST_HEAD(&lock->wait_list);
|
2009-01-12 06:01:47 -07:00
|
|
|
mutex_clear_owner(lock);
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
|
|
|
|
lock->spin_mlock = NULL;
|
|
|
|
#endif
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2006-07-03 01:24:55 -06:00
|
|
|
debug_mutex_init(lock, name, key);
|
2006-01-09 16:59:19 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(__mutex_init);
|
|
|
|
|
2007-10-11 14:11:12 -06:00
|
|
|
#ifndef CONFIG_DEBUG_LOCK_ALLOC
|
2006-01-09 16:59:19 -07:00
|
|
|
/*
|
|
|
|
* We split the mutex lock/unlock logic into separate fastpath and
|
|
|
|
* slowpath functions, to reduce the register pressure on the fastpath.
|
|
|
|
* We also put the fastpath first in the kernel image, to make sure the
|
|
|
|
* branch is predicted by the CPU as default-untaken.
|
|
|
|
*/
|
2008-11-24 01:17:42 -07:00
|
|
|
static __used noinline void __sched
|
2006-07-03 01:24:33 -06:00
|
|
|
__mutex_lock_slowpath(atomic_t *lock_count);
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2010-09-02 16:48:16 -06:00
|
|
|
/**
|
2006-01-09 16:59:19 -07:00
|
|
|
* mutex_lock - acquire the mutex
|
|
|
|
* @lock: the mutex to be acquired
|
|
|
|
*
|
|
|
|
* Lock the mutex exclusively for this task. If the mutex is not
|
|
|
|
* available right now, it will sleep until it can get it.
|
|
|
|
*
|
|
|
|
* The mutex must later on be released by the same task that
|
|
|
|
* acquired it. Recursive locking is not allowed. The task
|
|
|
|
* may not exit without first unlocking the mutex. Also, kernel
|
|
|
|
* memory where the mutex resides mutex must not be freed with
|
|
|
|
* the mutex still locked. The mutex must first be initialized
|
|
|
|
* (or statically defined) before it can be locked. memset()-ing
|
|
|
|
* the mutex to 0 is not allowed.
|
|
|
|
*
|
|
|
|
* ( The CONFIG_DEBUG_MUTEXES .config option turns on debugging
|
|
|
|
* checks that will enforce the restrictions and will also do
|
|
|
|
* deadlock debugging. )
|
|
|
|
*
|
|
|
|
* This function is similar to (but not equivalent to) down().
|
|
|
|
*/
|
2009-04-01 18:21:56 -06:00
|
|
|
void __sched mutex_lock(struct mutex *lock)
|
2006-01-09 16:59:19 -07:00
|
|
|
{
|
2006-01-10 14:10:36 -07:00
|
|
|
might_sleep();
|
2006-01-09 16:59:19 -07:00
|
|
|
/*
|
|
|
|
* The locking fastpath is the 1->0 transition from
|
|
|
|
* 'unlocked' into 'locked' state.
|
|
|
|
*/
|
|
|
|
__mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);
|
2009-01-12 06:01:47 -07:00
|
|
|
mutex_set_owner(lock);
|
2006-01-09 16:59:19 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(mutex_lock);
|
2007-10-11 14:11:12 -06:00
|
|
|
#endif
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2013-04-17 13:23:11 -06:00
|
|
|
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
/*
|
|
|
|
* In order to avoid a stampede of mutex spinners from acquiring the mutex
|
|
|
|
* more or less simultaneously, the spinners need to acquire a MCS lock
|
|
|
|
* first before spinning on the owner field.
|
|
|
|
*
|
|
|
|
* We don't inline mspin_lock() so that perf can correctly account for the
|
|
|
|
* time spent in this lock function.
|
|
|
|
*/
|
|
|
|
struct mspin_node {
|
|
|
|
struct mspin_node *next ;
|
|
|
|
int locked; /* 1 if lock acquired */
|
|
|
|
};
|
|
|
|
#define MLOCK(mutex) ((struct mspin_node **)&((mutex)->spin_mlock))
|
|
|
|
|
|
|
|
static noinline
|
|
|
|
void mspin_lock(struct mspin_node **lock, struct mspin_node *node)
|
|
|
|
{
|
|
|
|
struct mspin_node *prev;
|
|
|
|
|
|
|
|
/* Init node */
|
|
|
|
node->locked = 0;
|
|
|
|
node->next = NULL;
|
|
|
|
|
|
|
|
prev = xchg(lock, node);
|
|
|
|
if (likely(prev == NULL)) {
|
|
|
|
/* Lock acquired */
|
|
|
|
node->locked = 1;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
ACCESS_ONCE(prev->next) = node;
|
|
|
|
smp_wmb();
|
|
|
|
/* Wait until the lock holder passes the lock down */
|
|
|
|
while (!ACCESS_ONCE(node->locked))
|
|
|
|
arch_mutex_cpu_relax();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mspin_unlock(struct mspin_node **lock, struct mspin_node *node)
|
|
|
|
{
|
|
|
|
struct mspin_node *next = ACCESS_ONCE(node->next);
|
|
|
|
|
|
|
|
if (likely(!next)) {
|
|
|
|
/*
|
|
|
|
* Release the lock by setting it to NULL
|
|
|
|
*/
|
|
|
|
if (cmpxchg(lock, node, NULL) == node)
|
|
|
|
return;
|
|
|
|
/* Wait until the next pointer is set */
|
|
|
|
while (!(next = ACCESS_ONCE(node->next)))
|
|
|
|
arch_mutex_cpu_relax();
|
|
|
|
}
|
|
|
|
ACCESS_ONCE(next->locked) = 1;
|
|
|
|
smp_wmb();
|
|
|
|
}
|
|
|
|
|
2013-04-17 13:23:11 -06:00
|
|
|
/*
|
|
|
|
* Mutex spinning code migrated from kernel/sched/core.c
|
|
|
|
*/
|
|
|
|
|
|
|
|
static inline bool owner_running(struct mutex *lock, struct task_struct *owner)
|
|
|
|
{
|
|
|
|
if (lock->owner != owner)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ensure we emit the owner->on_cpu, dereference _after_ checking
|
|
|
|
* lock->owner still matches owner, if that fails, owner might
|
|
|
|
* point to free()d memory, if it still matches, the rcu_read_lock()
|
|
|
|
* ensures the memory stays valid.
|
|
|
|
*/
|
|
|
|
barrier();
|
|
|
|
|
|
|
|
return owner->on_cpu;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Look out! "owner" is an entirely speculative pointer
|
|
|
|
* access and not reliable.
|
|
|
|
*/
|
|
|
|
static noinline
|
|
|
|
int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
|
|
|
|
{
|
|
|
|
rcu_read_lock();
|
|
|
|
while (owner_running(lock, owner)) {
|
|
|
|
if (need_resched())
|
|
|
|
break;
|
|
|
|
|
|
|
|
arch_mutex_cpu_relax();
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We break out the loop above on need_resched() and when the
|
|
|
|
* owner changed, which is a sign for heavy contention. Return
|
|
|
|
* success only when lock->owner is NULL.
|
|
|
|
*/
|
|
|
|
return lock->owner == NULL;
|
|
|
|
}
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Initial check for entering the mutex spinning loop
|
|
|
|
*/
|
|
|
|
static inline int mutex_can_spin_on_owner(struct mutex *lock)
|
|
|
|
{
|
2013-07-19 12:31:01 -06:00
|
|
|
struct task_struct *owner;
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
int retval = 1;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2013-07-19 12:31:01 -06:00
|
|
|
owner = ACCESS_ONCE(lock->owner);
|
|
|
|
if (owner)
|
|
|
|
retval = owner->on_cpu;
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
rcu_read_unlock();
|
|
|
|
/*
|
|
|
|
* if lock->owner is not set, the mutex owner may have just acquired
|
|
|
|
* it and not set the owner yet or the mutex has been released.
|
|
|
|
*/
|
|
|
|
return retval;
|
|
|
|
}
|
2013-04-17 13:23:11 -06:00
|
|
|
#endif
|
|
|
|
|
2008-11-24 01:17:42 -07:00
|
|
|
static __used noinline void __sched __mutex_unlock_slowpath(atomic_t *lock_count);
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2010-09-02 16:48:16 -06:00
|
|
|
/**
|
2006-01-09 16:59:19 -07:00
|
|
|
* mutex_unlock - release the mutex
|
|
|
|
* @lock: the mutex to be released
|
|
|
|
*
|
|
|
|
* Unlock a mutex that has been locked by this task previously.
|
|
|
|
*
|
|
|
|
* This function must not be used in interrupt context. Unlocking
|
|
|
|
* of a not locked mutex is not allowed.
|
|
|
|
*
|
|
|
|
* This function is similar to (but not equivalent to) up().
|
|
|
|
*/
|
2008-02-08 05:19:53 -07:00
|
|
|
void __sched mutex_unlock(struct mutex *lock)
|
2006-01-09 16:59:19 -07:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The unlocking fastpath is the 0->1 transition from 'locked'
|
|
|
|
* into 'unlocked' state:
|
|
|
|
*/
|
2009-01-12 06:01:47 -07:00
|
|
|
#ifndef CONFIG_DEBUG_MUTEXES
|
|
|
|
/*
|
|
|
|
* When debugging is enabled we must not clear the owner before time,
|
|
|
|
* the slow path will always be taken, and that clears the owner field
|
|
|
|
* after verifying that it was indeed current.
|
|
|
|
*/
|
|
|
|
mutex_clear_owner(lock);
|
|
|
|
#endif
|
2006-01-09 16:59:19 -07:00
|
|
|
__mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);
|
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(mutex_unlock);
|
|
|
|
|
2013-06-24 02:30:04 -06:00
|
|
|
/**
|
|
|
|
* ww_mutex_unlock - release the w/w mutex
|
|
|
|
* @lock: the mutex to be released
|
|
|
|
*
|
|
|
|
* Unlock a mutex that has been locked by this task previously with any of the
|
|
|
|
* ww_mutex_lock* functions (with or without an acquire context). It is
|
|
|
|
* forbidden to release the locks after releasing the acquire context.
|
|
|
|
*
|
|
|
|
* This function must not be used in interrupt context. Unlocking
|
|
|
|
* of a unlocked mutex is not allowed.
|
|
|
|
*/
|
|
|
|
void __sched ww_mutex_unlock(struct ww_mutex *lock)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The unlocking fastpath is the 0->1 transition from 'locked'
|
|
|
|
* into 'unlocked' state:
|
|
|
|
*/
|
|
|
|
if (lock->ctx) {
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);
|
|
|
|
#endif
|
|
|
|
if (lock->ctx->acquired > 0)
|
|
|
|
lock->ctx->acquired--;
|
|
|
|
lock->ctx = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef CONFIG_DEBUG_MUTEXES
|
|
|
|
/*
|
|
|
|
* When debugging is enabled we must not clear the owner before time,
|
|
|
|
* the slow path will always be taken, and that clears the owner field
|
|
|
|
* after verifying that it was indeed current.
|
|
|
|
*/
|
|
|
|
mutex_clear_owner(&lock->base);
|
|
|
|
#endif
|
|
|
|
__mutex_fastpath_unlock(&lock->base.count, __mutex_unlock_slowpath);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(ww_mutex_unlock);
|
|
|
|
|
|
|
|
static inline int __sched
|
|
|
|
__mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
|
|
|
|
struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx);
|
|
|
|
|
|
|
|
if (!hold_ctx)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (unlikely(ctx == hold_ctx))
|
|
|
|
return -EALREADY;
|
|
|
|
|
|
|
|
if (ctx->stamp - hold_ctx->stamp <= LONG_MAX &&
|
|
|
|
(ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) {
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
DEBUG_LOCKS_WARN_ON(ctx->contending_lock);
|
|
|
|
ctx->contending_lock = ww;
|
|
|
|
#endif
|
|
|
|
return -EDEADLK;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
|
|
|
|
struct ww_acquire_ctx *ww_ctx)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
/*
|
|
|
|
* If this WARN_ON triggers, you used ww_mutex_lock to acquire,
|
|
|
|
* but released with a normal mutex_unlock in this call.
|
|
|
|
*
|
|
|
|
* This should never happen, always use ww_mutex_unlock.
|
|
|
|
*/
|
|
|
|
DEBUG_LOCKS_WARN_ON(ww->ctx);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Not quite done after calling ww_acquire_done() ?
|
|
|
|
*/
|
|
|
|
DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
|
|
|
|
|
|
|
|
if (ww_ctx->contending_lock) {
|
|
|
|
/*
|
|
|
|
* After -EDEADLK you tried to
|
|
|
|
* acquire a different ww_mutex? Bad!
|
|
|
|
*/
|
|
|
|
DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* You called ww_mutex_lock after receiving -EDEADLK,
|
|
|
|
* but 'forgot' to unlock everything else first?
|
|
|
|
*/
|
|
|
|
DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
|
|
|
|
ww_ctx->contending_lock = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Naughty, using a different class will lead to undefined behavior!
|
|
|
|
*/
|
|
|
|
DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
|
|
|
|
#endif
|
|
|
|
ww_ctx->acquired++;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* after acquiring lock with fastpath or when we lost out in contested
|
|
|
|
* slowpath, set ctx and wake up any waiters so they can recheck.
|
|
|
|
*
|
|
|
|
* This function is never called when CONFIG_DEBUG_LOCK_ALLOC is set,
|
|
|
|
* as the fastpath and opportunistic spinning are disabled in that case.
|
|
|
|
*/
|
|
|
|
static __always_inline void
|
|
|
|
ww_mutex_set_context_fastpath(struct ww_mutex *lock,
|
|
|
|
struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
struct mutex_waiter *cur;
|
|
|
|
|
|
|
|
ww_mutex_lock_acquired(lock, ctx);
|
|
|
|
|
|
|
|
lock->ctx = ctx;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The lock->ctx update should be visible on all cores before
|
|
|
|
* the atomic read is done, otherwise contended waiters might be
|
|
|
|
* missed. The contended waiters will either see ww_ctx == NULL
|
|
|
|
* and keep spinning, or it will acquire wait_lock, add itself
|
|
|
|
* to waiter list and sleep.
|
|
|
|
*/
|
|
|
|
smp_mb(); /* ^^^ */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check if lock is contended, if not there is nobody to wake up
|
|
|
|
*/
|
|
|
|
if (likely(atomic_read(&lock->base.count) == 0))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Uh oh, we raced in fastpath, wake up everyone in this case,
|
|
|
|
* so they can see the new lock->ctx.
|
|
|
|
*/
|
|
|
|
spin_lock_mutex(&lock->base.wait_lock, flags);
|
|
|
|
list_for_each_entry(cur, &lock->base.wait_list, list) {
|
|
|
|
debug_mutex_wake_waiter(&lock->base, cur);
|
|
|
|
wake_up_process(cur->task);
|
|
|
|
}
|
|
|
|
spin_unlock_mutex(&lock->base.wait_lock, flags);
|
|
|
|
}
|
|
|
|
|
2006-01-09 16:59:19 -07:00
|
|
|
/*
|
|
|
|
* Lock a mutex (possibly interruptible), slowpath:
|
|
|
|
*/
|
2013-06-24 02:30:04 -06:00
|
|
|
static __always_inline int __sched
|
2007-10-11 14:11:12 -06:00
|
|
|
__mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
|
2013-06-24 02:30:04 -06:00
|
|
|
struct lockdep_map *nest_lock, unsigned long ip,
|
2013-10-17 04:45:29 -06:00
|
|
|
struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
|
2006-01-09 16:59:19 -07:00
|
|
|
{
|
|
|
|
struct task_struct *task = current;
|
|
|
|
struct mutex_waiter waiter;
|
2006-06-26 01:24:31 -06:00
|
|
|
unsigned long flags;
|
2013-06-24 02:30:04 -06:00
|
|
|
int ret;
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2009-01-14 07:36:26 -07:00
|
|
|
preempt_disable();
|
2011-05-24 18:12:03 -06:00
|
|
|
mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
|
2009-12-02 12:49:16 -07:00
|
|
|
|
|
|
|
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
|
2009-01-12 06:01:47 -07:00
|
|
|
/*
|
|
|
|
* Optimistic spinning.
|
|
|
|
*
|
|
|
|
* We try to spin for acquisition when we find that there are no
|
|
|
|
* pending waiters and the lock owner is currently running on a
|
|
|
|
* (different) CPU.
|
|
|
|
*
|
|
|
|
* The rationale is that if the lock owner is running, it is likely to
|
|
|
|
* release the lock soon.
|
|
|
|
*
|
|
|
|
* Since this needs the lock owner, and this mutex implementation
|
|
|
|
* doesn't track the owner atomically in the lock field, we need to
|
|
|
|
* track it non-atomically.
|
|
|
|
*
|
|
|
|
* We can't do this for DEBUG_MUTEXES because that relies on wait_lock
|
|
|
|
* to serialize everything.
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
*
|
|
|
|
* The mutex spinners are queued up using MCS lock so that only one
|
|
|
|
* spinner can compete for the mutex. However, if mutex spinning isn't
|
|
|
|
* going to happen, there is no point in going through the lock/unlock
|
|
|
|
* overhead.
|
2009-01-12 06:01:47 -07:00
|
|
|
*/
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
if (!mutex_can_spin_on_owner(lock))
|
|
|
|
goto slowpath;
|
2009-01-12 06:01:47 -07:00
|
|
|
|
|
|
|
for (;;) {
|
2011-04-05 09:23:41 -06:00
|
|
|
struct task_struct *owner;
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
struct mspin_node node;
|
2009-01-12 06:01:47 -07:00
|
|
|
|
2013-10-17 04:45:29 -06:00
|
|
|
if (use_ww_ctx && ww_ctx->acquired > 0) {
|
2013-06-24 02:30:04 -06:00
|
|
|
struct ww_mutex *ww;
|
|
|
|
|
|
|
|
ww = container_of(lock, struct ww_mutex, base);
|
|
|
|
/*
|
|
|
|
* If ww->ctx is set the contents are undefined, only
|
|
|
|
* by acquiring wait_lock there is a guarantee that
|
|
|
|
* they are not invalid when reading.
|
|
|
|
*
|
|
|
|
* As such, when deadlock detection needs to be
|
|
|
|
* performed the optimistic spinning cannot be done.
|
|
|
|
*/
|
|
|
|
if (ACCESS_ONCE(ww->ctx))
|
2013-06-28 14:13:18 -06:00
|
|
|
goto slowpath;
|
2013-06-24 02:30:04 -06:00
|
|
|
}
|
|
|
|
|
2009-01-12 06:01:47 -07:00
|
|
|
/*
|
|
|
|
* If there's an owner, wait for it to either
|
|
|
|
* release the lock or go to sleep.
|
|
|
|
*/
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
mspin_lock(MLOCK(lock), &node);
|
2009-01-12 06:01:47 -07:00
|
|
|
owner = ACCESS_ONCE(lock->owner);
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
if (owner && !mutex_spin_on_owner(lock, owner)) {
|
|
|
|
mspin_unlock(MLOCK(lock), &node);
|
2013-06-28 14:13:18 -06:00
|
|
|
goto slowpath;
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
}
|
2009-01-12 06:01:47 -07:00
|
|
|
|
mutex: Make more scalable by doing less atomic operations
In the __mutex_lock_common() function, an initial entry into
the lock slow path will cause two atomic_xchg instructions to be
issued. Together with the atomic decrement in the fast path, a
total of three atomic read-modify-write instructions will be
issued in rapid succession. This can cause a lot of cache
bouncing when many tasks are trying to acquire the mutex at the
same time.
This patch will reduce the number of atomic_xchg instructions
used by checking the counter value first before issuing the
instruction. The atomic_read() function is just a simple memory
read. The atomic_xchg() function, on the other hand, can be up
to 2 order of magnitude or even more in cost when compared with
atomic_read(). By using atomic_read() to check the value first
before calling atomic_xchg(), we can avoid a lot of unnecessary
cache coherency traffic. The only downside with this change is
that a task on the slow path will have a tiny bit less chance of
getting the mutex when competing with another task in the fast
path.
The same is true for the atomic_cmpxchg() function in the
mutex-spin-on-owner loop. So an atomic_read() is also performed
before calling atomic_cmpxchg().
The mutex locking and unlocking code for the x86 architecture
can allow any negative number to be used in the mutex count to
indicate that some tasks are waiting for the mutex. I am not so
sure if that is the case for the other architectures. So the
default is to avoid atomic_xchg() if the count has already been
set to -1. For x86, the check is modified to include all
negative numbers to cover a larger case.
The following table shows the jobs per minutes (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the
high_systime workloads to 1/2/4/8 nodes with hyperthreading on
and off.
+-----------------+-----------+------------+----------+
| Configuration | Mean JPM | Mean JPM | % Change |
| | w/o patch | with patch | |
+-----------------+-----------------------------------+
| | User Range 1100 - 2000 |
+-----------------+-----------------------------------+
| 8 nodes, HT on | 36980 | 148590 | +301.8% |
| 8 nodes, HT off | 42799 | 145011 | +238.8% |
| 4 nodes, HT on | 61318 | 118445 | +51.1% |
| 4 nodes, HT off | 158481 | 158592 | +0.1% |
| 2 nodes, HT on | 180602 | 173967 | -3.7% |
| 2 nodes, HT off | 198409 | 198073 | -0.2% |
| 1 node , HT on | 149042 | 147671 | -0.9% |
| 1 node , HT off | 126036 | 126533 | +0.4% |
+-----------------+-----------------------------------+
| | User Range 200 - 1000 |
+-----------------+-----------------------------------+
| 8 nodes, HT on | 41525 | 122349 | +194.6% |
| 8 nodes, HT off | 49866 | 124032 | +148.7% |
| 4 nodes, HT on | 66409 | 106984 | +61.1% |
| 4 nodes, HT off | 119880 | 130508 | +8.9% |
| 2 nodes, HT on | 138003 | 133948 | -2.9% |
| 2 nodes, HT off | 132792 | 131997 | -0.6% |
| 1 node , HT on | 116593 | 115859 | -0.6% |
| 1 node , HT off | 104499 | 104597 | +0.1% |
+-----------------+------------+-----------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
AIM7 benchmark run has a pretty large run-to-run variance due to
random nature of the subtests executed. So a difference of less
than +-5% may not be really significant.
This patch improves high_systime workload performance at 4 nodes
and up by maintaining transaction rates without significant
drop-off at high node count. The patch has practically no
impact on 1 and 2 nodes system.
The table below shows the percentage time (as reported by perf
record -a -s -g) spent on the __mutex_lock_slowpath() function
by the high_systime workload at 1500 users for 2/4/8-node
configurations with hyperthreading off.
+---------------+-----------------+------------------+---------+
| Configuration | %Time w/o patch | %Time with patch | %Change |
+---------------+-----------------+------------------+---------+
| 8 nodes | 65.34% | 0.69% | -99% |
| 4 nodes | 8.70% | 1.02% | -88% |
| 2 nodes | 0.41% | 0.32% | -22% |
+---------------+-----------------+------------------+---------+
It is obvious that the dramatic performance improvement at 8
nodes was due to the drastic cut in the time spent within the
__mutex_lock_slowpath() function.
The table below show the improvements in other AIM7 workloads
(at 8 nodes, hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | +0.6% | +104.2% | +185.9% |
| five_sec | +1.9% | +0.9% | +0.9% |
| fserver | +1.4% | -7.7% | +5.1% |
| new_fserver | -0.5% | +3.2% | +3.1% |
| shared | +13.1% | +146.1% | +181.5% |
| short | +7.4% | +5.0% | +4.2% |
+--------------+---------------+----------------+-----------------+
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton: Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-3-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:12 -06:00
|
|
|
if ((atomic_read(&lock->count) == 1) &&
|
|
|
|
(atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
|
2009-01-14 09:29:31 -07:00
|
|
|
lock_acquired(&lock->dep_map, ip);
|
2013-10-17 04:45:29 -06:00
|
|
|
if (use_ww_ctx) {
|
2013-06-24 02:30:04 -06:00
|
|
|
struct ww_mutex *ww;
|
|
|
|
ww = container_of(lock, struct ww_mutex, base);
|
|
|
|
|
|
|
|
ww_mutex_set_context_fastpath(ww, ww_ctx);
|
|
|
|
}
|
|
|
|
|
2009-01-14 09:29:31 -07:00
|
|
|
mutex_set_owner(lock);
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
mspin_unlock(MLOCK(lock), &node);
|
2009-01-14 09:29:31 -07:00
|
|
|
preempt_enable();
|
|
|
|
return 0;
|
|
|
|
}
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
mspin_unlock(MLOCK(lock), &node);
|
2009-01-14 09:29:31 -07:00
|
|
|
|
2009-01-12 06:01:47 -07:00
|
|
|
/*
|
|
|
|
* When there's no owner, we might have preempted between the
|
|
|
|
* owner acquiring the lock and setting the owner field. If
|
|
|
|
* we're an RT task that will live-lock because we won't let
|
|
|
|
* the owner complete.
|
|
|
|
*/
|
|
|
|
if (!owner && (need_resched() || rt_task(task)))
|
2013-06-28 14:13:18 -06:00
|
|
|
goto slowpath;
|
2009-01-12 06:01:47 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The cpu_relax() call is a compiler barrier which forces
|
|
|
|
* everything in this loop to be re-loaded. We don't need
|
|
|
|
* memory barriers as we'll eventually observe the right
|
|
|
|
* values at the cost of a few extra spins.
|
|
|
|
*/
|
2010-11-22 07:47:36 -07:00
|
|
|
arch_mutex_cpu_relax();
|
2009-01-12 06:01:47 -07:00
|
|
|
}
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:13 -06:00
|
|
|
slowpath:
|
2009-01-12 06:01:47 -07:00
|
|
|
#endif
|
2006-06-26 01:24:31 -06:00
|
|
|
spin_lock_mutex(&lock->wait_lock, flags);
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2013-06-28 14:13:18 -06:00
|
|
|
/* once more, can we acquire the lock? */
|
|
|
|
if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, 0) == 1))
|
|
|
|
goto skip_wait;
|
|
|
|
|
2006-07-03 01:24:33 -06:00
|
|
|
debug_mutex_lock_common(lock, &waiter);
|
2007-05-09 03:35:16 -06:00
|
|
|
debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
|
2006-01-09 16:59:19 -07:00
|
|
|
|
|
|
|
/* add waiting tasks to the end of the waitqueue (FIFO): */
|
|
|
|
list_add_tail(&waiter.list, &lock->wait_list);
|
|
|
|
waiter.task = task;
|
|
|
|
|
2007-10-11 14:11:12 -06:00
|
|
|
lock_contended(&lock->dep_map, ip);
|
2007-07-19 02:48:58 -06:00
|
|
|
|
2006-01-09 16:59:19 -07:00
|
|
|
for (;;) {
|
|
|
|
/*
|
|
|
|
* Lets try to take the lock again - this is needed even if
|
|
|
|
* we get here for the first time (shortly after failing to
|
|
|
|
* acquire the lock), to make sure that we get a wakeup once
|
|
|
|
* it's unlocked. Later on, if we sleep, this is the
|
|
|
|
* operation that gives us the lock. We xchg it to -1, so
|
|
|
|
* that when we release the lock, we properly wake up the
|
|
|
|
* other waiters:
|
|
|
|
*/
|
mutex: Make more scalable by doing less atomic operations
In the __mutex_lock_common() function, an initial entry into
the lock slow path will cause two atomic_xchg instructions to be
issued. Together with the atomic decrement in the fast path, a
total of three atomic read-modify-write instructions will be
issued in rapid succession. This can cause a lot of cache
bouncing when many tasks are trying to acquire the mutex at the
same time.
This patch will reduce the number of atomic_xchg instructions
used by checking the counter value first before issuing the
instruction. The atomic_read() function is just a simple memory
read. The atomic_xchg() function, on the other hand, can be up
to 2 order of magnitude or even more in cost when compared with
atomic_read(). By using atomic_read() to check the value first
before calling atomic_xchg(), we can avoid a lot of unnecessary
cache coherency traffic. The only downside with this change is
that a task on the slow path will have a tiny bit less chance of
getting the mutex when competing with another task in the fast
path.
The same is true for the atomic_cmpxchg() function in the
mutex-spin-on-owner loop. So an atomic_read() is also performed
before calling atomic_cmpxchg().
The mutex locking and unlocking code for the x86 architecture
can allow any negative number to be used in the mutex count to
indicate that some tasks are waiting for the mutex. I am not so
sure if that is the case for the other architectures. So the
default is to avoid atomic_xchg() if the count has already been
set to -1. For x86, the check is modified to include all
negative numbers to cover a larger case.
The following table shows the jobs per minutes (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the
high_systime workloads to 1/2/4/8 nodes with hyperthreading on
and off.
+-----------------+-----------+------------+----------+
| Configuration | Mean JPM | Mean JPM | % Change |
| | w/o patch | with patch | |
+-----------------+-----------------------------------+
| | User Range 1100 - 2000 |
+-----------------+-----------------------------------+
| 8 nodes, HT on | 36980 | 148590 | +301.8% |
| 8 nodes, HT off | 42799 | 145011 | +238.8% |
| 4 nodes, HT on | 61318 | 118445 | +51.1% |
| 4 nodes, HT off | 158481 | 158592 | +0.1% |
| 2 nodes, HT on | 180602 | 173967 | -3.7% |
| 2 nodes, HT off | 198409 | 198073 | -0.2% |
| 1 node , HT on | 149042 | 147671 | -0.9% |
| 1 node , HT off | 126036 | 126533 | +0.4% |
+-----------------+-----------------------------------+
| | User Range 200 - 1000 |
+-----------------+-----------------------------------+
| 8 nodes, HT on | 41525 | 122349 | +194.6% |
| 8 nodes, HT off | 49866 | 124032 | +148.7% |
| 4 nodes, HT on | 66409 | 106984 | +61.1% |
| 4 nodes, HT off | 119880 | 130508 | +8.9% |
| 2 nodes, HT on | 138003 | 133948 | -2.9% |
| 2 nodes, HT off | 132792 | 131997 | -0.6% |
| 1 node , HT on | 116593 | 115859 | -0.6% |
| 1 node , HT off | 104499 | 104597 | +0.1% |
+-----------------+------------+-----------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
AIM7 benchmark run has a pretty large run-to-run variance due to
random nature of the subtests executed. So a difference of less
than +-5% may not be really significant.
This patch improves high_systime workload performance at 4 nodes
and up by maintaining transaction rates without significant
drop-off at high node count. The patch has practically no
impact on 1 and 2 nodes system.
The table below shows the percentage time (as reported by perf
record -a -s -g) spent on the __mutex_lock_slowpath() function
by the high_systime workload at 1500 users for 2/4/8-node
configurations with hyperthreading off.
+---------------+-----------------+------------------+---------+
| Configuration | %Time w/o patch | %Time with patch | %Change |
+---------------+-----------------+------------------+---------+
| 8 nodes | 65.34% | 0.69% | -99% |
| 4 nodes | 8.70% | 1.02% | -88% |
| 2 nodes | 0.41% | 0.32% | -22% |
+---------------+-----------------+------------------+---------+
It is obvious that the dramatic performance improvement at 8
nodes was due to the drastic cut in the time spent within the
__mutex_lock_slowpath() function.
The table below show the improvements in other AIM7 workloads
(at 8 nodes, hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | +0.6% | +104.2% | +185.9% |
| five_sec | +1.9% | +0.9% | +0.9% |
| fserver | +1.4% | -7.7% | +5.1% |
| new_fserver | -0.5% | +3.2% | +3.1% |
| shared | +13.1% | +146.1% | +181.5% |
| short | +7.4% | +5.0% | +4.2% |
+--------------+---------------+----------------+-----------------+
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton: Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-3-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-17 13:23:12 -06:00
|
|
|
if (MUTEX_SHOW_NO_WAITER(lock) &&
|
2013-06-28 14:13:18 -06:00
|
|
|
(atomic_xchg(&lock->count, -1) == 1))
|
2006-01-09 16:59:19 -07:00
|
|
|
break;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* got a signal? (This code gets eliminated in the
|
|
|
|
* TASK_UNINTERRUPTIBLE case.)
|
|
|
|
*/
|
2008-06-08 11:20:42 -06:00
|
|
|
if (unlikely(signal_pending_state(state, task))) {
|
2013-06-24 02:30:04 -06:00
|
|
|
ret = -EINTR;
|
|
|
|
goto err;
|
|
|
|
}
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2013-10-17 04:45:29 -06:00
|
|
|
if (use_ww_ctx && ww_ctx->acquired > 0) {
|
2013-06-24 02:30:04 -06:00
|
|
|
ret = __mutex_lock_check_stamp(lock, ww_ctx);
|
|
|
|
if (ret)
|
|
|
|
goto err;
|
2006-01-09 16:59:19 -07:00
|
|
|
}
|
2013-06-24 02:30:04 -06:00
|
|
|
|
2006-01-09 16:59:19 -07:00
|
|
|
__set_task_state(task, state);
|
|
|
|
|
2011-03-30 19:57:33 -06:00
|
|
|
/* didn't get the lock, go to sleep: */
|
2006-06-26 01:24:31 -06:00
|
|
|
spin_unlock_mutex(&lock->wait_lock, flags);
|
2011-03-21 05:33:18 -06:00
|
|
|
schedule_preempt_disabled();
|
2006-06-26 01:24:31 -06:00
|
|
|
spin_lock_mutex(&lock->wait_lock, flags);
|
2006-01-09 16:59:19 -07:00
|
|
|
}
|
2013-06-28 14:13:18 -06:00
|
|
|
mutex_remove_waiter(lock, &waiter, current_thread_info());
|
|
|
|
/* set it to 0 if there are no waiters left: */
|
|
|
|
if (likely(list_empty(&lock->wait_list)))
|
|
|
|
atomic_set(&lock->count, 0);
|
|
|
|
debug_mutex_free_waiter(&waiter);
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2013-06-28 14:13:18 -06:00
|
|
|
skip_wait:
|
|
|
|
/* got the lock - cleanup and rejoice! */
|
2008-10-16 15:17:09 -06:00
|
|
|
lock_acquired(&lock->dep_map, ip);
|
2009-01-12 06:01:47 -07:00
|
|
|
mutex_set_owner(lock);
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2013-10-17 04:45:29 -06:00
|
|
|
if (use_ww_ctx) {
|
2013-06-28 14:13:18 -06:00
|
|
|
struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
|
2013-06-24 02:30:04 -06:00
|
|
|
struct mutex_waiter *cur;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This branch gets optimized out for the common case,
|
|
|
|
* and is only important for ww_mutex_lock.
|
|
|
|
*/
|
|
|
|
ww_mutex_lock_acquired(ww, ww_ctx);
|
|
|
|
ww->ctx = ww_ctx;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Give any possible sleeping processes the chance to wake up,
|
|
|
|
* so they can recheck if they have to back off.
|
|
|
|
*/
|
|
|
|
list_for_each_entry(cur, &lock->wait_list, list) {
|
|
|
|
debug_mutex_wake_waiter(lock, cur);
|
|
|
|
wake_up_process(cur->task);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-06-26 01:24:31 -06:00
|
|
|
spin_unlock_mutex(&lock->wait_lock, flags);
|
2009-01-14 07:36:26 -07:00
|
|
|
preempt_enable();
|
2006-01-09 16:59:19 -07:00
|
|
|
return 0;
|
2013-06-24 02:30:04 -06:00
|
|
|
|
|
|
|
err:
|
|
|
|
mutex_remove_waiter(lock, &waiter, task_thread_info(task));
|
|
|
|
spin_unlock_mutex(&lock->wait_lock, flags);
|
|
|
|
debug_mutex_free_waiter(&waiter);
|
|
|
|
mutex_release(&lock->dep_map, 1, ip);
|
|
|
|
preempt_enable();
|
|
|
|
return ret;
|
2006-01-09 16:59:19 -07:00
|
|
|
}
|
|
|
|
|
2006-07-03 01:24:55 -06:00
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
void __sched
|
|
|
|
mutex_lock_nested(struct mutex *lock, unsigned int subclass)
|
|
|
|
{
|
|
|
|
might_sleep();
|
2013-06-24 02:30:04 -06:00
|
|
|
__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
|
2013-10-17 04:45:29 -06:00
|
|
|
subclass, NULL, _RET_IP_, NULL, 0);
|
2006-07-03 01:24:55 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL_GPL(mutex_lock_nested);
|
2006-12-08 03:36:17 -07:00
|
|
|
|
2011-05-24 18:12:03 -06:00
|
|
|
void __sched
|
|
|
|
_mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
|
|
|
|
{
|
|
|
|
might_sleep();
|
2013-06-24 02:30:04 -06:00
|
|
|
__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
|
2013-10-17 04:45:29 -06:00
|
|
|
0, nest, _RET_IP_, NULL, 0);
|
2011-05-24 18:12:03 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
|
|
|
|
|
2007-12-06 15:37:59 -07:00
|
|
|
int __sched
|
|
|
|
mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)
|
|
|
|
{
|
|
|
|
might_sleep();
|
2013-06-24 02:30:04 -06:00
|
|
|
return __mutex_lock_common(lock, TASK_KILLABLE,
|
2013-10-17 04:45:29 -06:00
|
|
|
subclass, NULL, _RET_IP_, NULL, 0);
|
2007-12-06 15:37:59 -07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);
|
|
|
|
|
2006-12-08 03:36:17 -07:00
|
|
|
int __sched
|
|
|
|
mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
|
|
|
|
{
|
|
|
|
might_sleep();
|
2009-01-12 06:01:47 -07:00
|
|
|
return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,
|
2013-10-17 04:45:29 -06:00
|
|
|
subclass, NULL, _RET_IP_, NULL, 0);
|
2006-12-08 03:36:17 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
|
2013-06-24 02:30:04 -06:00
|
|
|
|
2013-06-20 05:31:17 -06:00
|
|
|
static inline int
|
|
|
|
ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
|
|
|
|
unsigned tmp;
|
|
|
|
|
|
|
|
if (ctx->deadlock_inject_countdown-- == 0) {
|
|
|
|
tmp = ctx->deadlock_inject_interval;
|
|
|
|
if (tmp > UINT_MAX/4)
|
|
|
|
tmp = UINT_MAX;
|
|
|
|
else
|
|
|
|
tmp = tmp*2 + tmp + tmp/2;
|
|
|
|
|
|
|
|
ctx->deadlock_inject_interval = tmp;
|
|
|
|
ctx->deadlock_inject_countdown = tmp;
|
|
|
|
ctx->contending_lock = lock;
|
|
|
|
|
|
|
|
ww_mutex_unlock(lock);
|
|
|
|
|
|
|
|
return -EDEADLK;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2013-06-24 02:30:04 -06:00
|
|
|
|
|
|
|
int __sched
|
|
|
|
__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
2013-06-20 05:31:17 -06:00
|
|
|
int ret;
|
|
|
|
|
2013-06-24 02:30:04 -06:00
|
|
|
might_sleep();
|
2013-06-20 05:31:17 -06:00
|
|
|
ret = __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE,
|
2013-10-17 04:45:29 -06:00
|
|
|
0, &ctx->dep_map, _RET_IP_, ctx, 1);
|
2013-07-30 02:13:41 -06:00
|
|
|
if (!ret && ctx->acquired > 1)
|
2013-06-20 05:31:17 -06:00
|
|
|
return ww_mutex_deadlock_injection(lock, ctx);
|
|
|
|
|
|
|
|
return ret;
|
2013-06-24 02:30:04 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__ww_mutex_lock);
|
|
|
|
|
|
|
|
int __sched
|
|
|
|
__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
2013-06-20 05:31:17 -06:00
|
|
|
int ret;
|
|
|
|
|
2013-06-24 02:30:04 -06:00
|
|
|
might_sleep();
|
2013-06-20 05:31:17 -06:00
|
|
|
ret = __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE,
|
2013-10-17 04:45:29 -06:00
|
|
|
0, &ctx->dep_map, _RET_IP_, ctx, 1);
|
2013-06-20 05:31:17 -06:00
|
|
|
|
2013-07-30 02:13:41 -06:00
|
|
|
if (!ret && ctx->acquired > 1)
|
2013-06-20 05:31:17 -06:00
|
|
|
return ww_mutex_deadlock_injection(lock, ctx);
|
|
|
|
|
|
|
|
return ret;
|
2013-06-24 02:30:04 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible);
|
|
|
|
|
2006-07-03 01:24:55 -06:00
|
|
|
#endif
|
|
|
|
|
2006-01-09 16:59:19 -07:00
|
|
|
/*
|
|
|
|
* Release the lock, slowpath:
|
|
|
|
*/
|
2008-02-08 05:19:53 -07:00
|
|
|
static inline void
|
2006-07-03 01:24:55 -06:00
|
|
|
__mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
|
2006-01-09 16:59:19 -07:00
|
|
|
{
|
2006-01-10 15:15:02 -07:00
|
|
|
struct mutex *lock = container_of(lock_count, struct mutex, count);
|
2006-06-26 01:24:31 -06:00
|
|
|
unsigned long flags;
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2006-06-26 01:24:31 -06:00
|
|
|
spin_lock_mutex(&lock->wait_lock, flags);
|
2006-07-03 01:24:55 -06:00
|
|
|
mutex_release(&lock->dep_map, nested, _RET_IP_);
|
2006-07-03 01:24:33 -06:00
|
|
|
debug_mutex_unlock(lock);
|
2006-01-09 16:59:19 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* some architectures leave the lock unlocked in the fastpath failure
|
|
|
|
* case, others need to leave it locked. In the later case we have to
|
|
|
|
* unlock it here
|
|
|
|
*/
|
|
|
|
if (__mutex_slowpath_needs_to_unlock())
|
|
|
|
atomic_set(&lock->count, 1);
|
|
|
|
|
|
|
|
if (!list_empty(&lock->wait_list)) {
|
|
|
|
/* get the first entry from the wait-list: */
|
|
|
|
struct mutex_waiter *waiter =
|
|
|
|
list_entry(lock->wait_list.next,
|
|
|
|
struct mutex_waiter, list);
|
|
|
|
|
|
|
|
debug_mutex_wake_waiter(lock, waiter);
|
|
|
|
|
|
|
|
wake_up_process(waiter->task);
|
|
|
|
}
|
|
|
|
|
2006-06-26 01:24:31 -06:00
|
|
|
spin_unlock_mutex(&lock->wait_lock, flags);
|
2006-01-09 16:59:19 -07:00
|
|
|
}
|
|
|
|
|
2006-07-03 01:24:33 -06:00
|
|
|
/*
|
|
|
|
* Release the lock, slowpath:
|
|
|
|
*/
|
2008-11-24 01:17:42 -07:00
|
|
|
static __used noinline void
|
2006-07-03 01:24:33 -06:00
|
|
|
__mutex_unlock_slowpath(atomic_t *lock_count)
|
|
|
|
{
|
2006-07-03 01:24:55 -06:00
|
|
|
__mutex_unlock_common_slowpath(lock_count, 1);
|
2006-07-03 01:24:33 -06:00
|
|
|
}
|
|
|
|
|
2007-10-11 14:11:12 -06:00
|
|
|
#ifndef CONFIG_DEBUG_LOCK_ALLOC
|
2006-01-09 16:59:19 -07:00
|
|
|
/*
|
|
|
|
* Here come the less common (and hence less performance-critical) APIs:
|
|
|
|
* mutex_lock_interruptible() and mutex_trylock().
|
|
|
|
*/
|
2008-02-08 05:19:53 -07:00
|
|
|
static noinline int __sched
|
2013-06-20 05:31:05 -06:00
|
|
|
__mutex_lock_killable_slowpath(struct mutex *lock);
|
2007-12-06 15:37:59 -07:00
|
|
|
|
2008-02-08 05:19:53 -07:00
|
|
|
static noinline int __sched
|
2013-06-20 05:31:05 -06:00
|
|
|
__mutex_lock_interruptible_slowpath(struct mutex *lock);
|
2006-01-09 16:59:19 -07:00
|
|
|
|
2010-09-02 16:48:16 -06:00
|
|
|
/**
|
|
|
|
* mutex_lock_interruptible - acquire the mutex, interruptible
|
2006-01-09 16:59:19 -07:00
|
|
|
* @lock: the mutex to be acquired
|
|
|
|
*
|
|
|
|
* Lock the mutex like mutex_lock(), and return 0 if the mutex has
|
|
|
|
* been acquired or sleep until the mutex becomes available. If a
|
|
|
|
* signal arrives while waiting for the lock then this function
|
|
|
|
* returns -EINTR.
|
|
|
|
*
|
|
|
|
* This function is similar to (but not equivalent to) down_interruptible().
|
|
|
|
*/
|
2008-02-08 05:19:53 -07:00
|
|
|
int __sched mutex_lock_interruptible(struct mutex *lock)
|
2006-01-09 16:59:19 -07:00
|
|
|
{
|
2009-01-12 06:01:47 -07:00
|
|
|
int ret;
|
|
|
|
|
2006-01-10 14:10:36 -07:00
|
|
|
might_sleep();
|
2013-06-20 05:31:05 -06:00
|
|
|
ret = __mutex_fastpath_lock_retval(&lock->count);
|
|
|
|
if (likely(!ret)) {
|
2009-01-12 06:01:47 -07:00
|
|
|
mutex_set_owner(lock);
|
2013-06-20 05:31:05 -06:00
|
|
|
return 0;
|
|
|
|
} else
|
|
|
|
return __mutex_lock_interruptible_slowpath(lock);
|
2006-01-09 16:59:19 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(mutex_lock_interruptible);
|
|
|
|
|
2008-02-08 05:19:53 -07:00
|
|
|
int __sched mutex_lock_killable(struct mutex *lock)
|
2007-12-06 15:37:59 -07:00
|
|
|
{
|
2009-01-12 06:01:47 -07:00
|
|
|
int ret;
|
|
|
|
|
2007-12-06 15:37:59 -07:00
|
|
|
might_sleep();
|
2013-06-20 05:31:05 -06:00
|
|
|
ret = __mutex_fastpath_lock_retval(&lock->count);
|
|
|
|
if (likely(!ret)) {
|
2009-01-12 06:01:47 -07:00
|
|
|
mutex_set_owner(lock);
|
2013-06-20 05:31:05 -06:00
|
|
|
return 0;
|
|
|
|
} else
|
|
|
|
return __mutex_lock_killable_slowpath(lock);
|
2007-12-06 15:37:59 -07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mutex_lock_killable);
|
|
|
|
|
2008-11-24 01:17:42 -07:00
|
|
|
static __used noinline void __sched
|
2007-10-11 14:11:12 -06:00
|
|
|
__mutex_lock_slowpath(atomic_t *lock_count)
|
|
|
|
{
|
|
|
|
struct mutex *lock = container_of(lock_count, struct mutex, count);
|
|
|
|
|
2013-06-24 02:30:04 -06:00
|
|
|
__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0,
|
2013-10-17 04:45:29 -06:00
|
|
|
NULL, _RET_IP_, NULL, 0);
|
2007-10-11 14:11:12 -06:00
|
|
|
}
|
|
|
|
|
2008-02-08 05:19:53 -07:00
|
|
|
static noinline int __sched
|
2013-06-20 05:31:05 -06:00
|
|
|
__mutex_lock_killable_slowpath(struct mutex *lock)
|
2007-12-06 15:37:59 -07:00
|
|
|
{
|
2013-06-24 02:30:04 -06:00
|
|
|
return __mutex_lock_common(lock, TASK_KILLABLE, 0,
|
2013-10-17 04:45:29 -06:00
|
|
|
NULL, _RET_IP_, NULL, 0);
|
2007-12-06 15:37:59 -07:00
|
|
|
}
|
|
|
|
|
2008-02-08 05:19:53 -07:00
|
|
|
static noinline int __sched
|
2013-06-20 05:31:05 -06:00
|
|
|
__mutex_lock_interruptible_slowpath(struct mutex *lock)
|
2006-01-09 16:59:19 -07:00
|
|
|
{
|
2013-06-24 02:30:04 -06:00
|
|
|
return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0,
|
2013-10-17 04:45:29 -06:00
|
|
|
NULL, _RET_IP_, NULL, 0);
|
2013-06-24 02:30:04 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static noinline int __sched
|
|
|
|
__ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
return __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE, 0,
|
2013-10-17 04:45:29 -06:00
|
|
|
NULL, _RET_IP_, ctx, 1);
|
2006-01-09 16:59:19 -07:00
|
|
|
}
|
2013-06-24 02:30:04 -06:00
|
|
|
|
|
|
|
static noinline int __sched
|
|
|
|
__ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
|
|
|
|
struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
return __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE, 0,
|
2013-10-17 04:45:29 -06:00
|
|
|
NULL, _RET_IP_, ctx, 1);
|
2013-06-24 02:30:04 -06:00
|
|
|
}
|
|
|
|
|
2007-10-11 14:11:12 -06:00
|
|
|
#endif
|
2006-01-09 16:59:19 -07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Spinlock based trylock, we take the spinlock and check whether we
|
|
|
|
* can get the lock:
|
|
|
|
*/
|
|
|
|
static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
|
|
|
|
{
|
|
|
|
struct mutex *lock = container_of(lock_count, struct mutex, count);
|
2006-06-26 01:24:31 -06:00
|
|
|
unsigned long flags;
|
2006-01-09 16:59:19 -07:00
|
|
|
int prev;
|
|
|
|
|
2006-06-26 01:24:31 -06:00
|
|
|
spin_lock_mutex(&lock->wait_lock, flags);
|
2006-01-09 16:59:19 -07:00
|
|
|
|
|
|
|
prev = atomic_xchg(&lock->count, -1);
|
2006-07-03 01:24:55 -06:00
|
|
|
if (likely(prev == 1)) {
|
2009-01-12 06:01:47 -07:00
|
|
|
mutex_set_owner(lock);
|
2006-07-03 01:24:55 -06:00
|
|
|
mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
|
|
|
|
}
|
2009-01-12 06:01:47 -07:00
|
|
|
|
2006-01-09 16:59:19 -07:00
|
|
|
/* Set it back to 0 if there are no waiters: */
|
|
|
|
if (likely(list_empty(&lock->wait_list)))
|
|
|
|
atomic_set(&lock->count, 0);
|
|
|
|
|
2006-06-26 01:24:31 -06:00
|
|
|
spin_unlock_mutex(&lock->wait_lock, flags);
|
2006-01-09 16:59:19 -07:00
|
|
|
|
|
|
|
return prev == 1;
|
|
|
|
}
|
|
|
|
|
2010-09-02 16:48:16 -06:00
|
|
|
/**
|
|
|
|
* mutex_trylock - try to acquire the mutex, without waiting
|
2006-01-09 16:59:19 -07:00
|
|
|
* @lock: the mutex to be acquired
|
|
|
|
*
|
|
|
|
* Try to acquire the mutex atomically. Returns 1 if the mutex
|
|
|
|
* has been acquired successfully, and 0 on contention.
|
|
|
|
*
|
|
|
|
* NOTE: this function follows the spin_trylock() convention, so
|
2010-09-02 16:48:16 -06:00
|
|
|
* it is negated from the down_trylock() return values! Be careful
|
2006-01-09 16:59:19 -07:00
|
|
|
* about this when converting semaphore users to mutexes.
|
|
|
|
*
|
|
|
|
* This function must not be used in interrupt context. The
|
|
|
|
* mutex must be released by the same task that acquired it.
|
|
|
|
*/
|
2008-02-08 05:19:53 -07:00
|
|
|
int __sched mutex_trylock(struct mutex *lock)
|
2006-01-09 16:59:19 -07:00
|
|
|
{
|
2009-01-12 06:01:47 -07:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath);
|
|
|
|
if (ret)
|
|
|
|
mutex_set_owner(lock);
|
|
|
|
|
|
|
|
return ret;
|
2006-01-09 16:59:19 -07:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mutex_trylock);
|
2009-04-29 16:59:58 -06:00
|
|
|
|
2013-06-24 02:30:04 -06:00
|
|
|
#ifndef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
int __sched
|
|
|
|
__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
might_sleep();
|
|
|
|
|
|
|
|
ret = __mutex_fastpath_lock_retval(&lock->base.count);
|
|
|
|
|
|
|
|
if (likely(!ret)) {
|
|
|
|
ww_mutex_set_context_fastpath(lock, ctx);
|
|
|
|
mutex_set_owner(&lock->base);
|
|
|
|
} else
|
|
|
|
ret = __ww_mutex_lock_slowpath(lock, ctx);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__ww_mutex_lock);
|
|
|
|
|
|
|
|
int __sched
|
|
|
|
__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
might_sleep();
|
|
|
|
|
|
|
|
ret = __mutex_fastpath_lock_retval(&lock->base.count);
|
|
|
|
|
|
|
|
if (likely(!ret)) {
|
|
|
|
ww_mutex_set_context_fastpath(lock, ctx);
|
|
|
|
mutex_set_owner(&lock->base);
|
|
|
|
} else
|
|
|
|
ret = __ww_mutex_lock_interruptible_slowpath(lock, ctx);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__ww_mutex_lock_interruptible);
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2009-04-29 16:59:58 -06:00
|
|
|
/**
|
|
|
|
* atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
|
|
|
|
* @cnt: the atomic which we are to dec
|
|
|
|
* @lock: the mutex to return holding if we dec to 0
|
|
|
|
*
|
|
|
|
* return true and hold lock if we dec to 0, return false otherwise
|
|
|
|
*/
|
|
|
|
int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
|
|
|
|
{
|
|
|
|
/* dec if we can't possibly hit 0 */
|
|
|
|
if (atomic_add_unless(cnt, -1, 1))
|
|
|
|
return 0;
|
|
|
|
/* we might hit 0, so take the lock */
|
|
|
|
mutex_lock(lock);
|
|
|
|
if (!atomic_dec_and_test(cnt)) {
|
|
|
|
/* when we actually did the dec, we didn't hit 0 */
|
|
|
|
mutex_unlock(lock);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
/* we hit 0, and we hold the lock */
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(atomic_dec_and_mutex_lock);
|