![Waiman Long](/assets/img/avatar_default.png)
Currently, it is not possible to determine for sure if a reader owns a rwsem by looking at the content of the rwsem data structure. This patch adds a new state RWSEM_READER_OWNED to the owner field to indicate that readers currently own the lock. This enables us to address the following 2 issues in the rwsem optimistic spinning code: 1) rwsem_can_spin_on_owner() will disallow optimistic spinning if the owner field is NULL which can mean either the readers own the lock or the owning writer hasn't set the owner field yet. In the latter case, we miss the chance to do optimistic spinning. 2) While a writer is waiting in the OSQ and a reader takes the lock, the writer will continue to spin when out of the OSQ in the main rwsem_optimistic_spin() loop as the owner field is NULL wasting CPU cycles if some of readers are sleeping. Adding the new state will allow optimistic spinning to go forward as long as the owner field is not RWSEM_READER_OWNED and the owner is running, if set, but stop immediately when that state has been reached. On a 4-socket Haswell machine running on a 4.6-rc1 based kernel, the fio test with multithreaded randrw and randwrite tests on the same file on a XFS partition on top of a NVDIMM were run, the aggregated bandwidths before and after the patch were as follows: Test BW before patch BW after patch % change ---- --------------- -------------- -------- randrw 988 MB/s 1192 MB/s +21% randwrite 1513 MB/s 1623 MB/s +7.3% The perf profile of the rwsem_down_write_failed() function in randrw before and after the patch were: 19.95% 5.88% fio [kernel.vmlinux] [k] rwsem_down_write_failed 14.20% 1.52% fio [kernel.vmlinux] [k] rwsem_down_write_failed The actual CPU cycles spend in rwsem_down_write_failed() dropped from 5.88% to 1.52% after the patch. The xfstests was also run and no regression was observed. Signed-off-by: Waiman Long <Waiman.Long@hpe.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Jason Low <jason.low2@hp.com> Acked-by: Davidlohr Bueso <dave@stgolabs.net> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Dave Chinner <david@fromorbit.com> Cc: Douglas Hatch <doug.hatch@hpe.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Hurley <peter@hurleysoftware.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Scott J Norton <scott.norton@hpe.com> Cc: Thomas Gleixner <tglx@linutronix.de> Link: http://lkml.kernel.org/r/1463534783-38814-2-git-send-email-Waiman.Long@hpe.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
206 lines
3.8 KiB
C
206 lines
3.8 KiB
C
/* kernel/rwsem.c: R/W semaphores, public implementation
|
|
*
|
|
* Written by David Howells (dhowells@redhat.com).
|
|
* Derived from asm-i386/semaphore.h
|
|
*/
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/export.h>
|
|
#include <linux/rwsem.h>
|
|
#include <linux/atomic.h>
|
|
|
|
#include "rwsem.h"
|
|
|
|
/*
|
|
* lock for reading
|
|
*/
|
|
void __sched down_read(struct rw_semaphore *sem)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
|
|
|
|
LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
|
|
rwsem_set_reader_owned(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_read);
|
|
|
|
/*
|
|
* trylock for reading -- returns 1 if successful, 0 if contention
|
|
*/
|
|
int down_read_trylock(struct rw_semaphore *sem)
|
|
{
|
|
int ret = __down_read_trylock(sem);
|
|
|
|
if (ret == 1) {
|
|
rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
|
|
rwsem_set_reader_owned(sem);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_read_trylock);
|
|
|
|
/*
|
|
* lock for writing
|
|
*/
|
|
void __sched down_write(struct rw_semaphore *sem)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
|
|
|
|
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
|
|
rwsem_set_owner(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_write);
|
|
|
|
/*
|
|
* lock for writing
|
|
*/
|
|
int __sched down_write_killable(struct rw_semaphore *sem)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
|
|
|
|
if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
|
|
rwsem_release(&sem->dep_map, 1, _RET_IP_);
|
|
return -EINTR;
|
|
}
|
|
|
|
rwsem_set_owner(sem);
|
|
return 0;
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_write_killable);
|
|
|
|
/*
|
|
* trylock for writing -- returns 1 if successful, 0 if contention
|
|
*/
|
|
int down_write_trylock(struct rw_semaphore *sem)
|
|
{
|
|
int ret = __down_write_trylock(sem);
|
|
|
|
if (ret == 1) {
|
|
rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_);
|
|
rwsem_set_owner(sem);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_write_trylock);
|
|
|
|
/*
|
|
* release a read lock
|
|
*/
|
|
void up_read(struct rw_semaphore *sem)
|
|
{
|
|
rwsem_release(&sem->dep_map, 1, _RET_IP_);
|
|
|
|
__up_read(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(up_read);
|
|
|
|
/*
|
|
* release a write lock
|
|
*/
|
|
void up_write(struct rw_semaphore *sem)
|
|
{
|
|
rwsem_release(&sem->dep_map, 1, _RET_IP_);
|
|
|
|
rwsem_clear_owner(sem);
|
|
__up_write(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(up_write);
|
|
|
|
/*
|
|
* downgrade write lock to read lock
|
|
*/
|
|
void downgrade_write(struct rw_semaphore *sem)
|
|
{
|
|
/*
|
|
* lockdep: a downgraded write will live on as a write
|
|
* dependency.
|
|
*/
|
|
rwsem_set_reader_owned(sem);
|
|
__downgrade_write(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(downgrade_write);
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
void down_read_nested(struct rw_semaphore *sem, int subclass)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
|
|
|
|
LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
|
|
rwsem_set_reader_owned(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_read_nested);
|
|
|
|
void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
|
|
|
|
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
|
|
rwsem_set_owner(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(_down_write_nest_lock);
|
|
|
|
void down_read_non_owner(struct rw_semaphore *sem)
|
|
{
|
|
might_sleep();
|
|
|
|
__down_read(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_read_non_owner);
|
|
|
|
void down_write_nested(struct rw_semaphore *sem, int subclass)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
|
|
|
|
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
|
|
rwsem_set_owner(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_write_nested);
|
|
|
|
int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
|
|
|
|
if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
|
|
rwsem_release(&sem->dep_map, 1, _RET_IP_);
|
|
return -EINTR;
|
|
}
|
|
|
|
rwsem_set_owner(sem);
|
|
return 0;
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_write_killable_nested);
|
|
|
|
void up_read_non_owner(struct rw_semaphore *sem)
|
|
{
|
|
__up_read(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(up_read_non_owner);
|
|
|
|
#endif
|
|
|
|
|