![Waiman Long](/assets/img/avatar_default.png)
There are use cases where a rwsem can be acquired by one task, but released by another task. In thess cases, optimistic spinning may need to be disabled. One example will be the filesystem freeze/thaw code where the task that freezes the filesystem will acquire a write lock on a rwsem and then un-owns it before returning to userspace. Later on, another task will come along, acquire the ownership, thaw the filesystem and release the rwsem. Bit 0 of the owner field was used to designate that it is a reader owned rwsem. It is now repurposed to mean that the owner of the rwsem is not known. If only bit 0 is set, the rwsem is reader owned. If bit 0 and other bits are set, it is writer owned with an unknown owner. One such value for the latter case is (-1L). So we can set owner to 1 for reader-owned, -1 for writer-owned. The owner is unknown in both cases. To handle transfer of rwsem ownership, the higher level code should set the owner field to -1 to indicate a write-locked rwsem with unknown owner. Optimistic spinning will be disabled in this case. Once the higher level code figures who the new owner is, it can then set the owner field accordingly. Tested-by: Amir Goldstein <amir73il@gmail.com> Signed-off-by: Waiman Long <longman@redhat.com> Acked-by: Peter Zijlstra <peterz@infradead.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Davidlohr Bueso <dave@stgolabs.net> Cc: Jan Kara <jack@suse.cz> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Matthew Wilcox <willy@infradead.org> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Theodore Y. Ts'o <tytso@mit.edu> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Will Deacon <will.deacon@arm.com> Cc: linux-fsdevel@vger.kernel.org Link: http://lkml.kernel.org/r/1526420991-21213-2-git-send-email-longman@redhat.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
224 lines
4.4 KiB
C
224 lines
4.4 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/* kernel/rwsem.c: R/W semaphores, public implementation
|
|
*
|
|
* Written by David Howells (dhowells@redhat.com).
|
|
* Derived from asm-i386/semaphore.h
|
|
*/
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/sched/debug.h>
|
|
#include <linux/export.h>
|
|
#include <linux/rwsem.h>
|
|
#include <linux/atomic.h>
|
|
|
|
#include "rwsem.h"
|
|
|
|
/*
|
|
* lock for reading
|
|
*/
|
|
void __sched down_read(struct rw_semaphore *sem)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
|
|
|
|
LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
|
|
rwsem_set_reader_owned(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_read);
|
|
|
|
int __sched down_read_killable(struct rw_semaphore *sem)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
|
|
|
|
if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) {
|
|
rwsem_release(&sem->dep_map, 1, _RET_IP_);
|
|
return -EINTR;
|
|
}
|
|
|
|
rwsem_set_reader_owned(sem);
|
|
return 0;
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_read_killable);
|
|
|
|
/*
|
|
* trylock for reading -- returns 1 if successful, 0 if contention
|
|
*/
|
|
int down_read_trylock(struct rw_semaphore *sem)
|
|
{
|
|
int ret = __down_read_trylock(sem);
|
|
|
|
if (ret == 1) {
|
|
rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
|
|
rwsem_set_reader_owned(sem);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_read_trylock);
|
|
|
|
/*
|
|
* lock for writing
|
|
*/
|
|
void __sched down_write(struct rw_semaphore *sem)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
|
|
|
|
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
|
|
rwsem_set_owner(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_write);
|
|
|
|
/*
|
|
* lock for writing
|
|
*/
|
|
int __sched down_write_killable(struct rw_semaphore *sem)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
|
|
|
|
if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
|
|
rwsem_release(&sem->dep_map, 1, _RET_IP_);
|
|
return -EINTR;
|
|
}
|
|
|
|
rwsem_set_owner(sem);
|
|
return 0;
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_write_killable);
|
|
|
|
/*
|
|
* trylock for writing -- returns 1 if successful, 0 if contention
|
|
*/
|
|
int down_write_trylock(struct rw_semaphore *sem)
|
|
{
|
|
int ret = __down_write_trylock(sem);
|
|
|
|
if (ret == 1) {
|
|
rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_);
|
|
rwsem_set_owner(sem);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_write_trylock);
|
|
|
|
/*
|
|
* release a read lock
|
|
*/
|
|
void up_read(struct rw_semaphore *sem)
|
|
{
|
|
rwsem_release(&sem->dep_map, 1, _RET_IP_);
|
|
DEBUG_RWSEMS_WARN_ON(sem->owner != RWSEM_READER_OWNED);
|
|
|
|
__up_read(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(up_read);
|
|
|
|
/*
|
|
* release a write lock
|
|
*/
|
|
void up_write(struct rw_semaphore *sem)
|
|
{
|
|
rwsem_release(&sem->dep_map, 1, _RET_IP_);
|
|
DEBUG_RWSEMS_WARN_ON(sem->owner != current);
|
|
|
|
rwsem_clear_owner(sem);
|
|
__up_write(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(up_write);
|
|
|
|
/*
|
|
* downgrade write lock to read lock
|
|
*/
|
|
void downgrade_write(struct rw_semaphore *sem)
|
|
{
|
|
lock_downgrade(&sem->dep_map, _RET_IP_);
|
|
DEBUG_RWSEMS_WARN_ON(sem->owner != current);
|
|
|
|
rwsem_set_reader_owned(sem);
|
|
__downgrade_write(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(downgrade_write);
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
void down_read_nested(struct rw_semaphore *sem, int subclass)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
|
|
|
|
LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
|
|
rwsem_set_reader_owned(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_read_nested);
|
|
|
|
void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
|
|
|
|
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
|
|
rwsem_set_owner(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(_down_write_nest_lock);
|
|
|
|
void down_read_non_owner(struct rw_semaphore *sem)
|
|
{
|
|
might_sleep();
|
|
|
|
__down_read(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_read_non_owner);
|
|
|
|
void down_write_nested(struct rw_semaphore *sem, int subclass)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
|
|
|
|
LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
|
|
rwsem_set_owner(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_write_nested);
|
|
|
|
int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
|
|
{
|
|
might_sleep();
|
|
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
|
|
|
|
if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
|
|
rwsem_release(&sem->dep_map, 1, _RET_IP_);
|
|
return -EINTR;
|
|
}
|
|
|
|
rwsem_set_owner(sem);
|
|
return 0;
|
|
}
|
|
|
|
EXPORT_SYMBOL(down_write_killable_nested);
|
|
|
|
void up_read_non_owner(struct rw_semaphore *sem)
|
|
{
|
|
DEBUG_RWSEMS_WARN_ON(sem->owner != RWSEM_READER_OWNED);
|
|
__up_read(sem);
|
|
}
|
|
|
|
EXPORT_SYMBOL(up_read_non_owner);
|
|
|
|
#endif
|