![Waiman Long](/assets/img/avatar_default.png)
This patch introduces a new generic queued spinlock implementation that can serve as an alternative to the default ticket spinlock. Compared with the ticket spinlock, this queued spinlock should be almost as fair as the ticket spinlock. It has about the same speed in single-thread and it can be much faster in high contention situations especially when the spinlock is embedded within the data structure to be protected. Only in light to moderate contention where the average queue depth is around 1-3 will this queued spinlock be potentially a bit slower due to the higher slowpath overhead. This queued spinlock is especially suit to NUMA machines with a large number of cores as the chance of spinlock contention is much higher in those machines. The cost of contention is also higher because of slower inter-node memory traffic. Due to the fact that spinlocks are acquired with preemption disabled, the process will not be migrated to another CPU while it is trying to get a spinlock. Ignoring interrupt handling, a CPU can only be contending in one spinlock at any one time. Counting soft IRQ, hard IRQ and NMI, a CPU can only have a maximum of 4 concurrent lock waiting activities. By allocating a set of per-cpu queue nodes and used them to form a waiting queue, we can encode the queue node address into a much smaller 24-bit size (including CPU number and queue node index) leaving one byte for the lock. Please note that the queue node is only needed when waiting for the lock. Once the lock is acquired, the queue node can be released to be used later. Signed-off-by: Waiman Long <Waiman.Long@hp.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Daniel J Blueman <daniel@numascale.com> Cc: David Vrabel <david.vrabel@citrix.com> Cc: Douglas Hatch <doug.hatch@hp.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Paolo Bonzini <paolo.bonzini@gmail.com> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com> Cc: Rik van Riel <riel@redhat.com> Cc: Scott J Norton <scott.norton@hp.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: virtualization@lists.linux-foundation.org Cc: xen-devel@lists.xenproject.org Link: http://lkml.kernel.org/r/1429901803-29771-2-git-send-email-Waiman.Long@hp.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
251 lines
4.8 KiB
Plaintext
251 lines
4.8 KiB
Plaintext
#
|
|
# The ARCH_INLINE foo is necessary because select ignores "depends on"
|
|
#
|
|
config ARCH_INLINE_SPIN_TRYLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_TRYLOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_LOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_LOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_LOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_LOCK_IRQSAVE
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_UNLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_UNLOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_UNLOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE
|
|
bool
|
|
|
|
|
|
config ARCH_INLINE_READ_TRYLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_LOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_LOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_LOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_LOCK_IRQSAVE
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_UNLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_UNLOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_UNLOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_READ_UNLOCK_IRQRESTORE
|
|
bool
|
|
|
|
|
|
config ARCH_INLINE_WRITE_TRYLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_LOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_LOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_LOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_LOCK_IRQSAVE
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_UNLOCK
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_UNLOCK_BH
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_UNLOCK_IRQ
|
|
bool
|
|
|
|
config ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE
|
|
bool
|
|
|
|
config UNINLINE_SPIN_UNLOCK
|
|
bool
|
|
|
|
#
|
|
# lock_* functions are inlined when:
|
|
# - DEBUG_SPINLOCK=n and GENERIC_LOCKBREAK=n and ARCH_INLINE_*LOCK=y
|
|
#
|
|
# trylock_* functions are inlined when:
|
|
# - DEBUG_SPINLOCK=n and ARCH_INLINE_*LOCK=y
|
|
#
|
|
# unlock and unlock_irq functions are inlined when:
|
|
# - DEBUG_SPINLOCK=n and ARCH_INLINE_*LOCK=y
|
|
# or
|
|
# - DEBUG_SPINLOCK=n and PREEMPT=n
|
|
#
|
|
# unlock_bh and unlock_irqrestore functions are inlined when:
|
|
# - DEBUG_SPINLOCK=n and ARCH_INLINE_*LOCK=y
|
|
#
|
|
|
|
if !DEBUG_SPINLOCK
|
|
|
|
config INLINE_SPIN_TRYLOCK
|
|
def_bool y
|
|
depends on ARCH_INLINE_SPIN_TRYLOCK
|
|
|
|
config INLINE_SPIN_TRYLOCK_BH
|
|
def_bool y
|
|
depends on ARCH_INLINE_SPIN_TRYLOCK_BH
|
|
|
|
config INLINE_SPIN_LOCK
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_SPIN_LOCK
|
|
|
|
config INLINE_SPIN_LOCK_BH
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_SPIN_LOCK_BH
|
|
|
|
config INLINE_SPIN_LOCK_IRQ
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_SPIN_LOCK_IRQ
|
|
|
|
config INLINE_SPIN_LOCK_IRQSAVE
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_SPIN_LOCK_IRQSAVE
|
|
|
|
config INLINE_SPIN_UNLOCK_BH
|
|
def_bool y
|
|
depends on ARCH_INLINE_SPIN_UNLOCK_BH
|
|
|
|
config INLINE_SPIN_UNLOCK_IRQ
|
|
def_bool y
|
|
depends on !PREEMPT || ARCH_INLINE_SPIN_UNLOCK_IRQ
|
|
|
|
config INLINE_SPIN_UNLOCK_IRQRESTORE
|
|
def_bool y
|
|
depends on ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE
|
|
|
|
|
|
config INLINE_READ_TRYLOCK
|
|
def_bool y
|
|
depends on ARCH_INLINE_READ_TRYLOCK
|
|
|
|
config INLINE_READ_LOCK
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_READ_LOCK
|
|
|
|
config INLINE_READ_LOCK_BH
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_READ_LOCK_BH
|
|
|
|
config INLINE_READ_LOCK_IRQ
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_READ_LOCK_IRQ
|
|
|
|
config INLINE_READ_LOCK_IRQSAVE
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_READ_LOCK_IRQSAVE
|
|
|
|
config INLINE_READ_UNLOCK
|
|
def_bool y
|
|
depends on !PREEMPT || ARCH_INLINE_READ_UNLOCK
|
|
|
|
config INLINE_READ_UNLOCK_BH
|
|
def_bool y
|
|
depends on ARCH_INLINE_READ_UNLOCK_BH
|
|
|
|
config INLINE_READ_UNLOCK_IRQ
|
|
def_bool y
|
|
depends on !PREEMPT || ARCH_INLINE_READ_UNLOCK_IRQ
|
|
|
|
config INLINE_READ_UNLOCK_IRQRESTORE
|
|
def_bool y
|
|
depends on ARCH_INLINE_READ_UNLOCK_IRQRESTORE
|
|
|
|
|
|
config INLINE_WRITE_TRYLOCK
|
|
def_bool y
|
|
depends on ARCH_INLINE_WRITE_TRYLOCK
|
|
|
|
config INLINE_WRITE_LOCK
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_WRITE_LOCK
|
|
|
|
config INLINE_WRITE_LOCK_BH
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_WRITE_LOCK_BH
|
|
|
|
config INLINE_WRITE_LOCK_IRQ
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_WRITE_LOCK_IRQ
|
|
|
|
config INLINE_WRITE_LOCK_IRQSAVE
|
|
def_bool y
|
|
depends on !GENERIC_LOCKBREAK && ARCH_INLINE_WRITE_LOCK_IRQSAVE
|
|
|
|
config INLINE_WRITE_UNLOCK
|
|
def_bool y
|
|
depends on !PREEMPT || ARCH_INLINE_WRITE_UNLOCK
|
|
|
|
config INLINE_WRITE_UNLOCK_BH
|
|
def_bool y
|
|
depends on ARCH_INLINE_WRITE_UNLOCK_BH
|
|
|
|
config INLINE_WRITE_UNLOCK_IRQ
|
|
def_bool y
|
|
depends on !PREEMPT || ARCH_INLINE_WRITE_UNLOCK_IRQ
|
|
|
|
config INLINE_WRITE_UNLOCK_IRQRESTORE
|
|
def_bool y
|
|
depends on ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE
|
|
|
|
endif
|
|
|
|
config ARCH_SUPPORTS_ATOMIC_RMW
|
|
bool
|
|
|
|
config MUTEX_SPIN_ON_OWNER
|
|
def_bool y
|
|
depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW
|
|
|
|
config RWSEM_SPIN_ON_OWNER
|
|
def_bool y
|
|
depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW
|
|
|
|
config LOCK_SPIN_ON_OWNER
|
|
def_bool y
|
|
depends on MUTEX_SPIN_ON_OWNER || RWSEM_SPIN_ON_OWNER
|
|
|
|
config ARCH_USE_QUEUED_SPINLOCK
|
|
bool
|
|
|
|
config QUEUED_SPINLOCK
|
|
def_bool y if ARCH_USE_QUEUED_SPINLOCK
|
|
depends on SMP && !PARAVIRT_SPINLOCKS
|
|
|
|
config ARCH_USE_QUEUE_RWLOCK
|
|
bool
|
|
|
|
config QUEUE_RWLOCK
|
|
def_bool y if ARCH_USE_QUEUE_RWLOCK
|
|
depends on SMP
|