69f9cae909
When we allow for a max NR_CPUS < 2^14 we can optimize the pending wait-acquire and the xchg_tail() operations. By growing the pending bit to a byte, we reduce the tail to 16bit. This means we can use xchg16 for the tail part and do away with all the repeated compxchg() operations. This in turn allows us to unconditionally acquire; the locked state as observed by the wait loops cannot change. And because both locked and pending are now a full byte we can use simple stores for the state transition, obviating one atomic operation entirely. This optimization is needed to make the qspinlock achieve performance parity with ticket spinlock at light load. All this is horribly broken on Alpha pre EV56 (and any other arch that cannot do single-copy atomic byte stores). Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Waiman Long <Waiman.Long@hp.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Daniel J Blueman <daniel@numascale.com> Cc: David Vrabel <david.vrabel@citrix.com> Cc: Douglas Hatch <doug.hatch@hp.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Paolo Bonzini <paolo.bonzini@gmail.com> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com> Cc: Rik van Riel <riel@redhat.com> Cc: Scott J Norton <scott.norton@hp.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: virtualization@lists.linux-foundation.org Cc: xen-devel@lists.xenproject.org Link: http://lkml.kernel.org/r/1429901803-29771-6-git-send-email-Waiman.Long@hp.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
80 lines
2.2 KiB
C
80 lines
2.2 KiB
C
/*
|
|
* Queued spinlock
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
|
|
*
|
|
* Authors: Waiman Long <waiman.long@hp.com>
|
|
*/
|
|
#ifndef __ASM_GENERIC_QSPINLOCK_TYPES_H
|
|
#define __ASM_GENERIC_QSPINLOCK_TYPES_H
|
|
|
|
/*
|
|
* Including atomic.h with PARAVIRT on will cause compilation errors because
|
|
* of recursive header file incluson via paravirt_types.h. So don't include
|
|
* it if PARAVIRT is on.
|
|
*/
|
|
#ifndef CONFIG_PARAVIRT
|
|
#include <linux/types.h>
|
|
#include <linux/atomic.h>
|
|
#endif
|
|
|
|
typedef struct qspinlock {
|
|
atomic_t val;
|
|
} arch_spinlock_t;
|
|
|
|
/*
|
|
* Bitfields in the atomic value:
|
|
*
|
|
* When NR_CPUS < 16K
|
|
* 0- 7: locked byte
|
|
* 8: pending
|
|
* 9-15: not used
|
|
* 16-17: tail index
|
|
* 18-31: tail cpu (+1)
|
|
*
|
|
* When NR_CPUS >= 16K
|
|
* 0- 7: locked byte
|
|
* 8: pending
|
|
* 9-10: tail index
|
|
* 11-31: tail cpu (+1)
|
|
*/
|
|
#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
|
|
<< _Q_ ## type ## _OFFSET)
|
|
#define _Q_LOCKED_OFFSET 0
|
|
#define _Q_LOCKED_BITS 8
|
|
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
|
|
|
|
#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
|
|
#if CONFIG_NR_CPUS < (1U << 14)
|
|
#define _Q_PENDING_BITS 8
|
|
#else
|
|
#define _Q_PENDING_BITS 1
|
|
#endif
|
|
#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
|
|
|
|
#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
|
|
#define _Q_TAIL_IDX_BITS 2
|
|
#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
|
|
|
|
#define _Q_TAIL_CPU_OFFSET (_Q_TAIL_IDX_OFFSET + _Q_TAIL_IDX_BITS)
|
|
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
|
|
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
|
|
|
|
#define _Q_TAIL_OFFSET _Q_TAIL_IDX_OFFSET
|
|
#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
|
|
|
|
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
|
|
#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
|
|
|
|
#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
|